diff options
Diffstat (limited to 'lib')
299 files changed, 3403 insertions, 1499 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index b23b11d0c29..5e449022676 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -254,6 +254,7 @@ module API mount ::API::NugetProjectPackages mount ::API::PackageFiles mount ::API::Pages + mount ::API::PagesDomains mount ::API::PersonalAccessTokens::SelfInformation mount ::API::PersonalAccessTokens mount ::API::ProjectClusters @@ -296,6 +297,7 @@ module API mount ::API::UsageData mount ::API::UsageDataNonSqlMetrics mount ::API::UsageDataQueries + mount ::API::Users mount ::API::UserCounts mount ::API::Wikis @@ -318,7 +320,6 @@ module API mount ::API::Labels mount ::API::Notes mount ::API::NotificationSettings - mount ::API::PagesDomains mount ::API::ProjectEvents mount ::API::ProjectMilestones mount ::API::ProtectedTags @@ -333,7 +334,6 @@ module API mount ::API::Todos mount ::API::UsageData mount ::API::UsageDataNonSqlMetrics - mount ::API::Users mount ::API::Ml::Mlflow end diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index 2cef1b27504..99278bdf8b0 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -26,10 +26,11 @@ module API end params do optional :title, type: String, desc: 'Instance title on the sign in / sign up page' - optional :short_title, type: String, desc: 'Short title for Progressive Web App' + optional :pwa_short_name, type: String, desc: 'Optional, short name for Progressive Web App' optional :description, type: String, desc: 'Markdown text shown on the sign in / sign up page' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 optional :logo, type: File, desc: 'Instance image used on the sign in / sign up page' # rubocop:disable Scalability/FileUploads + optional :pwa_icon, type: File, desc: 'Icon used for Progressive Web App' # rubocop:disable Scalability/FileUploads optional :header_logo, type: File, desc: 'Instance image used for the main navigation bar' # rubocop:disable Scalability/FileUploads optional :favicon, type: File, desc: 'Instance favicon in .ico/.png format' # rubocop:disable Scalability/FileUploads optional :new_project_guidelines, type: String, desc: 'Markdown text shown on the new project page' diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 845e42c2ed8..5ae1a80a7fd 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -24,6 +24,7 @@ module API helpers do params :filter_params do optional :search, type: String, desc: 'Return list of branches matching the search criteria' + optional :regex, type: String, desc: 'Return list of branches matching the regex' optional :sort, type: String, desc: 'Return list of branches sorted by the given field', values: %w[name_asc updated_asc updated_desc] end end diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index a28db321348..6c07b15329e 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -33,7 +33,7 @@ module API end before do - not_found! unless ::BulkImports::Features.enabled? + not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? authenticate! end @@ -61,12 +61,30 @@ module API type: String, desc: 'Source entity type (only `group_entity` is supported)', values: %w[group_entity] - requires :source_full_path, type: String, desc: 'Source full path of the entity to import' - requires :destination_namespace, type: String, desc: 'Destination namespace for the entity' - optional :destination_slug, type: String, desc: 'Destination slug for the entity' + requires :source_full_path, + type: String, + desc: 'Relative path of the source entity to import', + source_full_path: true, + documentation: { example: "'source/full/path' not 'https://example.com/source/full/path'" } + requires :destination_namespace, + type: String, + desc: 'Destination namespace for the entity', + destination_namespace_path: true, + documentation: { example: "'destination_namespace' or 'destination/namespace'" } + optional :destination_slug, + type: String, + desc: 'Destination slug for the entity', + destination_slug_path: true, + documentation: { example: "'destination_slug' not 'destination/slug'" } optional :destination_name, type: String, - desc: 'Deprecated: Use :destination_slug instead. Destination slug for the entity' + desc: 'Deprecated: Use :destination_slug instead. Destination slug for the entity', + destination_slug_path: true, + documentation: { example: "'destination_slug' not 'destination/slug'" } + optional :migrate_projects, + type: Boolean, + default: true, + desc: 'Indicates group migration should include nested projects' mutually_exclusive :destination_slug, :destination_name at_least_one_of :destination_slug, :destination_name diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index bb57a717f7c..ed1c7dfbfa2 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -53,11 +53,15 @@ module API authorize_read_builds! - builds = user_project.builds.order('id DESC') + builds = user_project.builds.order(id: :desc) builds = filter_builds(builds, params[:scope]) builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) - present paginate(builds, without_count: true), with: Entities::Ci::Job + if Feature.enabled?(:jobs_api_keyset_pagination, user_project) + present paginate_with_strategies(builds, paginator_params: { without_count: true }), with: Entities::Ci::Job + else + present paginate(builds, without_count: true), with: Entities::Ci::Job + end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index b073eb49bf1..6b4394114df 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -312,6 +312,7 @@ module API optional :artifact_format, type: String, desc: %q(The format of artifact), default: 'zip', values: ::Ci::JobArtifact.file_formats.keys optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware)), documentation: { type: 'file' } + optional :accessibility, type: String, desc: %q(Specify accessibility level of artifact private/public) end post '/:id/artifacts', feature_category: :build_artifacts, urgency: :low do not_allowed! unless Gitlab.config.artifacts.enabled diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb index 76b996f2301..6fe3f432edb 100644 --- a/lib/api/concerns/packages/debian_distribution_endpoints.rb +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -80,10 +80,10 @@ module API use :optional_distribution_params end post '/' do - authorize_create_package!(project_or_group) + authorize_create_package!(project_or_group(:read_project)) distribution_params = declared_params(include_missing: false) - result = ::Packages::Debian::CreateDistributionService.new(project_or_group, current_user, distribution_params).execute + result = ::Packages::Debian::CreateDistributionService.new(project_or_group(:read_project), current_user, distribution_params).execute created_distribution = result.payload[:distribution] if result.success? @@ -183,7 +183,7 @@ module API use :optional_distribution_params end put '/:codename' do - authorize_create_package!(project_or_group) + authorize_create_package!(project_or_group(:read_project)) distribution_params = declared_params(include_missing: false).except(:codename) result = ::Packages::Debian::UpdateDistributionService.new(distribution, distribution_params).execute @@ -214,7 +214,7 @@ module API use :optional_distribution_params end delete '/:codename' do - authorize_destroy_package!(project_or_group) + authorize_destroy_package!(project_or_group(:read_project)) accepted! if distribution.destroy diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 842250d351b..181759a7f38 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -35,10 +35,10 @@ module API ::Packages::Debian::DistributionsFinder.new(container, codename_or_suite: params[:distribution]).execute.last! end - def present_distribution_package_file! + def present_distribution_package_file!(project) not_found! unless params[:package_name].start_with?(params[:letter]) - package_file = distribution_from!(user_project).package_files.with_file_name(params[:file_name]).last! + package_file = distribution_from!(project).package_files.with_file_name(params[:file_name]).last! present_package_file!(package_file) end diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index 31ecb529c3c..5f32f0544f4 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -64,7 +64,7 @@ module API tags %w[nuget_packages] end get 'index', format: :json, urgency: :default do - authorize_read_package!(project_or_group) + authorize_packages_access!(project_or_group, required_permission) track_package_event('cli_metadata', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) @@ -78,7 +78,7 @@ module API end namespace '/metadata/*package_name' do after_validation do - authorize_read_package!(project_or_group) + authorize_packages_access!(project_or_group, required_permission) end desc 'The NuGet Metadata Service - Package name level' do @@ -124,7 +124,7 @@ module API end namespace '/query' do after_validation do - authorize_read_package!(project_or_group) + authorize_packages_access!(project_or_group, required_permission) end desc 'The NuGet Search Service' do diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 105a0955912..483d0dd9c90 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -12,10 +12,6 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do - def user_project - @project ||= find_project!(params[:project_id]) - end - def project_or_group user_group end @@ -55,7 +51,7 @@ module API route_setting :authentication, authenticate_non_public: true get 'pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do - present_distribution_package_file! + present_distribution_package_file!(find_project!(params[:project_id])) end end end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 23a542e4183..353f64b8dd1 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -21,16 +21,16 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do helpers do def project_or_group - user_project + user_project(action: :read_package) end end after_validation do require_packages_enabled! - not_found! unless ::Feature.enabled?(:debian_packages, user_project) + not_found! unless ::Feature.enabled?(:debian_packages, project_or_group) - authorize_read_package! + authorize_read_package!(project_or_group) end params do @@ -58,7 +58,7 @@ module API route_setting :authentication, authenticate_non_public: true get 'pool/:distribution/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do - present_distribution_package_file! + present_distribution_package_file!(project_or_group) end params do diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index d3a25a076a0..768ffac41ce 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -254,7 +254,7 @@ module API def readable_discussion_notes(noteable, discussion_ids) notes = noteable.notes .with_discussion_ids(discussion_ids) - .inc_relations_for_view + .inc_relations_for_view(noteable) .includes(:noteable) .fresh diff --git a/lib/api/entities/appearance.rb b/lib/api/entities/appearance.rb index 94a39568393..cabdf68c23a 100644 --- a/lib/api/entities/appearance.rb +++ b/lib/api/entities/appearance.rb @@ -4,13 +4,17 @@ module API module Entities class Appearance < Grape::Entity expose :title - expose :short_title + expose :pwa_short_name expose :description expose :logo do |appearance, options| appearance.logo.url end + expose :pwa_icon do |appearance, options| + appearance.pwa_icon.url + end + expose :header_logo do |appearance, options| appearance.header_logo.url end diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index db51d4380d0..8aace9126d6 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -43,6 +43,11 @@ module API # This field is deprecated and always returns true expose(:housekeeping_bitmaps_enabled) { |_settings, _options| true } + + # These fields are deprecated and always returns housekeeping_optimize_repository_period value + expose(:housekeeping_full_repack_period) { |settings, _options| settings.housekeeping_optimize_repository_period } + expose(:housekeeping_gc_period) { |settings, _options| settings.housekeeping_optimize_repository_period } + expose(:housekeeping_incremental_repack_period) { |settings, _options| settings.housekeeping_optimize_repository_period } end end end diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index 2585b2d0b6d..f89e5adca6d 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -15,7 +15,10 @@ module API expose :ssh_url_to_repo, documentation: { type: 'string', example: 'git@gitlab.example.com:gitlab/gitlab.git' } expose :http_url_to_repo, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab.git' } expose :web_url, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab' } - expose :readme_url, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab/blob/master/README.md' } + with_options if: ->(_, _) { user_has_access_to_project_repository? } do + expose :readme_url, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab/blob/master/README.md' } + expose :forks_count, documentation: { type: 'integer', example: 1 } + end expose :license_url, if: :license, documentation: { type: 'string', example: 'https://gitlab.example.com/gitlab/gitlab/blob/master/LICENCE' } do |project| license = project.repository.license_blob @@ -33,7 +36,6 @@ module API project.avatar_url(only_path: false) end - expose :forks_count, documentation: { type: 'integer', example: 1 } expose :star_count, documentation: { type: 'integer', example: 1 } expose :last_activity_at, documentation: { type: 'dateTime', example: '2013-09-30T13:46:02Z' } expose :namespace, using: 'API::Entities::NamespaceBasic' @@ -74,6 +76,10 @@ module API project.topics.pluck(:name).sort # rubocop:disable CodeReuse/ActiveRecord end end + + def user_has_access_to_project_repository? + Ability.allowed?(options[:current_user], :read_code, project) + end end end end diff --git a/lib/api/entities/bulk_imports/entity.rb b/lib/api/entities/bulk_imports/entity.rb index 8f9fbe57935..176d10b2580 100644 --- a/lib/api/entities/bulk_imports/entity.rb +++ b/lib/api/entities/bulk_imports/entity.rb @@ -9,7 +9,11 @@ module API expose :status_name, as: :status, documentation: { type: 'string', example: 'created', values: %w[created started finished timeout failed] } + expose :entity_type, documentation: { type: 'string', values: %w[group project] } expose :source_full_path, documentation: { type: 'string', example: 'source_group' } + expose :full_path, as: :destination_full_path, documentation: { + type: 'string', example: 'some_group/source_project' + } expose :destination_name, documentation: { type: 'string', example: 'destination_slug' } # deprecated expose :destination_slug, documentation: { type: 'string', example: 'destination_slug' } expose :destination_namespace, documentation: { type: 'string', example: 'destination_path' } @@ -19,6 +23,7 @@ module API expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } expose :failures, using: EntityFailure, documentation: { is_array: true } + expose :migrate_projects, documentation: { type: 'boolean', example: true } end end end diff --git a/lib/api/entities/ml/mlflow/run_info.rb b/lib/api/entities/ml/mlflow/run_info.rb index d3934545ba4..60e4416e011 100644 --- a/lib/api/entities/ml/mlflow/run_info.rb +++ b/lib/api/entities/ml/mlflow/run_info.rb @@ -10,6 +10,7 @@ module API expose(:experiment_id) { |candidate| candidate.experiment.iid.to_s } expose(:start_time) { |candidate| candidate.start_time || 0 } expose :end_time, expose_nil: false + expose :name, as: :run_name, expose_nil: false expose(:status) { |candidate| candidate.status.to_s.upcase } expose(:artifact_uri) { |candidate, options| "#{options[:packages_url]}#{candidate.artifact_root}" } expose(:lifecycle_stage) { |candidate| 'active' } diff --git a/lib/api/entities/project_integration_basic.rb b/lib/api/entities/project_integration_basic.rb index aa0ad158b83..b7c56d7cca1 100644 --- a/lib/api/entities/project_integration_basic.rb +++ b/lib/api/entities/project_integration_basic.rb @@ -14,6 +14,7 @@ module API expose :commit_events, documentation: { type: 'boolean' } expose :push_events, documentation: { type: 'boolean' } expose :issues_events, documentation: { type: 'boolean' } + expose :incident_events, documentation: { type: 'boolean' } expose :confidential_issues_events, documentation: { type: 'boolean' } expose :merge_requests_events, documentation: { type: 'boolean' } expose :tag_push_events, documentation: { type: 'boolean' } diff --git a/lib/api/entities/remote_mirror.rb b/lib/api/entities/remote_mirror.rb index 9fb5b2697bc..f64ec71bffb 100644 --- a/lib/api/entities/remote_mirror.rb +++ b/lib/api/entities/remote_mirror.rb @@ -16,3 +16,5 @@ module API end end end + +API::Entities::RemoteMirror.prepend_mod diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 01d46ee7bfb..64510a9615a 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -12,6 +12,8 @@ module API feature_category :continuous_delivery urgency :low + MIN_SEARCH_LENGTH = 3 + params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end @@ -29,7 +31,7 @@ module API params do use :pagination optional :name, type: String, desc: 'Return the environment with this name. Mutually exclusive with search' - optional :search, type: String, desc: 'Return list of environments matching the search criteria. Mutually exclusive with name' + optional :search, type: String, desc: "Return list of environments matching the search criteria. Mutually exclusive with name. Must be at least #{MIN_SEARCH_LENGTH} characters." optional :states, type: String, values: Environment.valid_states.map(&:to_s), @@ -39,6 +41,10 @@ module API get ':id/environments' do authorize! :read_environment, user_project + if Feature.enabled?(:environment_search_api_min_chars, user_project) && params[:search].present? && params[:search].length < MIN_SEARCH_LENGTH + bad_request!("Search query is less than #{MIN_SEARCH_LENGTH} characters") + end + environments = ::Environments::EnvironmentsFinder.new(user_project, current_user, declared_params(include_missing: false)).execute present paginate(environments), with: Entities::Environment, current_user: current_user @@ -182,6 +188,35 @@ module API present environment, with: Entities::Environment, current_user: current_user end + desc 'Stop stale environments' do + detail 'It returns `200` if stale environment check was scheduled successfully' + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' } + ] + tags %w[environments] + end + params do + requires :before, + type: DateTime, + desc: 'Stop all environments that were last modified or deployed to before this date.' + end + post ':id/environments/stop_stale' do + authorize! :stop_environment, user_project + + bad_request!('Invalid Date') if params[:before] < 10.years.ago || params[:before] > 1.week.ago + + service_response = ::Environments::StopStaleService.new(user_project, current_user, params.slice(:before)).execute + + if service_response.error? + status 400 + else + status 200 + end + + present message: service_response.message + end + desc 'Get a specific environment' do success Entities::Environment failure [ diff --git a/lib/api/files.rb b/lib/api/files.rb index b02f1a8728b..18638abd184 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -151,19 +151,6 @@ module API present blame_ranges, with: Entities::BlameRange end - desc 'Get raw file metadata from repository' - params do - requires :file_path, type: String, file_path: true, - desc: 'The url encoded path to the file.', documentation: { example: 'lib%2Fclass%2Erb' } - optional :ref, type: String, - desc: 'The name of branch, tag or commit', allow_blank: false, documentation: { example: 'main' } - end - head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS, urgency: :low do - assign_file_vars! - - set_http_headers(blob_data) - end - desc 'Get raw file contents from the repository' do success File end diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb index 0364e2e7b56..8b6d4b8c4b2 100644 --- a/lib/api/group_debian_distributions.rb +++ b/lib/api/group_debian_distributions.rb @@ -19,7 +19,7 @@ module API namespace ':id/-' do helpers do - def project_or_group + def project_or_group(_ = nil) user_group end end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index eb0a01e0d3d..37dfbfdb925 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -64,67 +64,73 @@ module API end end - desc 'Start relations export' do - detail 'This feature was introduced in GitLab 13.12' - tags %w[group_export] - success code: 202 - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' }, - { code: 503, message: 'Service unavailable' } - ] - end - post ':id/export_relations' do - response = ::BulkImports::ExportService.new(portable: user_group, user: current_user).execute + resource do + before do + not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? + end - if response.success? - accepted! - else - render_api_error!(message: 'Group relations export could not be started.') + desc 'Start relations export' do + detail 'This feature was introduced in GitLab 13.12' + tags %w[group_export] + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end - end + post ':id/export_relations' do + response = ::BulkImports::ExportService.new(portable: user_group, user: current_user).execute - desc 'Download relations export' do - detail 'This feature was introduced in GitLab 13.12' - produces %w[application/octet-stream application/json] - tags %w[group_export] - success code: 200 - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' }, - { code: 503, message: 'Service unavailable' } - ] - end - params do - requires :relation, type: String, desc: 'Group relation name' - end - get ':id/export_relations/download' do - export = user_group.bulk_import_exports.find_by_relation(params[:relation]) - file = export&.upload&.export_file + if response.success? + accepted! + else + render_api_error!(message: 'Group relations export could not be started.') + end + end - if file - present_carrierwave_file!(file) - else - render_api_error!('404 Not found', 404) + desc 'Download relations export' do + detail 'This feature was introduced in GitLab 13.12' + produces %w[application/octet-stream application/json] + tags %w[group_export] + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] end - end + params do + requires :relation, type: String, desc: 'Group relation name' + end + get ':id/export_relations/download' do + export = user_group.bulk_import_exports.find_by_relation(params[:relation]) + file = export&.upload&.export_file - desc 'Relations export status' do - detail 'This feature was introduced in GitLab 13.12' - is_array true - tags %w[group_export] - success code: 200, model: Entities::BulkImports::ExportStatus - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' }, - { code: 503, message: 'Service unavailable' } - ] - end - get ':id/export_relations/status' do - present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + if file + present_carrierwave_file!(file) + else + render_api_error!('404 Not found', 404) + end + end + + desc 'Relations export status' do + detail 'This feature was introduced in GitLab 13.12' + is_array true + tags %w[group_export] + success code: 200, model: Entities::BulkImports::ExportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + end + get ':id/export_relations/status' do + present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 0b5a471ea12..38430aac455 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -608,6 +608,8 @@ module API if file.file_storage? present_disk_file!(file.path, file.filename) elsif supports_direct_download && file.class.direct_download_enabled? + return redirect(signed_head_url(file)) if head_request_on_aws_file?(file) + redirect(cdn_fronted_url(file)) else header(*Gitlab::Workhorse.send_url(file.url)) @@ -695,8 +697,31 @@ module API unprocessable_entity!('User must be authenticated to use search') end + def validate_search_rate_limit! + return unless Feature.enabled?(:rate_limit_issuable_searches) + + if current_user + check_rate_limit!(:search_rate_limit, scope: [current_user]) + else + check_rate_limit!(:search_rate_limit_unauthenticated, scope: [ip_address]) + end + end + private + def head_request_on_aws_file?(file) + request.head? && file.fog_credentials[:provider] == 'AWS' + end + + def signed_head_url(file) + fog_storage = ::Fog::Storage.new(file.fog_credentials) + fog_dir = fog_storage.directories.new(key: file.fog_directory) + fog_file = fog_dir.files.new(key: file.path) + expire_at = ::Fog::Time.now + file.fog_authenticated_url_expiration + + fog_file.collection.head_url(fog_file.key, expire_at) + end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def initial_current_user return @initial_current_user if defined?(@initial_current_user) diff --git a/lib/api/helpers/award_emoji.rb b/lib/api/helpers/award_emoji.rb index f8417366ea4..f0b3cafc3d2 100644 --- a/lib/api/helpers/award_emoji.rb +++ b/lib/api/helpers/award_emoji.rb @@ -6,7 +6,7 @@ module API def self.awardables [ { type: 'issue', resource: :projects, find_by: :iid, feature_category: :team_planning }, - { type: 'merge_request', resource: :projects, find_by: :iid, feature_category: :code_review }, + { type: 'merge_request', resource: :projects, find_by: :iid, feature_category: :code_review_workflow }, { type: 'snippet', resource: :projects, find_by: :id, feature_category: :source_code_management } ] end diff --git a/lib/api/helpers/discussions_helpers.rb b/lib/api/helpers/discussions_helpers.rb index 182ada54a12..d497bc66015 100644 --- a/lib/api/helpers/discussions_helpers.rb +++ b/lib/api/helpers/discussions_helpers.rb @@ -9,8 +9,8 @@ module API { Issue => :team_planning, Snippet => :source_code_management, - MergeRequest => :code_review, - Commit => :code_review + MergeRequest => :code_review_workflow, + Commit => :code_review_workflow } end end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 543449c0349..31328facd69 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -63,6 +63,12 @@ module API }, { required: false, + name: :incident_channel, + type: String, + desc: 'The name of the channel to receive incident_events notifications' + }, + { + required: false, name: :confidential_issue_channel, type: String, desc: 'The name of the channel to receive confidential_issues_events notifications' @@ -116,6 +122,12 @@ module API }, { required: false, + name: :incident_events, + type: Boolean, + desc: 'Enable notifications for incident_events' + }, + { + required: false, name: :confidential_issues_events, type: Boolean, desc: 'Enable notifications for confidential_issues_events' @@ -161,6 +173,26 @@ module API def self.integrations { + 'apple-app-store' => [ + { + required: true, + name: :app_store_issuer_id, + type: String, + desc: 'The Apple App Store Connect Issuer ID' + }, + { + required: true, + name: :app_store_key_id, + type: String, + desc: 'The Apple App Store Connect Key ID' + }, + { + required: true, + name: :app_store_private_key, + type: String, + desc: 'The Apple App Store Connect Private Key' + } + ], 'asana' => [ { required: true, @@ -871,6 +903,7 @@ module API def self.integration_classes [ + ::Integrations::AppleAppStore, ::Integrations::Asana, ::Integrations::Assembla, ::Integrations::Bamboo, diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 6a3cf5c87ae..b0ea4388d9b 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -40,6 +40,9 @@ module API end def source_members(source) + return source.namespace_members if source.is_a?(Project) && + Feature.enabled?(:project_members_index_by_project_namespace, source) + source.members end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 302dac4abf7..da499abe475 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -8,7 +8,7 @@ module API def self.feature_category_per_noteable_type { Issue => :team_planning, - MergeRequest => :code_review, + MergeRequest => :code_review_workflow, Snippet => :source_code_management } end diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 8d913268405..1d35c316913 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -6,6 +6,7 @@ module API extend ::Gitlab::Utils::Override MAX_PACKAGE_FILE_SIZE = 50.megabytes.freeze + ALLOWED_REQUIRED_PERMISSIONS = %i[read_package read_group].freeze def require_packages_enabled! not_found! unless ::Gitlab.config.packages.enabled @@ -27,9 +28,15 @@ module API authorize!(:destroy_package, subject) end - def authorize_packages_access!(subject = user_project) + def authorize_packages_access!(subject = user_project, required_permission = :read_package) require_packages_enabled! - authorize_read_package!(subject) + return forbidden! unless required_permission.in?(ALLOWED_REQUIRED_PERMISSIONS) + + if required_permission == :read_package + authorize_read_package!(subject) + else + authorize!(required_permission, subject) + end end def authorize_workhorse!(subject: user_project, has_length: true, maximum_size: MAX_PACKAGE_FILE_SIZE) diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb index 4e244ea589e..5fbc3081ee8 100644 --- a/lib/api/helpers/pagination_strategies.rb +++ b/lib/api/helpers/pagination_strategies.rb @@ -3,13 +3,14 @@ module API module Helpers module PaginationStrategies - def paginate_with_strategies(relation, request_scope = nil) + # paginator_params are only currently supported with offset pagination + def paginate_with_strategies(relation, request_scope = nil, paginator_params: {}) paginator = paginator(relation, request_scope) result = if block_given? - yield(paginator.paginate(relation)) + yield(paginator.paginate(relation, **paginator_params)) else - paginator.paginate(relation) + paginator.paginate(relation, **paginator_params) end result.tap do |records, _| diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 9d370176e62..c5636fa06de 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -205,6 +205,9 @@ module API def filter_attributes_using_license!(attrs) end + def filter_attributes_under_feature_flag!(attrs, project) + end + def validate_git_import_url!(import_url) return if import_url.blank? diff --git a/lib/api/helpers/rate_limiter.rb b/lib/api/helpers/rate_limiter.rb index 03f3cd649b1..be92277c25a 100644 --- a/lib/api/helpers/rate_limiter.rb +++ b/lib/api/helpers/rate_limiter.rb @@ -10,25 +10,14 @@ module API # See app/controllers/concerns/check_rate_limit.rb for Rails controllers version module RateLimiter def check_rate_limit!(key, scope:, **options) - return if bypass_header_set? - return unless rate_limiter.throttled?(key, scope: scope, **options) - - rate_limiter.log_request(request, "#{key}_request_limit".to_sym, current_user) + return unless Gitlab::ApplicationRateLimiter.throttled_request?( + request, current_user, key, scope: scope, **options + ) return yield if block_given? render_api_error!({ error: _('This endpoint has been requested too many times. Try again later.') }, 429) end - - private - - def rate_limiter - ::Gitlab::ApplicationRateLimiter - end - - def bypass_header_set? - ::Gitlab::Throttle.bypass_header.present? && request.get_header(Gitlab::Throttle.bypass_header) == '1' - end end end end diff --git a/lib/api/helpers/remote_mirrors_helpers.rb b/lib/api/helpers/remote_mirrors_helpers.rb new file mode 100644 index 00000000000..efd81a5ac5a --- /dev/null +++ b/lib/api/helpers/remote_mirrors_helpers.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module API + module Helpers + module RemoteMirrorsHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :mirror_branches_setting_ce do + optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored' + end + + params :mirror_branches_setting_ee do + end + + params :mirror_branches_setting do + use :mirror_branches_setting_ce + use :mirror_branches_setting_ee + end + + def verify_mirror_branches_setting(attrs, project); end + end + end +end + +API::Helpers::RemoteMirrorsHelpers.prepend_mod diff --git a/lib/api/helpers/resource_events_helpers.rb b/lib/api/helpers/resource_events_helpers.rb index c47a58e8fce..11cb65056cd 100644 --- a/lib/api/helpers/resource_events_helpers.rb +++ b/lib/api/helpers/resource_events_helpers.rb @@ -7,7 +7,7 @@ module API # This is a method instead of a constant, allowing EE to more easily extend it. { Issue => { feature_category: :team_planning, id_field: 'IID' }, - MergeRequest => { feature_category: :code_review, id_field: 'IID' } + MergeRequest => { feature_category: :code_review_workflow, id_field: 'IID' } } end end diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index d742e3732a8..6330a4458f3 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -8,6 +8,7 @@ module API urgency :low rescue_from Octokit::Unauthorized, with: :provider_unauthorized + rescue_from Gitlab::GithubImport::RateLimitError, with: :too_many_requests helpers do def client @@ -33,6 +34,10 @@ module API def provider_unauthorized error!("Access denied to your #{Gitlab::ImportSources.title(provider.to_s)} account.", 401) end + + def too_many_requests + error!('Too Many Requests', 429) + end end desc 'Import a GitHub project' do @@ -51,7 +56,7 @@ module API requires :personal_access_token, type: String, desc: 'GitHub personal access token' requires :repo_id, type: Integer, desc: 'GitHub repository ID' optional :new_name, type: String, desc: 'New repo name' - requires :target_namespace, type: String, desc: 'Namespace to import repo into' + requires :target_namespace, type: String, allow_blank: false, desc: 'Namespace or group to import repository into' optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname' optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' end @@ -92,5 +97,32 @@ module API render_api_error!(result[:message], result[:http_status]) end end + + desc 'Import User Gists' do + detail 'This feature was introduced in GitLab 15.8' + success code: 202 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 422, message: 'Unprocessable Entity' }, + { code: 429, message: 'Too Many Requests' } + ] + end + params do + requires :personal_access_token, type: String, desc: 'GitHub personal access token' + end + post 'import/github/gists' do + not_found! if Feature.disabled?(:github_import_gists) + + authorize! :create_snippet + + result = Import::Github::GistsImportService.new(current_user, client, access_params).execute + + if result[:status] == :success + status 202 + else + status result[:http_status] + { errors: result[:message] } + end + end end end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index dbd5c5f9db1..3f6e052f7b6 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -44,16 +44,12 @@ module API # This is a separate method so that EE can alter its behaviour more # easily. - if Feature.enabled?(:rate_limit_gitlab_shell) - check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], actor.key_or_user]) - end + check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], actor.key_or_user]) - if Feature.enabled?(:rate_limit_gitlab_shell_by_ip, actor.user) - rate_limiter = Gitlab::Auth::IpRateLimiter.new(request.ip) + rate_limiter = Gitlab::Auth::IpRateLimiter.new(request.ip) - unless rate_limiter.trusted_ip? - check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], rate_limiter.ip]) - end + unless rate_limiter.trusted_ip? + check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], rate_limiter.ip]) end # Stores some Git-specific env thread-safely diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 6aefdf146cf..872dab26469 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -90,7 +90,7 @@ module API .new(current_user, update_params) .execute(invite) - updated_member = result[:member] + updated_member = result[:members].first if result[:status] == :success present_members updated_member diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b08819e34e3..7b6306938cf 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -116,6 +116,7 @@ module API get '/issues_statistics' do authenticate! unless params[:scope] == 'all' validate_anonymous_search_access! if params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? present issues_statistics, with: Grape::Presenters::Presenter end @@ -134,6 +135,7 @@ module API get do authenticate! unless params[:scope] == 'all' validate_anonymous_search_access! if params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? issues = paginate(find_issues) options = { @@ -173,6 +175,7 @@ module API end get ":id/issues" do validate_anonymous_search_access! if declared_params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? issues = paginate(find_issues(group_id: user_group.id, include_subgroups: true)) options = { @@ -192,6 +195,7 @@ module API end get ":id/issues_statistics" do validate_anonymous_search_access! if declared_params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? present issues_statistics(group_id: user_group.id, include_subgroups: true), with: Grape::Presenters::Presenter end @@ -211,6 +215,7 @@ module API end get ":id/issues" do validate_anonymous_search_access! if declared_params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? issues = paginate(find_issues(project_id: user_project.id)) options = { @@ -230,6 +235,7 @@ module API end get ":id/issues_statistics" do validate_anonymous_search_access! if declared_params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? present issues_statistics(project_id: user_project.id), with: Grape::Presenters::Presenter end diff --git a/lib/api/members.rb b/lib/api/members.rb index 76f4364106b..32c5227a939 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -151,7 +151,7 @@ module API .new(current_user, declared_params(include_missing: false)) .execute(member) - updated_member = result[:member] + updated_member = result[:members].first if result[:status] == :success present_members updated_member diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index c7f0f88eacc..e7193035ce0 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - feature_category :code_review + feature_category :code_review_workflow params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a9572cf7ce6..25fbeca01dc 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -13,7 +13,7 @@ module API # These endpoints are defined in `TimeTrackingEndpoints` and is shared by # API::Issues. In order to be able to define the feature category of these # endpoints, we need to define them at the top-level by route. - feature_category :code_review, [ + feature_category :code_review_workflow, [ '/projects/:id/merge_requests/:merge_request_iid/time_estimate', '/projects/:id/merge_requests/:merge_request_iid/reset_time_estimate', '/projects/:id/merge_requests/:merge_request_iid/add_spent_time', @@ -105,20 +105,12 @@ module API options end - def authorize_push_to_merge_request!(merge_request) - forbidden!('Source branch does not exist') unless - merge_request.source_branch_exists? + def authorize_merge_request_rebase!(merge_request) + result = ::MergeRequests::RebaseService + .new(project: merge_request.source_project, current_user: current_user) + .validate(merge_request) - user_access = Gitlab::UserAccess.new( - current_user, - container: merge_request.source_project - ) - - forbidden!('Cannot push to source branch') unless - user_access.can_push_to_branch?(merge_request.source_branch) - - forbidden!('Source branch is protected from force push') unless - merge_request.permits_force_push? + forbidden!(result.message) if result.error? end def recheck_mergeability_of(merge_requests:) @@ -146,9 +138,10 @@ module API use :merge_requests_params use :optional_scope_param end - get feature_category: :code_review, urgency: :low do + get feature_category: :code_review_workflow, urgency: :low do authenticate! unless params[:scope] == 'all' validate_anonymous_search_access! if params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests present merge_requests, serializer_options_for(merge_requests) @@ -175,8 +168,9 @@ module API default: true, desc: 'Returns merge requests from non archived projects only.' end - get ":id/merge_requests", feature_category: :code_review, urgency: :low do + get ":id/merge_requests", feature_category: :code_review_workflow, urgency: :low do validate_anonymous_search_access! if declared_params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) present merge_requests, serializer_options_for(merge_requests).merge(group: user_group) @@ -241,9 +235,10 @@ module API desc: 'Returns the request having the given `iid`.', documentation: { is_array: true } end - get ":id/merge_requests", feature_category: :code_review, urgency: :low do + get ":id/merge_requests", feature_category: :code_review_workflow, urgency: :low do authorize! :read_merge_request, user_project validate_anonymous_search_access! if declared_params[:search].present? + validate_search_rate_limit! if declared_params[:search].present? merge_requests = find_merge_requests(project_id: user_project.id) @@ -286,7 +281,7 @@ module API desc: 'The target project of the merge request defaults to the :id of the project.' use :optional_params end - post ":id/merge_requests", feature_category: :code_review, urgency: :low do + post ":id/merge_requests", feature_category: :code_review_workflow, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20770') authorize! :create_merge_request_from, user_project @@ -314,7 +309,7 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The internal ID of the merge request.' end - delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review, urgency: :low do + delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review_workflow, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) @@ -339,7 +334,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request, @@ -360,7 +355,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) participants = ::Kaminari.paginate_array(merge_request.visible_participants(current_user)) @@ -376,7 +371,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid/reviewers', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid/reviewers', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) reviewers = ::Kaminari.paginate_array(merge_request.merge_request_reviewers) @@ -392,7 +387,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) commits = @@ -410,7 +405,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review, urgency: :high do + get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review_workflow, urgency: :high do merge_request = find_merge_request_with_access(params[:merge_request_iid]) context_commits = paginate(merge_request.merge_request_context_commits).map(&:to_commit) @@ -434,7 +429,7 @@ module API ] tags %w[merge_requests] end - post ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do + post ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review_workflow do commit_ids = params[:commits] if commit_ids.size > CONTEXT_COMMITS_POST_LIMIT @@ -471,7 +466,7 @@ module API ] tags %w[merge_requests] end - delete ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do + delete ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review_workflow do commit_ids = params[:commits] merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -495,7 +490,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request, @@ -517,7 +512,7 @@ module API params do use :pagination end - get ':id/merge_requests/:merge_request_iid/diffs', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid/diffs', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) present paginate(merge_request.merge_request_diff.paginated_diffs(params[:page], params[:per_page])).diffs, with: Entities::Diff @@ -585,7 +580,7 @@ module API use :optional_params at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of) end - put ':id/merge_requests/:merge_request_iid', feature_category: :code_review, urgency: :low do + put ':id/merge_requests/:merge_request_iid', feature_category: :code_review_workflow, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20772') merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) @@ -627,7 +622,7 @@ module API optional :sha, type: String, desc: 'If present, then this SHA must match the HEAD of the source branch, otherwise the merge fails.' optional :squash, type: Grape::API::Boolean, desc: 'If `true`, the commits are squashed into a single commit on merge.' end - put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review, urgency: :low do + put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review_workflow, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796') merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -678,7 +673,7 @@ module API ] tags %w[merge_requests] end - get ':id/merge_requests/:merge_request_iid/merge_ref', feature_category: :code_review do + get ':id/merge_requests/:merge_request_iid/merge_ref', feature_category: :code_review_workflow do merge_request = find_project_merge_request(params[:merge_request_iid]) result = ::MergeRequests::MergeabilityCheckService.new(merge_request).execute(recheck: true) @@ -701,7 +696,7 @@ module API ] tags %w[merge_requests] end - post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds', feature_category: :code_review do + post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds', feature_category: :code_review_workflow do merge_request = find_project_merge_request(params[:merge_request_iid]) unauthorized! unless merge_request.can_cancel_auto_merge?(current_user) @@ -721,10 +716,10 @@ module API params do optional :skip_ci, type: Boolean, desc: 'Set to true to skip creating a CI pipeline.' end - put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review, urgency: :low do + put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review_workflow, urgency: :low do merge_request = find_project_merge_request(params[:merge_request_iid]) - authorize_push_to_merge_request!(merge_request) + authorize_merge_request_rebase!(merge_request) merge_request.rebase_async(current_user.id, skip_ci: params[:skip_ci]) @@ -744,7 +739,7 @@ module API params do use :pagination end - get ':id/merge_requests/:merge_request_iid/closes_issues', feature_category: :code_review, urgency: :low do + get ':id/merge_requests/:merge_request_iid/closes_issues', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user)) issues = paginate(issues) diff --git a/lib/api/ml/mlflow.rb b/lib/api/ml/mlflow.rb index 54bbe0ee465..e7ed8e2e70c 100644 --- a/lib/api/ml/mlflow.rb +++ b/lib/api/ml/mlflow.rb @@ -166,9 +166,10 @@ module API default: 0 optional :user_id, type: String, desc: 'This will be ignored' optional :tags, type: Array, desc: 'Tags are stored, but not displayed' + optional :run_name, type: String, desc: 'A name for this run' end post 'create', urgency: :low do - present candidate_repository.create!(experiment, params[:start_time], params[:tags]), + present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]), with: Entities::Ml::Mlflow::Run, packages_url: packages_url end diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index c93b24ee544..2afcb915b06 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -42,6 +42,10 @@ module API def snowplow_gitlab_standard_context { namespace: find_authorized_group! } end + + def required_permission + :read_group + end end params do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index aa517661791..8e974cb9cbe 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -90,6 +90,10 @@ module API created! end + + def required_permission + :read_package + end end params do diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 967847a8e62..15c1a78839f 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -54,7 +54,7 @@ module API end params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' + requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project owned by the authenticated user' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do @@ -63,6 +63,8 @@ module API desc 'Get all pages domains' do success Entities::PagesDomain + tags %w[pages_domains] + is_array true end params do use :pagination diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb index 1e27f5c8856..856b4097b5a 100644 --- a/lib/api/project_debian_distributions.rb +++ b/lib/api/project_debian_distributions.rb @@ -14,13 +14,13 @@ module API after_validation do require_packages_enabled! - not_found! unless ::Feature.enabled?(:debian_packages, user_project) + not_found! unless ::Feature.enabled?(:debian_packages, project_or_group) end namespace ':id' do helpers do - def project_or_group - user_project + def project_or_group(action = :read_package) + user_project(action: action) end end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index e4e950fb603..19e5ed3f9e0 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -5,109 +5,114 @@ module API feature_category :importers urgency :low - before do - not_found! unless Gitlab::CurrentSettings.project_export_enabled? - authorize_admin_project - end - params do requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' end resource :projects, requirements: { id: %r{[^/]+} } do - desc 'Get export status' do - detail 'This feature was introduced in GitLab 10.6.' - success code: 200, model: Entities::ProjectExportStatus - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' }, - { code: 503, message: 'Service unavailable' } - ] - tags ['project_export'] - end - get ':id/export' do - present user_project, with: Entities::ProjectExportStatus - end + resource do + before do + not_found! unless Gitlab::CurrentSettings.project_export_enabled? - desc 'Download export' do - detail 'This feature was introduced in GitLab 10.6.' - success code: 200 - failure [ - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' }, - { code: 503, message: 'Service unavailable' } - ] - tags ['project_export'] - produces %w[application/octet-stream application/json] - end - get ':id/export/download' do - check_rate_limit! :project_download_export, scope: [current_user, user_project.namespace] + authorize_admin_project + end + + desc 'Get export status' do + detail 'This feature was introduced in GitLab 10.6.' + success code: 200, model: Entities::ProjectExportStatus + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] + end + get ':id/export' do + present user_project, with: Entities::ProjectExportStatus + end - if user_project.export_file_exists? - if user_project.export_archive_exists? - present_carrierwave_file!(user_project.export_file) + desc 'Download export' do + detail 'This feature was introduced in GitLab 10.6.' + success code: 200 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] + produces %w[application/octet-stream application/json] + end + get ':id/export/download' do + check_rate_limit! :project_download_export, scope: [current_user, user_project.namespace] + + if user_project.export_file_exists? + if user_project.export_archive_exists? + present_carrierwave_file!(user_project.export_file) + else + render_api_error!('The project export file is not available yet', 404) + end else - render_api_error!('The project export file is not available yet', 404) + render_api_error!('404 Not found or has expired', 404) end - else - render_api_error!('404 Not found or has expired', 404) end - end - desc 'Start export' do - detail 'This feature was introduced in GitLab 10.6.' - success code: 202 - failure [ - { code: 400, message: 'Bad request' }, - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' }, - { code: 429, message: 'Too many requests' }, - { code: 503, message: 'Service unavailable' } - ] - tags ['project_export'] - end - params do - optional :description, type: String, desc: 'Override the project description' - optional :upload, type: Hash do - optional :url, type: String, desc: 'The URL to upload the project' - optional :http_method, type: String, default: 'PUT', values: %w[PUT POST], - desc: 'HTTP method to upload the exported project' + desc 'Start export' do + detail 'This feature was introduced in GitLab 10.6.' + success code: 202 + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' }, + { code: 429, message: 'Too many requests' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_export'] end - end - post ':id/export' do - check_rate_limit! :project_export, scope: current_user + params do + optional :description, type: String, desc: 'Override the project description' + optional :upload, type: Hash do + optional :url, type: String, desc: 'The URL to upload the project' + optional :http_method, type: String, default: 'PUT', values: %w[PUT POST], + desc: 'HTTP method to upload the exported project' + end + end + post ':id/export' do + check_rate_limit! :project_export, scope: current_user - user_project.remove_exports + user_project.remove_exports - project_export_params = declared_params(include_missing: false) - after_export_params = project_export_params.delete(:upload) || {} + project_export_params = declared_params(include_missing: false) + after_export_params = project_export_params.delete(:upload) || {} - export_strategy = if after_export_params[:url].present? - params = after_export_params.slice(:url, :http_method).symbolize_keys + export_strategy = if after_export_params[:url].present? + params = after_export_params.slice(:url, :http_method).symbolize_keys - Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(**params) - end + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(**params) + end - if export_strategy&.invalid? - render_validation_error!(export_strategy) - else - begin - user_project.add_export_job(current_user: current_user, - after_export_strategy: export_strategy, - params: project_export_params) - rescue Project::ExportLimitExceeded => e - render_api_error!(e.message, 400) + if export_strategy&.invalid? + render_validation_error!(export_strategy) + else + begin + user_project.add_export_job(current_user: current_user, + after_export_strategy: export_strategy, + params: project_export_params) + rescue Project::ExportLimitExceeded => e + render_api_error!(e.message, 400) + end end - end - accepted! + accepted! + end end resource do before do - not_found! unless ::Feature.enabled?(:bulk_import) + not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? + + authorize_admin_project end desc 'Start relations export' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index de39419b70b..5077f02fcc1 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -490,6 +490,7 @@ module API attrs = translate_params_for_compatibility(attrs) attrs = add_import_params(attrs) filter_attributes_using_license!(attrs) + filter_attributes_under_feature_flag!(attrs, user_project) verify_update_project_attrs!(user_project, attrs) user_project.remove_avatar! if attrs.key?(:avatar) && attrs[:avatar].nil? diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index 0e83d086a6e..b21bcb4a903 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -56,7 +56,7 @@ module API params do requires :name, type: String, desc: 'The name of the link. Link names must be unique in the release' requires :url, type: String, desc: 'The URL of the link. Link URLs must be unique in the release.' - optional :filepath, type: String, desc: 'Optional path for a direct asset link' + optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link', as: :filepath optional :link_type, type: String, values: %w[other runbook image package], @@ -108,7 +108,7 @@ module API params do optional :name, type: String, desc: 'The name of the link' optional :url, type: String, desc: 'The URL of the link' - optional :filepath, type: String, desc: 'Optional path for a direct asset link' + optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link', as: :filepath optional :link_type, type: String, values: %w[other runbook image package], diff --git a/lib/api/releases.rb b/lib/api/releases.rb index e6884e66200..e69dc756551 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -150,18 +150,19 @@ module API params do requires :tag_name, type: String, desc: 'The Git tag the release is associated with', as: :tag - requires :file_path, + requires :direct_asset_path, type: String, file_path: true, - desc: 'The path to the file to download, as specified when creating the release asset' + desc: 'The path to the file to download, as specified when creating the release asset', + as: :filepath end route_setting :authentication, job_token_allowed: true - get ':id/releases/:tag_name/downloads/*file_path', format: false, requirements: RELEASE_ENDPOINT_REQUIREMENTS do + get ':id/releases/:tag_name/downloads/*direct_asset_path', format: false, requirements: RELEASE_ENDPOINT_REQUIREMENTS do authorize_read_code! not_found! unless release - link = release.links.find_by_filepath!("/#{params[:file_path]}") + link = release.links.find_by_filepath!("/#{params[:filepath]}") not_found! unless link @@ -237,7 +238,7 @@ module API optional :links, type: Array do requires :name, type: String, desc: 'The name of the link. Link names must be unique within the release' requires :url, type: String, desc: 'The URL of the link. Link URLs must be unique within the release' - optional :filepath, type: String, desc: 'Optional path for a direct asset link' + optional :direct_asset_path, type: String, desc: 'Optional path for a direct asset link', as: :filepath optional :link_type, type: String, desc: 'The type of the link: `other`, `runbook`, `image`, `package`. Defaults to `other`' end end diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index f7ea5a6ad2b..c3c7d9370e0 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -3,6 +3,7 @@ module API class RemoteMirrors < ::API::Base include PaginationParams + helpers Helpers::RemoteMirrorsHelpers feature_category :source_code_management @@ -60,14 +61,13 @@ module API params do requires :url, type: String, desc: 'The URL for a remote mirror', documentation: { example: 'https://*****:*****@example.com/gitlab/example.git' } optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled', documentation: { example: false } - optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored', - documentation: { example: false } optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target', documentation: { example: false } + use :mirror_branches_setting end post ':id/remote_mirrors' do create_params = declared_params(include_missing: false) - + verify_mirror_branches_setting(create_params, user_project) new_mirror = user_project.remote_mirrors.create(create_params) if new_mirror.persisted? @@ -89,10 +89,9 @@ module API params do requires :mirror_id, type: String, desc: 'The ID of a remote mirror' optional :enabled, type: Boolean, desc: 'Determines if the mirror is enabled', documentation: { example: true } - optional :only_protected_branches, type: Boolean, desc: 'Determines if only protected branches are mirrored', - documentation: { example: false } optional :keep_divergent_refs, type: Boolean, desc: 'Determines if divergent refs are kept on the target', documentation: { example: false } + use :mirror_branches_setting end put ':id/remote_mirrors/:mirror_id' do mirror = user_project.remote_mirrors.find(params[:mirror_id]) @@ -100,6 +99,7 @@ module API mirror_params = declared_params(include_missing: false) mirror_params[:id] = mirror_params.delete(:mirror_id) + verify_mirror_branches_setting(mirror_params, user_project) update_params = { remote_mirrors_attributes: mirror_params } result = ::Projects::UpdateService diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb index 5640e88ae6e..3eff3e8ad36 100644 --- a/lib/api/resource_milestone_events.rb +++ b/lib/api/resource_milestone_events.rb @@ -11,7 +11,7 @@ module API { Issue => :team_planning, - MergeRequest => :code_review + MergeRequest => :code_review_workflow }.each do |eventable_type, feature_category| parent_type = eventable_type.parent_class.to_s.underscore eventables_str = eventable_type.to_s.underscore.pluralize diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index af0ceb1acfc..896d8fcc727 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -25,13 +25,19 @@ module API .sent_through(:http_token) end + helpers do + def project + user_project(action: :read_package) + end + end + before do require_packages_enabled! authenticate_non_get! end after_validation do - not_found! unless Feature.enabled?(:rubygem_packages, user_project) + not_found! unless Feature.enabled?(:rubygem_packages, project) end params do @@ -85,14 +91,14 @@ module API requires :file_name, type: String, desc: 'Package file name', documentation: { type: 'file' } end get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do - authorize_read_package!(user_project) + authorize_read_package!(project) package_files = ::Packages::PackageFile - .for_rubygem_with_file_name(user_project, params[:file_name]) + .for_rubygem_with_file_name(project, params[:file_name]) package_file = package_files.installable.last! - track_package_event('pull_package', :rubygems, project: user_project, namespace: user_project.namespace) + track_package_event('pull_package', :rubygems, project: project, namespace: project.namespace) present_package_file!(package_file) end @@ -109,9 +115,9 @@ module API end post 'gems/authorize' do authorize_workhorse!( - subject: user_project, + subject: project, has_length: false, - maximum_size: user_project.actual_limits.rubygems_max_file_size + maximum_size: project.actual_limits.rubygems_max_file_size ) end @@ -129,16 +135,16 @@ module API requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } end post 'gems' do - authorize_upload!(user_project) - bad_request!('File is too large') if user_project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size) + authorize_upload!(project) + bad_request!('File is too large') if project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size) - track_package_event('push_package', :rubygems, user: current_user, project: user_project, namespace: user_project.namespace) + track_package_event('push_package', :rubygems, user: current_user, project: project, namespace: project.namespace) package_file = nil ApplicationRecord.transaction do package = ::Packages::CreateTemporaryPackageService.new( - user_project, current_user, declared_params.merge(build: current_authenticated_job) + project, current_user, declared_params.merge(build: current_authenticated_job) ).execute(:rubygems, name: ::Packages::Rubygems::TEMPORARY_PACKAGE_NAME) file_params = { @@ -159,7 +165,7 @@ module API bad_request!('Package creation failed') end rescue ObjectStorage::RemoteStoreError => e - Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: user_project.id }) + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id }) forbidden! end @@ -179,13 +185,13 @@ module API optional :gems, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma delimited gem names' end get 'dependencies' do - authorize_read_package! + authorize_read_package!(project) if params[:gems].blank? status :ok else results = params[:gems].map do |gem_name| - service_result = Packages::Rubygems::DependencyResolverService.new(user_project, current_user, gem_name: gem_name).execute + service_result = Packages::Rubygems::DependencyResolverService.new(project, current_user, gem_name: gem_name).execute render_api_error!(service_result.message, service_result.http_status) if service_result.error? service_result.payload diff --git a/lib/api/search.rb b/lib/api/search.rb index cf6a1385783..2204437f2ec 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -59,8 +59,13 @@ module API end def search(additional_params = {}) + search_service = search_service(additional_params) + if search_service.global_search? && !search_service.global_search_enabled_for_scope? + forbidden!('Global Search is disabled for this scope') + end + @search_duration_s = Benchmark.realtime do - @results = search_service(additional_params).search_objects(preload_method) + @results = search_service.search_objects(preload_method) end set_global_search_log_information(additional_params) @@ -68,7 +73,7 @@ module API Gitlab::Metrics::GlobalSearchSlis.record_apdex( elapsed: @search_duration_s, search_type: search_type(additional_params), - search_level: search_service(additional_params).level, + search_level: search_service.level, search_scope: search_scope ) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 8b47604fe86..06b576a982b 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -85,9 +85,15 @@ module API optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' given housekeeping_enabled: ->(val) { val } do - requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." - requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." - requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + optional :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + optional :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + optional :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + + optional :housekeeping_optimize_repository_period, type: Integer, desc: "Number of Git pushes after which Gitaly is asked to optimize a repository." + + # Requires either all three deprecated attributes (housekeeping_full_repack_period, housekeeping_gc_period, housekeeping_incremental_repack_period) or housekeeping_optimize_repository_period + all_or_none_of :housekeeping_full_repack_period, :housekeeping_gc_period, :housekeeping_incremental_repack_period + exactly_one_of :housekeeping_incremental_repack_period, :housekeeping_optimize_repository_period end optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' optional :import_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, @@ -188,6 +194,7 @@ module API optional :jira_connect_application_key, type: String, desc: "Application ID of the OAuth application that should be used to authenticate with the GitLab.com for Jira Cloud app" optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab.com for Jira Cloud app" optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer' + optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token' Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index cda30dc957f..52e5ab30d06 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -15,7 +15,7 @@ module API entity: Entities::MergeRequest, source: Project, finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) }, - feature_category: :code_review + feature_category: :code_review_workflow }, { type: 'issues', diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index 6260983087f..eee83d5655b 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -4,7 +4,7 @@ module API class Suggestions < ::API::Base before { authenticate! } - feature_category :code_review + feature_category :code_review_workflow resource :suggestions do desc 'Apply suggestion patch in the Merge Request it was created' do diff --git a/lib/api/users.rb b/lib/api/users.rb index d2d45c94291..a9b09596728 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -62,6 +62,7 @@ module API optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for user', documentation: { type: 'file' } optional :theme_id, type: Integer, desc: 'The GitLab theme for the user' optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer' + # TODO: Add `allow_blank: false` in 16.0. Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/387005 optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' optional :note, type: String, desc: 'Admin note for this user' optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' @@ -294,6 +295,12 @@ module API authenticated_as_admin! params = declared_params(include_missing: false) + + # TODO: Remove in 16.0. Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/387005 + if params.key?(:private_profile) && params[:private_profile].nil? + params[:private_profile] = Gitlab::CurrentSettings.user_defaults_to_private_profile + end + user = ::Users::AuthorizedCreateService.new(current_user, params).execute if user.persisted? @@ -341,6 +348,12 @@ module API .where.not(id: user.id).exists? user_params = declared_params(include_missing: false) + + # TODO: Remove in 16.0. Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/387005 + if user_params.key?(:private_profile) && user_params[:private_profile].nil? + user_params[:private_profile] = Gitlab::CurrentSettings.user_defaults_to_private_profile + end + admin_making_changes_for_another_user = (current_user != user) if user_params[:password].present? @@ -824,7 +837,8 @@ module API elsif user.deactivated? forbidden!('Deactivated users cannot be unblocked by the API') else - user.activate + result = ::Users::UnblockService.new(current_user).execute(user) + result.success? end end # rubocop: enable CodeReuse/ActiveRecord @@ -1020,6 +1034,25 @@ module API end end + helpers do + def set_user_status(include_missing_params:) + forbidden! unless can?(current_user, :update_user_status, current_user) + + if ::Users::SetStatusService.new(current_user, declared_params(include_missing: include_missing_params)).execute + present current_user.status, with: Entities::UserStatus + else + render_validation_error!(current_user.status) + end + end + + params :set_user_status_params do + optional :emoji, type: String, desc: "The emoji to set on the status" + optional :message, type: String, desc: "The status message to set" + optional :availability, type: String, desc: "The availability of user to set" + optional :clear_status_after, type: String, desc: "Automatically clear emoji, message and availability fields after a certain time", values: UserStatus::CLEAR_STATUS_QUICK_OPTIONS.keys + end + end + desc "Get the currently authenticated user's SSH keys" do success Entities::SSHKey end @@ -1299,21 +1332,30 @@ module API desc 'Set the status of the current user' do success Entities::UserStatus + detail 'Any parameters that are not passed will be nullified.' end params do - optional :emoji, type: String, desc: "The emoji to set on the status" - optional :message, type: String, desc: "The status message to set" - optional :availability, type: String, desc: "The availability of user to set" - optional :clear_status_after, type: String, desc: "Automatically clear emoji, message and availability fields after a certain time", values: UserStatus::CLEAR_STATUS_QUICK_OPTIONS.keys + use :set_user_status_params end put "status", feature_category: :users do - forbidden! unless can?(current_user, :update_user_status, current_user) + set_user_status(include_missing_params: true) + end - if ::Users::SetStatusService.new(current_user, declared_params).execute - present current_user.status, with: Entities::UserStatus - else - render_validation_error!(current_user.status) + desc 'Set the status of the current user' do + success Entities::UserStatus + detail 'Any parameters that are not passed will be ignored.' + end + params do + use :set_user_status_params + end + patch "status", feature_category: :users do + if declared_params(include_missing: false).empty? + status :ok + + break end + + set_user_status(include_missing_params: false) end desc 'get the status of the current user' do diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb new file mode 100644 index 00000000000..8d49607f64c --- /dev/null +++ b/lib/api/validations/validators/bulk_imports.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module API + module Validations + module Validators + module BulkImports + class DestinationSlugPath < Grape::Validations::Base + def validate_param!(attr_name, params) + unless params[attr_name] =~ Gitlab::Regex.group_path_regex # rubocop: disable Style/GuardClause + raise Grape::Exceptions::Validation.new( + params: [@scope.full_name(attr_name)], + message: "cannot start with a dash or forward slash, or end with a period or forward slash. " \ + "It can only contain alphanumeric characters, periods, underscores, and dashes. " \ + "E.g. 'destination_namespace' not 'destination/namespace'" + ) + end + end + end + + class DestinationNamespacePath < Grape::Validations::Base + def validate_param!(attr_name, params) + return if params[attr_name].blank? + + unless params[attr_name] =~ Gitlab::Regex.bulk_import_namespace_path_regex # rubocop: disable Style/GuardClause + raise Grape::Exceptions::Validation.new( + params: [@scope.full_name(attr_name)], + message: "cannot start with a dash or forward slash, or end with a period or forward slash. " \ + "It can only contain alphanumeric characters, periods, underscores, forward slashes " \ + "and dashes. E.g. 'destination_namespace' or 'destination/namespace'" + ) + end + end + end + + class SourceFullPath < Grape::Validations::Base + def validate_param!(attr_name, params) + unless params[attr_name] =~ Gitlab::Regex.bulk_import_namespace_path_regex # rubocop: disable Style/GuardClause + raise Grape::Exceptions::Validation.new( + params: [@scope.full_name(attr_name)], + message: "must be a relative path and not include protocol, sub-domain, or domain information. " \ + "E.g. 'source/full/path' not 'https://example.com/source/full/path'" \ + ) + end + end + end + end + end + end +end diff --git a/lib/atlassian/jira_connect/jwt/asymmetric.rb b/lib/atlassian/jira_connect/jwt/asymmetric.rb index 7c1cf1cabb6..8698be70eb9 100644 --- a/lib/atlassian/jira_connect/jwt/asymmetric.rb +++ b/lib/atlassian/jira_connect/jwt/asymmetric.rb @@ -82,7 +82,7 @@ module Atlassian def public_key_cdn_url_setting @public_key_cdn_url_setting ||= - if Gitlab::CurrentSettings.jira_connect_proxy_url + if Gitlab::CurrentSettings.jira_connect_proxy_url.present? Gitlab::Utils.append_path(Gitlab::CurrentSettings.jira_connect_proxy_url, PROXY_PUBLIC_KEY_PATH) end end diff --git a/lib/banzai/filter/dollar_math_post_filter.rb b/lib/banzai/filter/dollar_math_post_filter.rb new file mode 100644 index 00000000000..94d1b4bcb48 --- /dev/null +++ b/lib/banzai/filter/dollar_math_post_filter.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by: +# - app/assets/javascripts/behaviors/markdown/marks/math.js +# - app/assets/javascripts/behaviors/markdown/nodes/code_block.js +module Banzai + module Filter + # HTML filter that implements our dollar math syntax, one of three filters: + # DollarMathPreFilter, DollarMathPostFilter, and MathFilter + # + class DollarMathPostFilter < HTML::Pipeline::Filter + # Based on the Pandoc heuristics, + # https://pandoc.org/MANUAL.html#extension-tex_math_dollars + # + # Handle the $...$ and $$...$$ inline syntax in this filter, after markdown processing + # but before post-handling of escaped characters. Any escaped $ will have been specially + # encoded and will therefore not interfere with the detection of the dollar syntax. + + # Corresponds to the "$...$" syntax + DOLLAR_INLINE_PATTERN = %r{ + (?<matched>\$(?<math>(?:\S[^$\n]*?\S|[^$\s]))\$)(?:[^\d]|$) + }x.freeze + + # Corresponds to the "$$...$$" syntax + DOLLAR_DISPLAY_INLINE_PATTERN = %r{ + (?<matched>\$\$\ *(?<math>[^$\n]+?)\ *\$\$) + }x.freeze + + # Order dependent. Handle the `$$` syntax before the `$` syntax + DOLLAR_MATH_PIPELINE = [ + { pattern: DOLLAR_DISPLAY_INLINE_PATTERN, style: :display }, + { pattern: DOLLAR_INLINE_PATTERN, style: :inline } + ].freeze + + # Do not recognize math inside these tags + IGNORED_ANCESTOR_TAGS = %w[pre code tt].to_set + + def call + process_dollar_pipeline + + doc + end + + def process_dollar_pipeline + doc.xpath('descendant-or-self::text()').each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + + node_html = node.to_html + next unless node_html.match?(DOLLAR_INLINE_PATTERN) || + node_html.match?(DOLLAR_DISPLAY_INLINE_PATTERN) + + temp_doc = Nokogiri::HTML.fragment(node_html) + + DOLLAR_MATH_PIPELINE.each do |pipeline| + temp_doc.xpath('child::text()').each do |temp_node| + html = temp_node.to_html + temp_node.content.scan(pipeline[:pattern]).each do |matched, math| + html.sub!(matched, math_html(math: math, style: pipeline[:style])) + end + + temp_node.replace(html) + end + end + + node.replace(temp_doc) + end + end + + private + + def math_html(math:, style:) + "<code data-math-style=\"#{style}\">#{math}</code>" + end + end + end +end diff --git a/lib/banzai/filter/dollar_math_pre_filter.rb b/lib/banzai/filter/dollar_math_pre_filter.rb new file mode 100644 index 00000000000..aaa186f87a6 --- /dev/null +++ b/lib/banzai/filter/dollar_math_pre_filter.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by: +# - app/assets/javascripts/behaviors/markdown/marks/math.js +# - app/assets/javascripts/behaviors/markdown/nodes/code_block.js +module Banzai + module Filter + # HTML filter that implements our dollar math syntax, one of three filters: + # DollarMathPreFilter, DollarMathPostFilter, and MathFilter + # + class DollarMathPreFilter < HTML::Pipeline::TextFilter + # Based on the Pandoc heuristics, + # https://pandoc.org/MANUAL.html#extension-tex_math_dollars + # + # Handle the $$\n...\n$$ syntax in this filter, before markdown processing, + # by converting it into the ```math syntax. In this way, we can ensure + # that it's considered a code block and will not have any markdown processed inside it. + + # Corresponds to the "$$\n...\n$$" syntax + REGEX = %r{ + #{::Gitlab::Regex.markdown_code_or_html_blocks} + | + (?=(?<=^\n|\A)\$\$\ *\n.*\n\$\$\ *(?=\n$|\z))(?: + # Display math block: + # $$ + # latex math + # $$ + + (?<=^\n|\A)\$\$\ *\n + (?<display_math> + (?:.)+? + ) + \n\$\$\ *(?=\n$|\z) + ) + }mx.freeze + + def call + @text.gsub(REGEX) do + if $~[:display_math] + # change from $$ to ```math + "```math\n#{$~[:display_math]}\n```" + else + $~[0] + end + end + end + end + end +end diff --git a/lib/banzai/filter/inline_observability_filter.rb b/lib/banzai/filter/inline_observability_filter.rb index 27b89073a0e..334c04f2b59 100644 --- a/lib/banzai/filter/inline_observability_filter.rb +++ b/lib/banzai/filter/inline_observability_filter.rb @@ -3,6 +3,12 @@ module Banzai module Filter class InlineObservabilityFilter < ::Banzai::Filter::InlineEmbedsFilter + def call + return doc unless can_view_observability? + + super + end + # Placeholder element for the frontend to use as an # injection point for observability. def create_element(url) @@ -25,6 +31,16 @@ module Banzai create_element(url) end + + private + + def can_view_observability? + Feature.enabled?(:observability_group_tab, group) + end + + def group + context[:group] || context[:project]&.group + end end end end diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb index 09ae09a22ae..8c0bd62f80a 100644 --- a/lib/banzai/filter/markdown_post_escape_filter.rb +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -2,33 +2,69 @@ module Banzai module Filter + # See comments in MarkdownPreEscapeFilter for details on strategy class MarkdownPostEscapeFilter < HTML::Pipeline::Filter LITERAL_KEYWORD = MarkdownPreEscapeFilter::LITERAL_KEYWORD LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-(.*?)-#{LITERAL_KEYWORD}}.freeze NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze SPAN_REGEX = %r{<span>(.*?)</span>}.freeze - CSS_A = 'a' - XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_A).freeze - CSS_LANG_TAG = 'pre' - XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_LANG_TAG).freeze + XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath('a').freeze + XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath('pre').freeze + XPATH_CODE_SPAN = Gitlab::Utils::Nokogiri.css_to_xpath('code > span').freeze def call return doc unless result[:escaped_literals] - # For any literals that actually didn't get escape processed - # (for example in code blocks), remove the special sequence. - html.gsub!(NOT_LITERAL_REGEX, '\1') + new_html = unescaped_literals(doc.to_html) + new_html = add_spans(new_html) - # Replace any left over literal sequences with `span` so that our - # reference processing is short-circuited - html.gsub!(LITERAL_REGEX, '<span>\1</span>') + @doc = parse_html(new_html) - # Since literals are converted in links, we need to remove any surrounding `span`. - # Note: this could have been done in the renderer, - # Banzai::Renderer::CommonMark::HTML. However, we eventually want to use - # the built-in compiled renderer, rather than the ruby version, for speed. - # So let's do this work here. + remove_spans_in_certain_attributes + remove_spans_in_code + + doc + end + + private + + # For any literals that actually didn't get escape processed + # (for example in code blocks), remove the special sequence. + def unescaped_literals(html) + html.gsub!(NOT_LITERAL_REGEX) do |match| + last_match = ::Regexp.last_match(1) + last_match_token = last_match.sub('%5C', '\\') + + escaped_item = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.find { |item| item[:token] == last_match_token } + escaped_char = escaped_item ? escaped_item[:escaped] : last_match + + escaped_char = escaped_char.sub('\\', '%5C') if last_match.start_with?('%5C') + + escaped_char + end + + html + end + + # Replace any left over literal sequences with `span` so that our + # reference processing is short-circuited + def add_spans(html) + html.gsub!(LITERAL_REGEX) do |match| + last_match = ::Regexp.last_match(1) + last_match_token = "\\#{last_match}" + + escaped_item = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.find { |item| item[:token] == last_match_token } + escaped_char = escaped_item ? escaped_item[:char] : ::Regexp.last_match(1) + + "<span>#{escaped_char}</span>" + end + + html + end + + # Since literals are converted in links, we need to remove any surrounding `span`. + def remove_spans_in_certain_attributes doc.xpath(XPATH_A).each do |node| node.attributes['href'].value = node.attributes['href'].value.gsub(SPAN_REGEX, '\1') if node.attributes['href'] node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title'] @@ -37,8 +73,16 @@ module Banzai doc.xpath(XPATH_LANG_TAG).each do |node| node.attributes['lang'].value = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang'] end + end - doc + # Any `<span>` that makes it into a `<code>` element is from the math processing, + # convert back to the escaped character, such as `\$` + def remove_spans_in_code + doc.xpath(XPATH_CODE_SPAN).each do |node| + escaped_item = Banzai::Filter::MarkdownPreEscapeFilter::ESCAPABLE_CHARS.find { |item| item[:char] == node.content && item[:latex] } + + node.replace(escaped_item[:escaped]) if escaped_item + end end end end diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb index 8d54d140877..8cc7b0defd6 100644 --- a/lib/banzai/filter/markdown_pre_escape_filter.rb +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -10,6 +10,10 @@ module Banzai # This way CommonMark will properly handle the backslash escaped chars # but we will maintain knowledge (the sequence) that it was a literal. # + # This processing is also important for the handling of escaped characters + # in LaTeX math. These will need to be converted back into their escaped + # versions if they are detected in math blocks. + # # We need to surround the character, not just prefix it. It could # get converted into an entity by CommonMark and we wouldn't know how many # characters there are. The entire literal needs to be surrounded with @@ -24,9 +28,36 @@ module Banzai # This filter does the initial surrounding, and MarkdownPostEscapeFilter # does the conversion into span tags. class MarkdownPreEscapeFilter < HTML::Pipeline::TextFilter - # We just need to target those that are special GitLab references - REFERENCE_CHARACTERS = '@#!$&~%^' - ASCII_PUNCTUATION = %r{(\\[#{REFERENCE_CHARACTERS}])}.freeze + # Table of characters that need this special handling. It consists of the + # GitLab special reference characters and special LaTeX characters. + # + # The `token` is used when we do the initial replacement - for example converting + # `\$` into `cmliteral-\+a-cmliteral`. We don't simply replace `\$` with `$`, + # because this can cause difficulties in parsing math blocks that use `$` as a + # delimiter. We also include a character that _can_ be escaped, `\+`. By examining + # the text once it's been passed to markdown, we can determine that `cmliteral-\+a-cmliteral` + # was in a block that markdown did _not_ escape the character, for example an inline + # code block or some other element. In this case, we must convert back to the + # original escaped version, `\$`. However if we detect `cmliteral-+a-cmliteral`, + # then we know markdown considered it an escaped character, and we should replace it + # with the non-escaped version, `$`. + # See the MarkdownPostEscapeFilter for how this is done. + ESCAPABLE_CHARS = [ + { char: '$', escaped: '\$', token: '\+a', reference: true, latex: true }, + { char: '%', escaped: '\%', token: '\+b', reference: true, latex: true }, + { char: '#', escaped: '\#', token: '\+c', reference: true, latex: true }, + { char: '&', escaped: '\&', token: '\+d', reference: true, latex: true }, + { char: '{', escaped: '\{', token: '\+e', reference: false, latex: true }, + { char: '}', escaped: '\}', token: '\+f', reference: false, latex: true }, + { char: '_', escaped: '\_', token: '\+g', reference: false, latex: true }, + { char: '@', escaped: '\@', token: '\+h', reference: true, latex: false }, + { char: '!', escaped: '\!', token: '\+i', reference: true, latex: false }, + { char: '~', escaped: '\~', token: '\+j', reference: true, latex: false }, + { char: '^', escaped: '\^', token: '\+k', reference: true, latex: false } + ].freeze + + TARGET_CHARS = ESCAPABLE_CHARS.pluck(:char).join.freeze + ASCII_PUNCTUATION = %r{(\\[#{TARGET_CHARS}])}.freeze LITERAL_KEYWORD = 'cmliteral' def call @@ -35,7 +66,10 @@ module Banzai # are found, we can bypass the post filter result[:escaped_literals] = true - "#{LITERAL_KEYWORD}-#{match}-#{LITERAL_KEYWORD}" + escaped_item = ESCAPABLE_CHARS.find { |item| item[:escaped] == match } + token = escaped_item ? escaped_item[:token] : match + + "#{LITERAL_KEYWORD}-#{token}-#{LITERAL_KEYWORD}" end end end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 1d854d6599b..9b6fc71077a 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -1,55 +1,29 @@ # frozen_string_literal: true -require 'uri' - # Generated HTML is transformed back to GFM by: # - app/assets/javascripts/behaviors/markdown/marks/math.js # - app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter - # HTML filter that implements our math syntax, adding class="code math" + # HTML filter that implements the original GitLab math syntax, one of three filters: + # DollarMathPreFilter, DollarMathPostFilter, and MathFilter # class MathFilter < HTML::Pipeline::Filter + # Handle the $`...`$ and ```math syntax in this filter. + # Also add necessary classes any existing math blocks. + CSS_MATH = 'pre[lang="math"] > code' XPATH_MATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_MATH).freeze CSS_CODE = 'code' XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze - - # These are based on the Pandoc heuristics, - # https://pandoc.org/MANUAL.html#extension-tex_math_dollars - # Note: at this time, using a dollar sign literal, `\$` inside - # a math statement does not work correctly. - # Corresponds to the "$...$" syntax - DOLLAR_INLINE_PATTERN = %r{ - (?<matched>\$(?<math>(?:\S[^$\n]*?\S|[^$\s]))\$)(?:[^\d]|$) - }x.freeze - - # Corresponds to the "$$...$$" syntax - DOLLAR_DISPLAY_INLINE_PATTERN = %r{ - (?<matched>\$\$\ *(?<math>[^$\n]+?)\ *\$\$) - }x.freeze - - # Corresponds to the $$\n...\n$$ syntax - DOLLAR_DISPLAY_BLOCK_PATTERN = %r{ - ^(?<matched>\$\$\ *\n(?<math>.*)\n\$\$\ *)$ - }mx.freeze - - # Order dependent. Handle the `$$` syntax before the `$` syntax - DOLLAR_MATH_PIPELINE = [ - { pattern: DOLLAR_DISPLAY_INLINE_PATTERN, tag: :code, style: :display }, - { pattern: DOLLAR_DISPLAY_BLOCK_PATTERN, tag: :pre, style: :display }, - { pattern: DOLLAR_INLINE_PATTERN, tag: :code, style: :inline } - ].freeze - - # Do not recognize math inside these tags - IGNORED_ANCESTOR_TAGS = %w[pre code tt].to_set + CSS_INLINE_CODE = 'code[data-math-style]' + XPATH_INLINE_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_INLINE_CODE).freeze # Attribute indicating inline or display math. STYLE_ATTRIBUTE = 'data-math-style' # Class used for tagging elements that should be rendered TAG_CLASS = 'js-render-math' - MATH_CLASSES = "code math #{TAG_CLASS}" DOLLAR_SIGN = '$' @@ -61,47 +35,31 @@ module Banzai def call @nodes_count = 0 - process_dollar_pipeline if Feature.enabled?(:markdown_dollar_math, group) - + process_existing process_dollar_backtick_inline process_math_codeblock doc end - def process_dollar_pipeline - doc.xpath('descendant-or-self::text()').each do |node| - next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - - node_html = node.to_html - next unless node_html.match?(DOLLAR_INLINE_PATTERN) || - node_html.match?(DOLLAR_DISPLAY_INLINE_PATTERN) || - node_html.match?(DOLLAR_DISPLAY_BLOCK_PATTERN) - - temp_doc = Nokogiri::HTML.fragment(node_html) - DOLLAR_MATH_PIPELINE.each do |pipeline| - temp_doc.xpath('child::text()').each do |temp_node| - html = temp_node.to_html - temp_node.content.scan(pipeline[:pattern]).each do |matched, math| - html.sub!(matched, math_html(tag: pipeline[:tag], style: pipeline[:style], math: math)) - - @nodes_count += 1 - break if @nodes_count >= RENDER_NODES_LIMIT - end + private - temp_node.replace(html) + # Add necessary classes to any existing math blocks + def process_existing + doc.xpath(XPATH_INLINE_CODE).each do |code| + break if @nodes_count >= RENDER_NODES_LIMIT - break if @nodes_count >= RENDER_NODES_LIMIT - end - end + code[:class] = MATH_CLASSES - node.replace(temp_doc) + @nodes_count += 1 end end # Corresponds to the "$`...`$" syntax def process_dollar_backtick_inline doc.xpath(XPATH_CODE).each do |code| + break if @nodes_count >= RENDER_NODES_LIMIT + closing = code.next opening = code.previous @@ -112,17 +70,16 @@ module Banzai closing.content.first == DOLLAR_SIGN && opening.content.last == DOLLAR_SIGN - code[:class] = MATH_CLASSES code[STYLE_ATTRIBUTE] = 'inline' + code[:class] = MATH_CLASSES closing.content = closing.content[1..] opening.content = opening.content[0..-2] @nodes_count += 1 - break if @nodes_count >= RENDER_NODES_LIMIT end end - # corresponds to the "```math...```" syntax + # Corresponds to the "```math...```" syntax def process_math_codeblock doc.xpath(XPATH_MATH).each do |node| pre_node = node.parent @@ -130,21 +87,6 @@ module Banzai pre_node[:class] = TAG_CLASS end end - - private - - def math_html(tag:, math:, style:) - case tag - when :code - "<code class=\"#{MATH_CLASSES}\" data-math-style=\"#{style}\">#{math}</code>" - when :pre - "<pre class=\"#{MATH_CLASSES}\" data-math-style=\"#{style}\"><code>#{math}</code></pre>" - end - end - - def group - context[:group] || context[:project]&.group - end end end end diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index 86beeae01b7..ddc3f5cf715 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -90,14 +90,14 @@ module Banzai end def get_uri(html_attr) - uri = URI(html_attr.value) + uri = Addressable::URI.parse(html_attr.value) uri if uri.relative? && uri.path.present? rescue URI::Error, Addressable::URI::InvalidURIError end def process_link_to_repository_attr(html_attr) - uri = URI(html_attr.value) + uri = Addressable::URI.parse(html_attr.value) if uri.relative? && uri.path.present? html_attr.value = rebuild_relative_uri(uri).to_s diff --git a/lib/banzai/filter/service_desk_upload_link_filter.rb b/lib/banzai/filter/service_desk_upload_link_filter.rb new file mode 100644 index 00000000000..9f26dfb8ae5 --- /dev/null +++ b/lib/banzai/filter/service_desk_upload_link_filter.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # HTML filter for service desk emails. + # Context options: + # :replace_upload_links + class ServiceDeskUploadLinkFilter < BaseRelativeLinkFilter + def call + return doc unless context[:uploads_as_attachments].present? + + linkable_attributes.reject! do |attr| + replace_upload_link(attr) + end + + doc + end + + protected + + def replace_upload_link(html_attr) + return unless html_attr.name == 'href' + return unless html_attr.value.start_with?('/uploads/') + + secret, filename_in_link = html_attr.value.scan(FileUploader::DYNAMIC_PATH_PATTERN).first + return unless context[:uploads_as_attachments].include?("#{secret}/#{filename_in_link}") + + parent = html_attr.parent + filename_in_text = parent.text + final_filename = if filename_in_link != filename_in_text + "#{filename_in_text} (#{filename_in_link})" + else + filename_in_text + end + + final_element = Nokogiri::HTML::DocumentFragment.parse("<strong>#{final_filename}</strong>") + parent.replace(final_element) + end + end + end +end diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index 1da0f72996b..205bbc2140d 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -3,10 +3,17 @@ module Banzai module Pipeline class PlainMarkdownPipeline < BasePipeline + # DollarMathPreFilter and DollarMathPostFilter need to be included here, + # rather than in another pipeline. However, since dollar math would most + # likely be supported as an extension in any other markdown parser we used, + # it is not out of place. We are considering this a part of the actual + # markdown processing def self.filters FilterArray[ Filter::MarkdownPreEscapeFilter, + Filter::DollarMathPreFilter, Filter::MarkdownFilter, + Filter::DollarMathPostFilter, Filter::MarkdownPostEscapeFilter ] end diff --git a/lib/banzai/pipeline/service_desk_email_pipeline.rb b/lib/banzai/pipeline/service_desk_email_pipeline.rb new file mode 100644 index 00000000000..cc7cd8a92b8 --- /dev/null +++ b/lib/banzai/pipeline/service_desk_email_pipeline.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class ServiceDeskEmailPipeline < EmailPipeline + def self.filters + super.insert_before(Filter::ExternalLinkFilter, Banzai::Filter::ServiceDeskUploadLinkFilter) + end + end + end +end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 8129ff6151c..6c36875111b 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -8,6 +8,7 @@ module BulkImports API_VERSION = 'v4' DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 30 + PAT_ENDPOINT_MIN_VERSION = '15.5.0' def initialize(url:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION) @url = url @@ -66,38 +67,57 @@ module BulkImports instance_version >= BulkImport.min_gl_version_for_project_migration end - private + def options + { headers: { 'Content-Type' => 'application/json' }, query: { private_token: @token } } + end - def validate_instance_version! - return if @compatible_instance_version + def validate_import_scopes! + return true unless instance_version >= ::Gitlab::VersionInfo.parse(PAT_ENDPOINT_MIN_VERSION) - if instance_version.major < BulkImport::MIN_MAJOR_VERSION - raise ::BulkImports::Error.unsupported_gitlab_version - else - @compatible_instance_version = true + response = with_error_handling do + Gitlab::HTTP.get(resource_url("personal_access_tokens/self"), options) end + + return true if response['scopes']&.include?('api') + + raise ::BulkImports::Error.scope_validation_failure + end + + def validate_instance_version! + return true unless instance_version.major < BulkImport::MIN_MAJOR_VERSION + + raise ::BulkImports::Error.unsupported_gitlab_version end + private + def metadata response = begin with_error_handling do - Gitlab::HTTP.get(resource_url(:version), default_options) + Gitlab::HTTP.get(resource_url(:version), options) end rescue BulkImports::NetworkError # `version` endpoint is not available, try `metadata` endpoint instead with_error_handling do - Gitlab::HTTP.get(resource_url(:metadata), default_options) + Gitlab::HTTP.get(resource_url(:metadata), options) end end response.parsed_response + rescue BulkImports::NetworkError => e + case e&.response&.code + when 401, 403 + raise ::BulkImports::Error.scope_validation_failure + when 404 + raise ::BulkImports::Error.invalid_url + else + raise + end end strong_memoize_attr :metadata # rubocop:disable GitlabSecurity/PublicSend def request(method, resource, options = {}, &block) - validate_instance_version! - with_error_handling do Gitlab::HTTP.public_send( method, @@ -134,9 +154,10 @@ module BulkImports def with_error_handling response = yield - raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}. Body: #{response.parsed_response}", response: response) unless response.success? + return response if response.success? + + raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}. Body: #{response.parsed_response}", response: response) - response rescue *Gitlab::HTTP::HTTP_ERRORS => e raise ::BulkImports::NetworkError, e end diff --git a/lib/bulk_imports/error.rb b/lib/bulk_imports/error.rb index 988982d3cdf..38f26028276 100644 --- a/lib/bulk_imports/error.rb +++ b/lib/bulk_imports/error.rb @@ -5,5 +5,14 @@ module BulkImports def self.unsupported_gitlab_version self.new("Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MIN_MAJOR_VERSION}.") end + + def self.scope_validation_failure + self.new("Import aborted as the provided personal access token does not have the required 'api' scope or " \ + "is no longer valid.") + end + + def self.invalid_url + self.new("Import aborted as it was not possible to connect to the provided GitLab instance URL.") + end end end diff --git a/lib/bulk_imports/features.rb b/lib/bulk_imports/features.rb index 952e8e62d71..9fdceb03655 100644 --- a/lib/bulk_imports/features.rb +++ b/lib/bulk_imports/features.rb @@ -2,10 +2,6 @@ module BulkImports module Features - def self.enabled? - ::Feature.enabled?(:bulk_import) - end - def self.project_migration_enabled?(destination_namespace = nil) if destination_namespace.present? root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 0db2b1f0698..7a777f1c8e1 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -71,7 +71,7 @@ module BulkImports end def project_entities_pipeline - if project_pipeline_available? && feature_flag_enabled? + if migrate_projects? && project_pipeline_available? && feature_flag_enabled? { project_entities: { pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline, @@ -83,6 +83,10 @@ module BulkImports end end + def migrate_projects? + bulk_import_entity.migrate_projects + end + def project_pipeline_available? @bulk_import.source_version_info >= BulkImport.min_gl_version_for_project_migration end diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb index d8fb937ecd2..fcf9ed62388 100644 --- a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb +++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb @@ -10,7 +10,8 @@ module BulkImports source_full_path: entry['full_path'], destination_name: entry['path'], destination_namespace: context.entity.group.full_path, - parent_id: context.entity.id + parent_id: context.entity.id, + migrate_projects: context.entity.migrate_projects } end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 723935f8aaf..c879ec41d86 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -5,6 +5,7 @@ module ContainerRegistry include Gitlab::Utils::StrongMemoize attr_accessor :uri + attr_reader :options, :base_uri REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version' REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features' diff --git a/lib/event_filter.rb b/lib/event_filter.rb index f14b0a6b9e7..ed14affda71 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -38,7 +38,7 @@ class EventFilter when TEAM events.where(action: Event::TEAM_ACTIONS) when ISSUE - events.where(action: Event::ISSUE_ACTIONS, target_type: 'Issue') + events.where(action: Event::ISSUE_ACTIONS).for_issue when WIKI wiki_events(events) when DESIGNS @@ -97,7 +97,7 @@ class EventFilter when ISSUE in_operator_params( array_data: array_data, - scope: Event.where(target_type: Issue.name), + scope: Event.for_issue, in_column: :action, in_values: Event.actions.values_at(*Event::ISSUE_ACTIONS) ) diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb index 22d8874db57..3abf380d461 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb @@ -5,7 +5,7 @@ module Gitlab module CycleAnalytics module Aggregated # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage + # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index ae675b6ad27..0db027b9861 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics module CycleAnalytics # Arguments: - # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage + # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::Stage # params: # current_user: an instance of User # from: DateTime diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index ac9c465bf7d..d058782ae87 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -106,7 +106,7 @@ module Gitlab def use_aggregated_backend? # for now it's only available on the group-level - group.present? && aggregation.enabled + group.present? end def aggregation_attributes @@ -118,14 +118,14 @@ module Gitlab end def aggregation - @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_group(group) + @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_namespace(group) end def group_data_attributes { id: group.id, + namespace_id: group.id, name: group.name, - parent_id: group.parent_id, full_path: group.full_path, avatar_url: group.avatar_url } diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 5b1bf99e297..a788586ebec 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -115,6 +115,38 @@ module Gitlab value > threshold_value end + # Similar to #throttled? above but checks for the bypass header in the request and logs the request when it is over the rate limit + # + # @param request [Http::Request] - Web request used to check the header and log + # @param current_user [User] Current user of the request, it can be nil + # @param key [Symbol] Key attribute registered in `.rate_limits` + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings + # or Symbols to scope throttling to a specific request (e.g. per user + # per project) + # @param resource [ActiveRecord] An ActiveRecord model to count an action + # for (e.g. limit unique project (resource) downloads (action) to five + # per user (scope)) + # @param threshold [Integer] Optional threshold value to override default + # one registered in `.rate_limits` + # @param interval [Integer] Optional interval value to override default + # one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to + # exclude from the limit. This param will only be functional if Scope + # includes a current user. + # @param peek [Boolean] Optional. When true the key will not be + # incremented but the current throttled state will be returned. + # + # @return [Boolean] Whether or not a request should be throttled + def throttled_request?(request, current_user, key, scope:, **options) + if ::Gitlab::Throttle.bypass_header.present? && request.get_header(Gitlab::Throttle.bypass_header) == '1' + return false + end + + throttled?(key, scope: scope, **options).tap do |throttled| + log_request(request, "#{key}_request_limit".to_sym, current_user) if throttled + end + end + # Returns the current rate limited state without incrementing the count. # # @param key [Symbol] Key attribute registered in `.rate_limits` diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 7e8f9c76dea..c97ef5a10ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -31,6 +31,7 @@ module Gitlab # Scopes used for GitLab as admin SUDO_SCOPE = :sudo + ADMIN_MODE_SCOPE = :admin_mode ADMIN_SCOPES = [SUDO_SCOPE].freeze # Default scopes for OAuth applications that don't define their own @@ -366,6 +367,7 @@ module Gitlab def available_scopes_for(current_user) scopes = non_admin_available_scopes scopes += ADMIN_SCOPES if current_user.admin? + scopes end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 26be7c8aa60..242390c3e89 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -260,7 +260,7 @@ module Gitlab if sync_profile_from_provider? UserSyncedAttributesMetadata::SYNCABLE_ATTRIBUTES.each do |key| if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) - gl_user[key] = auth_hash.public_send(key) # rubocop:disable GitlabSecurity/PublicSend + gl_user.public_send("#{key}=".to_sym, auth_hash.public_send(key)) # rubocop:disable GitlabSecurity/PublicSend metadata.set_attribute_synced(key, true) else metadata.set_attribute_synced(key, false) diff --git a/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb b/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb deleted file mode 100644 index b39c0953fb1..00000000000 --- a/lib/gitlab/background_migration/add_primary_email_to_emails_if_user_confirmed.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Add user primary email to emails table if confirmed - class AddPrimaryEmailToEmailsIfUserConfirmed - INNER_BATCH_SIZE = 1_000 - - # Stubbed class to access the User table - class User < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'users' - self.inheritance_column = :_type_disabled - - scope :confirmed, -> { where.not(confirmed_at: nil) } - - has_many :emails - end - - # Stubbed class to access the Emails table - class Email < ActiveRecord::Base - self.table_name = 'emails' - self.inheritance_column = :_type_disabled - - belongs_to :user - end - - def perform(start_id, end_id) - User.confirmed.where(id: start_id..end_id).select(:id, :email, :confirmed_at).each_batch(of: INNER_BATCH_SIZE) do |users| - current_time = Time.now.utc - - attributes = users.map do |user| - { - user_id: user.id, - email: user.email, - confirmed_at: user.confirmed_at, - created_at: current_time, - updated_at: current_time - } - end - - Email.insert_all(attributes) - end - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'AddPrimaryEmailToEmailsIfUserConfirmed', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb new file mode 100644 index 00000000000..82e607ac7a7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill `admin_mode` scope for a range of personal access tokens + class BackfillAdminModeScopeForPersonalAccessTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) do + relation.joins('INNER JOIN users ON personal_access_tokens.user_id = users.id') + .where(users: { admin: true }) + .where(revoked: [false, nil]) + .where.not('expires_at IS NOT NULL AND expires_at <= ?', Time.current) + end + + operation_name :update_all + feature_category :authentication_and_authorization + + ADMIN_MODE_SCOPE = ['admin_mode'].freeze + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |token| + token.update!(scopes: (YAML.safe_load(token.scopes) + ADMIN_MODE_SCOPE).uniq.to_yaml) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb index 249c9d7af57..1dca82486ac 100644 --- a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -17,6 +17,7 @@ module Gitlab end operation_name :update_all + feature_category :database def perform each_sub_batch(batching_scope: RELATION) do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_environment_tiers.rb b/lib/gitlab/background_migration/backfill_environment_tiers.rb index 6f381577274..ebfabf1b28e 100644 --- a/lib/gitlab/background_migration/backfill_environment_tiers.rb +++ b/lib/gitlab/background_migration/backfill_environment_tiers.rb @@ -7,6 +7,7 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/issues/300741 for more information. class BackfillEnvironmentTiers < BatchedMigrationJob operation_name :backfill_environment_tiers + feature_category :database # Equivalent to `Environment#guess_tier` pattern matching. PRODUCTION_TIER = 0 diff --git a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb index bd61d1a0f07..ee64a8ca2d5 100644 --- a/lib/gitlab/background_migration/backfill_epic_cache_counts.rb +++ b/lib/gitlab/background_migration/backfill_epic_cache_counts.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class BackfillEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform; end end # rubocop: enable Style/Documentation diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb index 4ea664e2529..c45dcad5b2d 100644 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -6,6 +6,7 @@ module Gitlab class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob job_arguments :batch_size operation_name :upsert_group_features + feature_category :database def perform each_sub_batch( diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb index c95fed512c9..8c151bc36ac 100644 --- a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -10,6 +10,7 @@ module Gitlab SUB_BATCH_SIZE = 1_000 operation_name :update_search_data + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb index fe05b4ec3c1..2202cbb2f85 100644 --- a/lib/gitlab/background_migration/backfill_internal_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -6,6 +6,7 @@ module Gitlab class BackfillInternalOnNotes < BatchedMigrationJob scope_to -> (relation) { relation.where(confidential: true) } operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb index 640d9379351..57254c09f78 100644 --- a/lib/gitlab/background_migration/backfill_namespace_details.rb +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -5,6 +5,7 @@ module Gitlab # Backfill namespace_details for a range of namespaces class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob operation_name :backfill_namespace_details + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb index dca7f9fa921..8600510b6ef 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb @@ -5,6 +5,7 @@ module Gitlab # Sets the `namespace_id` of the existing `vulnerability_reads` records class BackfillNamespaceIdOfVulnerabilityReads < BatchedMigrationJob operation_name :set_namespace_id + feature_category :database UPDATE_SQL = <<~SQL UPDATE diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index 6520cd63711..ff20a7ed177 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -18,6 +18,7 @@ module Gitlab end operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_project_import_level.rb b/lib/gitlab/background_migration/backfill_project_import_level.rb index 21c239e0070..1a4b1e6731f 100644 --- a/lib/gitlab/background_migration/backfill_project_import_level.rb +++ b/lib/gitlab/background_migration/backfill_project_import_level.rb @@ -4,6 +4,7 @@ module Gitlab module BackgroundMigration class BackfillProjectImportLevel < BatchedMigrationJob operation_name :update_import_level + feature_category :database LEVEL = { Gitlab::Access::NO_ACCESS => [0], diff --git a/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb b/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb index c2e37269b5e..1bf029f5001 100644 --- a/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb +++ b/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Backfills the `members.member_namespace_id` column for `type=ProjectMember` class BackfillProjectMemberNamespaceId < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) diff --git a/lib/gitlab/background_migration/backfill_project_namespace_details.rb b/lib/gitlab/background_migration/backfill_project_namespace_details.rb index 9bee3cf21e8..4f4db50321d 100644 --- a/lib/gitlab/background_migration/backfill_project_namespace_details.rb +++ b/lib/gitlab/background_migration/backfill_project_namespace_details.rb @@ -4,6 +4,7 @@ module Gitlab # Backfill project namespace_details for a range of projects class BackfillProjectNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob operation_name :backfill_project_namespace_details + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb index 34dd3321125..0c4953486f4 100644 --- a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -7,6 +7,7 @@ module Gitlab MAX_UPDATE_RETRIES = 3 operation_name :update_all + feature_category :database def perform each_sub_batch( diff --git a/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb index ec813022b8f..01cae3e2d50 100644 --- a/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb +++ b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Back-fill container_registry_size for project_statistics class BackfillProjectStatisticsContainerRepositorySize < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform # no-op end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb index 1a3dd88ea31..da865ed935a 100644 --- a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Back-fill storage_size for project_statistics class BackfillProjectStatisticsStorageSizeWithoutUploadsSize < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform # no-op end diff --git a/lib/gitlab/background_migration/backfill_releases_author_id.rb b/lib/gitlab/background_migration/backfill_releases_author_id.rb new file mode 100644 index 00000000000..8982fe1acca --- /dev/null +++ b/lib/gitlab/background_migration/backfill_releases_author_id.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills releases with empty release authors. + # More details on: + # 1) https://gitlab.com/groups/gitlab-org/-/epics/8375 + # 2) https://gitlab.com/gitlab-org/gitlab/-/issues/367522#note_1156503600 + class BackfillReleasesAuthorId < BatchedMigrationJob + operation_name :backfill_releases_author_id + job_arguments :ghost_user_id + feature_category :database + + scope_to ->(relation) { relation.where(author_id: nil) } + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(author_id: ghost_user_id) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_user_details_fields.rb b/lib/gitlab/background_migration/backfill_user_details_fields.rb index 8d8619256b0..26489d06a85 100644 --- a/lib/gitlab/background_migration/backfill_user_details_fields.rb +++ b/lib/gitlab/background_migration/backfill_user_details_fields.rb @@ -11,6 +11,7 @@ module Gitlab # * organization class BackfillUserDetailsFields < BatchedMigrationJob operation_name :backfill_user_details_fields + feature_category :database def perform query = <<~SQL diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 37b1a37569b..20c3c68ec40 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -5,6 +5,7 @@ module Gitlab # Backfills the `vulnerability_reads.casted_cluster_agent_id` column class BackfillVulnerabilityReadsClusterAgent < Gitlab::BackgroundMigration::BatchedMigrationJob operation_name :update_all + feature_category :database CLUSTER_AGENTS_JOIN = <<~SQL INNER JOIN cluster_agents diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index a020cabd1f4..fc0d0ce3a57 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -5,6 +5,8 @@ module Gitlab # Backfills the `issues.work_item_type_id` column, replacing any # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` class BackfillWorkItemTypeIdForIssues < BatchedMigrationJob + feature_category :database + # Basic AR model for issues table class MigrationIssue < ApplicationRecord self.table_name = 'issues' diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 973ab20f547..4039a79cfa7 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -27,7 +27,7 @@ module Gitlab end def operation_name(operation) - define_method('operation_name') do + define_method(:operation_name) do operation end end diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb index 0cd19dc5df9..5c0ddf0ba8b 100644 --- a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb +++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb @@ -8,6 +8,8 @@ module Gitlab class CleanupOrphanedRoutes < Gitlab::BackgroundMigration::BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers + feature_category :database + def perform # there should really be no records to fix, there is none gitlab.com, but taking the safer route, just in case. fix_missing_namespace_id_routes diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 136293242b2..033b2c87152 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -16,6 +16,7 @@ module Gitlab class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob job_arguments :copy_from, :copy_to operation_name :update_all + feature_category :database def perform assignment_clauses = build_assignment_clauses(copy_from, copy_to) diff --git a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb index 739197898d9..c7c063e8ccf 100644 --- a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb +++ b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't delete approval rules # as this feature exists only in EE class DeleteApprovalRulesWithVulnerability < BatchedMigrationJob + feature_category :database + def perform end end diff --git a/lib/gitlab/background_migration/delete_invalid_epic_issues.rb b/lib/gitlab/background_migration/delete_invalid_epic_issues.rb index 3af59ab4931..6c0eb6b1950 100644 --- a/lib/gitlab/background_migration/delete_invalid_epic_issues.rb +++ b/lib/gitlab/background_migration/delete_invalid_epic_issues.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class DeleteInvalidEpicIssues < BatchedMigrationJob + feature_category :database + def perform end end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb index f93dcf83c49..6953ae65651 100644 --- a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -17,6 +17,8 @@ module Gitlab SQL operation_name :delete_orphaned_operational_vulnerabilities + feature_category :database + scope_to ->(relation) do relation .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb index 4b7b7d42c77..e77d56d68cb 100644 --- a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb +++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb @@ -7,6 +7,7 @@ module Gitlab scope_to ->(relation) { relation.where(report_type: 4) } operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb index 33aa1a8d29d..28809df8694 100644 --- a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb +++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb @@ -5,6 +5,7 @@ module Gitlab # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) class DeleteOrphansApprovalProjectRules < BatchedMigrationJob operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb index 9eb0d4489d6..79aae719d03 100644 --- a/lib/gitlab/background_migration/destroy_invalid_group_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -10,6 +10,7 @@ module Gitlab end operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb index b274c71f24f..9a70dc39960 100644 --- a/lib/gitlab/background_migration/destroy_invalid_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -5,6 +5,7 @@ module Gitlab class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(member_namespace_id: nil) } operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb index 53b4712ef6e..5f6bb840f77 100644 --- a/lib/gitlab/background_migration/destroy_invalid_project_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -5,6 +5,7 @@ module Gitlab class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(source_type: 'Project') } operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb index b32e88581dd..c4ce88b9404 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -8,6 +8,7 @@ module Gitlab THRESHOLD_DATE = '2022-02-17 09:00:00' operation_name :disable_legacy_open_source_licence_for_recent_public_projects + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb index 5685b782a71..6114aa33a43 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb @@ -9,6 +9,7 @@ module Gitlab LAST_ACTIVITY_DATE = '2021-07-01' operation_name :disable_legacy_open_source_license_available + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb index b5e5555bd2d..2eb7c5230ba 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb @@ -7,6 +7,7 @@ module Gitlab PUBLIC = 20 operation_name :disable_legacy_open_source_license_for_no_issues_no_repo_projects + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb index 89863458676..8953836c705 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -7,6 +7,7 @@ module Gitlab PUBLIC = 20 operation_name :disable_legacy_open_source_license_for_one_member_no_repo_projects + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb index dcef4f086e2..b2805289b30 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb @@ -10,6 +10,7 @@ module Gitlab end operation_name :disable_legacy_open_source_license_for_projects_less_than_five_mb + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb index 7d93f2d4fda..15c80a6cac2 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -6,6 +6,7 @@ module Gitlab class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } operation_name :disable_legacy_open_source_license_for_projects_less_than_one_mb + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/expire_o_auth_tokens.rb b/lib/gitlab/background_migration/expire_o_auth_tokens.rb index 08bcdb8a789..20dacd642de 100644 --- a/lib/gitlab/background_migration/expire_o_auth_tokens.rb +++ b/lib/gitlab/background_migration/expire_o_auth_tokens.rb @@ -4,21 +4,15 @@ module Gitlab module BackgroundMigration # Add expiry to all OAuth access tokens class ExpireOAuthTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob - operation_name :update_oauth_tokens + scope_to ->(relation) { relation.where(expires_in: nil) } + operation_name :update_all + feature_category :database def perform - each_sub_batch( - batching_scope: ->(relation) { relation.where(expires_in: nil) } - ) do |sub_batch| - update_oauth_tokens(sub_batch) + each_sub_batch do |sub_batch| + sub_batch.update_all(expires_in: 2.hours) end end - - private - - def update_oauth_tokens(relation) - relation.update_all(expires_in: 7_200) - end end end end diff --git a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb index 4b283bae79d..bfbed0408e1 100644 --- a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb +++ b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't update approval project rules # as this feature exists only in EE class FixApprovalProjectRulesWithoutProtectedBranches < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index 97a9913fa74..452167d4d61 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -186,7 +186,7 @@ module Gitlab end def migrate_instance_cluster? - if instance_variable_defined?('@migrate_instance_cluster') + if instance_variable_defined?(:@migrate_instance_cluster) @migrate_instance_cluster else @migrate_instance_cluster = Migratable::Cluster.instance_type.has_prometheus_application? diff --git a/lib/gitlab/background_migration/fix_security_scan_statuses.rb b/lib/gitlab/background_migration/fix_security_scan_statuses.rb index b60e739f870..1cfc9a278b7 100644 --- a/lib/gitlab/background_migration/fix_security_scan_statuses.rb +++ b/lib/gitlab/background_migration/fix_security_scan_statuses.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Fixes the `status` attribute of `security_scans` records class FixSecurityScanStatuses < BatchedMigrationJob + feature_category :database + def perform # no-op. The logic is defined in EE module. end diff --git a/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb b/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb index bea0120f093..d1acb8ca2d2 100644 --- a/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb +++ b/lib/gitlab/background_migration/migrate_shared_vulnerability_scanners.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class MigrateSharedVulnerabilityScanners < BatchedMigrationJob + feature_category :database + def perform end end diff --git a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb index 81b29b5a6cd..84f7462e6b8 100644 --- a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb +++ b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration class MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb index 2257dc016be..00d7b1b9664 100644 --- a/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb +++ b/lib/gitlab/background_migration/populate_approval_merge_request_rules_with_security_orchestration.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't delete merge request level rules # as this feature exists only in EE class PopulateApprovalMergeRequestRulesWithSecurityOrchestration < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb index 1d0c0010551..e5f283db926 100644 --- a/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb +++ b/lib/gitlab/background_migration/populate_approval_project_rules_with_security_orchestration.rb @@ -5,6 +5,8 @@ module Gitlab # This class doesn't delete merge request level rules # as this feature exists only in EE class PopulateApprovalProjectRulesWithSecurityOrchestration < BatchedMigrationJob + feature_category :database + def perform; end end end diff --git a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb index 3dd867fa1fe..46758bc8fed 100644 --- a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb +++ b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb @@ -7,6 +7,7 @@ module Gitlab # The operations_access_level setting is being split into three seperate toggles. class PopulateOperationVisibilityPermissionsFromOperations < BatchedMigrationJob operation_name :populate_operations_visibility + feature_category :database def perform each_sub_batch do |batch| diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb index 085d576637e..8417dc91b1b 100644 --- a/lib/gitlab/background_migration/populate_projects_star_count.rb +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -7,6 +7,7 @@ module Gitlab MAX_UPDATE_RETRIES = 3 operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb index a91cda2c427..3b4b55276fa 100644 --- a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb +++ b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb @@ -8,6 +8,7 @@ module Gitlab scope_to ->(relation) { relation.where("updated_at < ?", EXPIRES_IN.ago) } operation_name :delete_all + feature_category :database def perform each_sub_batch(&:delete_all) diff --git a/lib/gitlab/background_migration/re_expire_o_auth_tokens.rb b/lib/gitlab/background_migration/re_expire_o_auth_tokens.rb new file mode 100644 index 00000000000..c327b14669d --- /dev/null +++ b/lib/gitlab/background_migration/re_expire_o_auth_tokens.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class ReExpireOAuthTokens < Gitlab::BackgroundMigration::ExpireOAuthTokens # rubocop:disable Migration/BackgroundMigrationBaseClass + end + # rubocop: enable Style/Documentation + end +end diff --git a/lib/gitlab/background_migration/recount_epic_cache_counts.rb b/lib/gitlab/background_migration/recount_epic_cache_counts.rb index 42f84a33a5a..cec17ef7cff 100644 --- a/lib/gitlab/background_migration/recount_epic_cache_counts.rb +++ b/lib/gitlab/background_migration/recount_epic_cache_counts.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # rubocop: disable Style/Documentation class RecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + def perform; end end # rubocop: enable Style/Documentation diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb index dc7c16d7947..7b88e10f39c 100644 --- a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -7,6 +7,7 @@ module Gitlab # These job artifacts will not be deleted and will have their `expire_at` removed. class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob operation_name :update_all + feature_category :database # The migration would have backfilled `expire_at` # to midnight on the 22nd of the month of the local timezone, diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb index a284c04d4f5..cf3897208b8 100644 --- a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -5,6 +5,7 @@ module Gitlab # Removes obsolete wiki notes class RemoveSelfManagedWikiNotes < BatchedMigrationJob operation_name :delete_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb index 1b13c2ab7ef..0615d8a6783 100644 --- a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -14,6 +14,7 @@ module Gitlab } operation_name :update_all + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb index 832385fd662..64eae1e934e 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -5,6 +5,7 @@ module Gitlab # A job to nullify duplicate token_encrypted values in ci_runners table in batches class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob operation_name :nullify_duplicate_ci_runner_token_encrypted_values + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb index 5f552accd8d..fd15caa5644 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -5,6 +5,7 @@ module Gitlab # A job to nullify duplicate token values in ci_runners table in batches class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob operation_name :nullify_duplicate_ci_runner_token_values + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb index 09cd3b1895f..0dbe2781327 100644 --- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -13,6 +13,7 @@ module Gitlab scope_to ->(relation) { relation.where(status: DELETE_SCHEDULED_STATUS) } operation_name :reset_status_on_container_repositories + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb index d3ef6ac3019..2df0b8a4d93 100644 --- a/lib/gitlab/background_migration/sanitize_confidential_todos.rb +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -13,6 +13,7 @@ module Gitlab scope_to ->(relation) { relation.where(confidential: true) } operation_name :delete_invalid_todos + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/second_recount_epic_cache_counts.rb b/lib/gitlab/background_migration/second_recount_epic_cache_counts.rb new file mode 100644 index 00000000000..4d7c4a682a9 --- /dev/null +++ b/lib/gitlab/background_migration/second_recount_epic_cache_counts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class SecondRecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::SecondRecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index dfd71bb8b5f..49ef75d7ba8 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -8,6 +8,7 @@ module Gitlab scope_to ->(relation) { relation.where.not(dismissed_at: nil) } operation_name :update_vulnerabilities_state + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb index 4ae7ad897cf..86fcfa18dc3 100644 --- a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb +++ b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb @@ -7,6 +7,7 @@ module Gitlab PUBLIC = 20 operation_name :set_legacy_open_source_license_available + feature_category :database # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord diff --git a/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles.rb b/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles.rb new file mode 100644 index 00000000000..5ae1698b910 --- /dev/null +++ b/lib/gitlab/background_migration/truncate_overlong_vulnerability_html_titles.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Truncate the Vulnerability html_title if it exceeds 800 chars + class TruncateOverlongVulnerabilityHtmlTitles < BatchedMigrationJob + feature_category :vulnerability_management + scope_to ->(relation) { relation.where("LENGTH(title_html) > 800") } + operation_name :truncate_vulnerability_title_htmls + + class Vulnerability < ApplicationRecord # rubocop:disable Style/Documentation + self.table_name = "vulnerabilities" + end + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all("title_html = left(title_html, 800)") + end + end + end + end +end diff --git a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb index 84183753158..77b4a9ab7e4 100644 --- a/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb +++ b/lib/gitlab/background_migration/update_ci_pipeline_artifacts_unknown_locked_status.rb @@ -9,6 +9,8 @@ module Gitlab # value of the associated `ci_pipelines.locked` value. This class # does an UPDATE join to make the values match. class UpdateCiPipelineArtifactsUnknownLockedStatus < BatchedMigrationJob + feature_category :database + def perform connection.exec_query(<<~SQL) UPDATE ci_pipeline_artifacts diff --git a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb index b2cf8298e4f..a7faa5703da 100644 --- a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb +++ b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb @@ -11,6 +11,7 @@ module Gitlab end operation_name :set_delayed_project_removal_to_null_for_user_namespace + feature_category :database def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb index 8aab7d13b45..6d59a5c8651 100644 --- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration class UpdateJiraTrackerDataDeploymentTypeBasedOnUrl < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :database + # rubocop: disable Gitlab/NamespacedClass class JiraTrackerData < ActiveRecord::Base self.table_name = "jira_tracker_data" diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb index 76f2a4ae38c..9b4cb9d0134 100644 --- a/lib/gitlab/chat_name_token.rb +++ b/lib/gitlab/chat_name_token.rb @@ -16,9 +16,7 @@ module Gitlab def get Gitlab::Redis::SharedState.with do |redis| data = redis.get(redis_shared_state_key) - params = Gitlab::Json.parse(data, symbolize_names: true) if data - params[:integration_id] ||= params.delete(:service_id) if params && params[:service_id] - params + Gitlab::Json.parse(data, symbolize_names: true) if data end end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index a5481071fc5..a635f409109 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_KEYS = %i[key untracked paths when policy unprotect].freeze ALLOWED_POLICY = %w[pull-push push pull].freeze DEFAULT_POLICY = 'pull-push' ALLOWED_WHEN = %w[on_success on_failure always].freeze @@ -33,18 +33,22 @@ module Gitlab entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' + entry :unprotect, ::Gitlab::Config::Entry::Boolean, + description: 'Unprotect the cache from a protected ref.' + entry :untracked, ::Gitlab::Config::Entry::Boolean, description: 'Cache all untracked files.' entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - attributes :policy, :when + attributes :policy, :when, :unprotect def value result = super result[:key] = key_value + result[:unprotect] = unprotect || false result[:policy] = policy || DEFAULT_POLICY # Use self.when to avoid conflict with reserved word result[:when] = self.when || DEFAULT_WHEN diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index e0a052ffdfd..e0f0903174c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -27,9 +27,9 @@ module Gitlab validates :config, disallowed_keys: { in: %i[only except start_in], - message: 'key may not be used with `rules`' - }, - if: :has_rules? + message: 'key may not be used with `rules`', + ignore_nil: true + }, if: :has_rules_value? with_options allow_nil: true do validates :extends, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb index 5c78a8f68c7..e91714e3f5c 100644 --- a/lib/gitlab/ci/config/entry/product/parallel.rb +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -12,7 +12,7 @@ module Gitlab strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) } - PARALLEL_LIMIT = 50 + PARALLEL_LIMIT = 200 class ParallelBuilds < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 16844fa88db..6408f412e6f 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -54,7 +54,7 @@ module Gitlab end def value - @config.transform_values do |value| + @config.compact.transform_values do |value| if value.is_a?(Hash) value else diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index decb568ffc9..a5c6aaa1e3a 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -54,9 +54,7 @@ module Gitlab validates :key, alphanumeric: true validates :config_value, alphanumeric: true, allow_nil: true validates :config_description, alphanumeric: true, allow_nil: true - validates :config_expand, boolean: true, allow_nil: true, if: -> { - ci_raw_variables_in_yaml_config_enabled? - } + validates :config_expand, boolean: true, allow_nil: true validates :config_options, array_of_strings: true, allow_nil: true validate do @@ -82,16 +80,10 @@ module Gitlab end def value_with_data - if ci_raw_variables_in_yaml_config_enabled? - { - value: config_value.to_s, - raw: (!config_expand if has_config_expand?) - }.compact - else - { - value: config_value.to_s - }.compact - end + { + value: config_value.to_s, + raw: (!config_expand if has_config_expand?) + }.compact end def value_with_prefill_data @@ -100,10 +92,6 @@ module Gitlab options: config_options ).compact end - - def ci_raw_variables_in_yaml_config_enabled? - YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) - end end class UnknownStrategy < ::Gitlab::Config::Entry::Node diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 21a57640aee..140cbfac5c1 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -38,10 +38,6 @@ module Gitlab private - def project - context&.parent_pipeline&.project - end - def validate_context! context.logger.instrument(:config_file_artifact_validate_context) do if !creating_child_pipeline? diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 65caf4ac47d..7899fe0ff73 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,7 +47,6 @@ module Gitlab end def validate! - context.check_execution_time! if ::Feature.disabled?(:ci_refactoring_external_mapper, context.project) validate_location! validate_context! if valid? fetch_and_validate_content! if valid? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index a41bc2b39f2..61b4d1ada10 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,18 +7,6 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - # Will be removed with FF ci_refactoring_external_mapper - FILE_CLASSES = [ - External::File::Local, - External::File::Project, - External::File::Remote, - External::File::Template, - External::File::Artifact - ].freeze - - # Will be removed with FF ci_refactoring_external_mapper - FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze - Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) @@ -32,11 +20,7 @@ module Gitlab return [] if @locations.empty? context.logger.instrument(:config_mapper_process) do - if ::Feature.enabled?(:ci_refactoring_external_mapper, context.project) - process_without_instrumentation - else - legacy_process_without_instrumentation - end + process_without_instrumentation end end @@ -57,138 +41,6 @@ module Gitlab files end - - # This and the following methods will be removed with FF ci_refactoring_external_mapper - def legacy_process_without_instrumentation - @locations - .map(&method(:normalize_location)) - .filter_map(&method(:verify_rules)) - .flat_map(&method(:expand_project_files)) - .flat_map(&method(:expand_wildcard_paths)) - .map(&method(:expand_variables)) - .map(&method(:select_first_matching)) - .each(&method(:verify!)) - end - - # convert location if String to canonical form - def normalize_location(location) - if location.is_a?(String) - expanded_location = expand_variables(location) - normalize_location_string(expanded_location) - else - location.deep_symbolize_keys - end - end - - def verify_rules(location) - logger.instrument(:config_mapper_rules) do - verify_rules_without_instrumentation(location) - end - end - - def verify_rules_without_instrumentation(location) - return unless Rules.new(location[:rules]).evaluate(context).pass? - - location - end - - def expand_project_files(location) - return location unless location[:project] - - Array.wrap(location[:file]).map do |file| - location.merge(file: file) - end - end - - def expand_wildcard_paths(location) - logger.instrument(:config_mapper_wildcards) do - expand_wildcard_paths_without_instrumentation(location) - end - end - - def expand_wildcard_paths_without_instrumentation(location) - # We only support local files for wildcard paths - return location unless location[:local] && location[:local].include?('*') - - context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| - { local: path } - end - end - - def normalize_location_string(location) - if ::Gitlab::UrlSanitizer.valid?(location) - { remote: location } - else - { local: location } - end - end - - def select_first_matching(location) - logger.instrument(:config_mapper_select) do - select_first_matching_without_instrumentation(location) - end - end - - def select_first_matching_without_instrumentation(location) - matching = FILE_CLASSES.map do |file_class| - file_class.new(location, context) - end.select(&:matching?) - - if matching.one? - matching.first - elsif matching.empty? - raise AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" - else - raise AmbigiousSpecificationError, "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" - end - end - - def verify!(location_object) - verify_max_includes! - location_object.validate! - expandset.add(location_object) - end - - def verify_max_includes! - if expandset.count >= context.max_includes - raise TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" - end - end - - def expand_variables(data) - logger.instrument(:config_mapper_variables) do - expand_variables_without_instrumentation(data) - end - end - - def expand_variables_without_instrumentation(data) - if data.is_a?(String) - expand(data) - else - transform(data) - end - end - - def transform(data) - data.transform_values do |values| - case values - when Array - values.map { |value| expand(value.to_s) } - when String - expand(values) - else - values - end - end - end - - def expand(data) - ExpandVariables.expand(data, -> { context.variables_hash }) - end - - def masked_location(location) - context.mask_variables_from(location) - end end end end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index ab5203252a2..e6a2e5c3b33 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -17,7 +17,7 @@ module Gitlab secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] }.freeze - VERSIONS_TO_REMOVE_IN_16_0 = [].freeze + VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze DEPRECATED_VERSIONS = { cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0, @@ -30,6 +30,8 @@ module Gitlab secret_detection: VERSIONS_TO_REMOVE_IN_16_0 }.freeze + CURRENT_VERSIONS = SUPPORTED_VERSIONS.to_h { |k, v| [k, v - DEPRECATED_VERSIONS[k]] } + class Schema def root_path File.join(__dir__, 'schemas') @@ -129,6 +131,11 @@ module Gitlab end def report_uses_deprecated_schema_version? + # Avoid deprecation warnings for GitLab security scanners + # To be removed via https://gitlab.com/gitlab-org/gitlab/-/issues/386798 + return if report_data.dig('scan', 'scanner', 'vendor', 'name')&.downcase == 'gitlab' + return if report_data.dig('scan', 'analyzer', 'vendor', 'name')&.downcase == 'gitlab' + DEPRECATED_VERSIONS[report_type].include?(report_version) end @@ -182,11 +189,15 @@ module Gitlab def add_deprecated_report_version_message log_warnings(problem_type: 'using_deprecated_schema_version') - template = _("Version %{report_version} for report type %{report_type} has been deprecated,"\ - " supported versions for this report type are: %{supported_schema_versions}."\ - " GitLab will attempt to parse and ingest this report if valid.") + template = _("version %{report_version} for report type %{report_type} is deprecated. "\ + "However, GitLab will still attempt to parse and ingest this report. "\ + "Upgrade the security report to one of the following versions: %{current_schema_versions}.") - message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + message = format( + template, + report_version: report_version, + report_type: report_type, + current_schema_versions: current_schema_versions) add_message_as(level: :deprecation_warning, message: message) end @@ -207,6 +218,10 @@ module Gitlab ) end + def current_schema_versions + CURRENT_VERSIONS[report_type].join(", ") + end + def supported_schema_versions SUPPORTED_VERSIONS[report_type].join(", ") end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 31b130b5ab7..d2dc712e366 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -127,6 +127,10 @@ module Gitlab .observe({ plan: project.actual_plan_name }, jobs_count) end + def observe_pipeline_includes_count(pipeline) + logger.observe(:pipeline_includes_count, pipeline.config_metadata&.[](:includes)&.count, once: true) + end + def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb index a8276d84b87..99e438ddbae 100644 --- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -6,7 +6,7 @@ module Gitlab module Chain class CreateDeployments < Chain::Base def perform! - create_deployments! + create_deployments! if Feature.disabled?(:move_create_deployments_to_worker, pipeline.project) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 654e24be8e1..c59ef2ba6a4 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -18,7 +18,8 @@ module Gitlab pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? - return error('No stages / jobs for this pipeline.') + return error('Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.') end if pipeline.invalid? diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb index 89befb2a65b..e7a9009f8f4 100644 --- a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -22,8 +22,7 @@ module Gitlab private def set_pipeline_name - return if Feature.disabled?(:pipeline_name, pipeline.project) || - @command.yaml_processor_result.workflow_name.blank? + return if @command.yaml_processor_result.workflow_name.blank? name = @command.yaml_processor_result.workflow_name name = ExpandVariables.expand(name, -> { global_context.variables.sort_and_expand_all }) diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index de147914850..dd097187955 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -30,6 +30,7 @@ module Gitlab @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines + @command.observe_pipeline_includes_count(@pipeline) @pipeline end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index f393406b549..8286dfc6560 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -121,7 +121,7 @@ module Gitlab def enabled? ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end - strong_memoize_attr :enabled?, :enabled + strong_memoize_attr :enabled? def observations @observations ||= {} diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index b0b79b994c1..684b58474ad 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -53,7 +53,7 @@ module Gitlab end end end - strong_memoize_attr :included?, :inclusion + strong_memoize_attr :included? def errors logger.instrument(:pipeline_seed_build_errors) do @@ -261,7 +261,7 @@ module Gitlab def reuse_build_in_seed_context? Feature.enabled?(:ci_reuse_build_in_seed_context, @pipeline.project) end - strong_memoize_attr :reuse_build_in_seed_context?, :reuse_build_in_seed_context + strong_memoize_attr :reuse_build_in_seed_context? end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 781065a63db..409b6658cc0 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -14,6 +14,7 @@ module Gitlab @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) @when = local_cache.delete(:when) + @unprotect = local_cache.delete(:unprotect) @custom_key_prefix = custom_key_prefix raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? @@ -25,7 +26,8 @@ module Gitlab paths: @paths, policy: @policy, untracked: @untracked, - when: @when + when: @when, + unprotect: @unprotect }.compact end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index 0074f3675e0..5e77db3d336 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -22,14 +22,26 @@ module Gitlab def illustration_content if can?(user, :update_build, subject) - _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + manual_job_action_message else generic_permission_failure_message end end + def manual_job_action_message + if subject.retryable? + _("You can modify this job's CI/CD variables before running it again.") + else + _('This job does not start automatically and must be started manually. You can add CI/CD variables below for last-minute configuration changes before starting the job.') + end + end + def generic_permission_failure_message - _("This job does not run automatically and must be started manually, but you do not have access to it.") + if subject.outdated_deployment? + _("This deployment job does not run automatically and must be started manually, but it's older than the latest deployment, and therefore can't run.") + else + _("This job does not run automatically and must be started manually, but you do not have access to it.") + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b4beeb60dfd..47b79302828 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.87.3" + CODE_QUALITY_IMAGE_TAG: "0.89.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7a208584c4c..6884a9556b4 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 292b0a0036d..dc7e5f445d2 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index ba03ad6304f..9e15b07f5d1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.45.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 2c5027cdb43..8b49d2de8cf 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -255,7 +255,7 @@ sobelow-sast: when: never - if: $CI_COMMIT_BRANCH exists: - - 'mix.exs' + - '**/mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 58709d3ab62..1c4dbe6cd0f 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -332,12 +332,12 @@ sobelow-sast: when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - - 'mix.exs' + - '**/mix.exs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. exists: - - 'mix.exs' + - '**/mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index e9766061072..9960a6fbdf5 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,8 +72,36 @@ module Gitlab Collection.new(@variables.reject(&block)) end - # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. - def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) + def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true) + sorted = Sort.new(self) + return self.class.new(self, sorted.errors) unless sorted.valid? + + new_collection = self.class.new + + sorted.tsort.each do |item| + unless item.depends_on + new_collection.append(item) + next + end + + # expand variables as they are added + variable = item.to_runner_variable + variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, + expand_file_refs: expand_file_refs, + expand_raw_refs: expand_raw_refs) + new_collection.append(variable) + end + + new_collection + end + + def to_s + "#{@variables_by_key.keys}, @errors='#{@errors}'" + end + + protected + + def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -88,19 +116,20 @@ module Gitlab if variable # VARIABLE_NAME is an existing variable if variable.file? - # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 - if project - # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` - # when the variables are sent to Runner. - Gitlab::AppJsonLogger.info(event: 'file_variable_is_referenced_in_another_variable', - project_id: project.id, - variable: variable_name) - end - expand_file_refs ? variable.value : full_match elsif variable.raw? - # With `full_match`, we defer the expansion of raw variables to the runner. If we expand them here, - # the runner will not know the expanded value is a raw variable and it tries to expand it again. + # Normally, it's okay to expand a raw variable if it's referenced in another variable because + # its rawness is not broken. However, the runner also tries to expand variables. + # Here, with `full_match`, we defer the expansion of raw variables to the runner. + # If we expand them here, the runner will not know that the expanded value is a raw variable + # and it tries to expand it again. + # Example: `A` is a normal variable with value `normal`. + # `B` is a raw variable with value `raw-$A`. + # `C` is a normal variable with value `$B`. + # If we expanded `C` here, the runner would receive `C` as `raw-$A`. And since `A` is a normal + # variable, the runner would expand it. So, the result would be `raw-normal`. + # With `full_match`, the runner receives `C` as `$B`. And since `B` is a raw variable, the + # runner expanded it as `raw-$A`, which is what we want. # Discussion: https://gitlab.com/gitlab-org/gitlab/-/issues/353991#note_1103274951 expand_raw_refs ? variable.value : full_match else @@ -115,36 +144,7 @@ module Gitlab end end - # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. - def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) - sorted = Sort.new(self) - return self.class.new(self, sorted.errors) unless sorted.valid? - - new_collection = self.class.new - - sorted.tsort.each do |item| - unless item.depends_on - new_collection.append(item) - next - end - - # expand variables as they are added - variable = item.to_runner_variable - variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_refs: expand_file_refs, - expand_raw_refs: expand_raw_refs, - project: project) - new_collection.append(variable) - end - - new_collection - end - - def to_s - "#{@variables_by_key.keys}, @errors='#{@errors}'" - end - - protected + private attr_reader :variables end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f2c1ad0575d..d867439b10b 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -64,12 +64,7 @@ module Gitlab private def assign_valid_attributes - @root_variables = if YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) - transform_to_array(@ci_config.variables_with_data) - else - transform_to_array(@ci_config.variables) - end - + @root_variables = transform_to_array(@ci_config.variables_with_data) @root_variables_with_prefill_data = @ci_config.variables_with_prefill_data @stages = @ci_config.stages diff --git a/lib/gitlab/config/entry/attributable.rb b/lib/gitlab/config/entry/attributable.rb index c8ad2521574..2e5b226678a 100644 --- a/lib/gitlab/config/entry/attributable.rb +++ b/lib/gitlab/config/entry/attributable.rb @@ -24,6 +24,10 @@ module Gitlab define_method("has_#{attribute_method}?") do config.is_a?(Hash) && config.key?(attribute) end + + define_method("has_#{attribute_method}_value?") do + config.is_a?(Hash) && config.key?(attribute) && !config[attribute].nil? + end end end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index b88a6766d92..9e6a3d86e92 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -17,6 +17,7 @@ module Gitlab class DisallowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) + value = value.try(:compact) if options[:ignore_nil] present_keys = value.try(:keys).to_a & options[:in] if present_keys.any? diff --git a/lib/gitlab/counters.rb b/lib/gitlab/counters.rb new file mode 100644 index 00000000000..5ff664f53bd --- /dev/null +++ b/lib/gitlab/counters.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module Counters + Increment = Struct.new(:amount, :ref, keyword_init: true) + end +end diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb index 56593b642a9..3e232c78e45 100644 --- a/lib/gitlab/counters/buffered_counter.rb +++ b/lib/gitlab/counters/buffered_counter.rb @@ -8,6 +8,17 @@ module Gitlab WORKER_DELAY = 10.minutes WORKER_LOCK_TTL = 10.minutes + # Refresh keys are set to expire after a very long time, + # so that they do not occupy Redis memory indefinitely, + # if for any reason they are not deleted. + # In practice, a refresh is not expected to take longer than this TTL. + REFRESH_KEYS_TTL = 14.days + CLEANUP_BATCH_SIZE = 50 + CLEANUP_INTERVAL_SECONDS = 0.1 + + # Limit size of bitmap key to 2^26-1 (~8MB) + MAX_BITMAP_OFFSET = 67108863 + LUA_FLUSH_INCREMENT_SCRIPT = <<~LUA local increment_key, flushed_key = KEYS[1], KEYS[2] local increment_value = redis.call("get", increment_key) or 0 @@ -31,9 +42,47 @@ module Gitlab end end - def increment(amount) + LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT = <<~LUA + local counter_key, refresh_key, refresh_indicator_key = KEYS[1], KEYS[2], KEYS[3] + local tracking_shard_key, opposing_tracking_shard_key, shards_key = KEYS[4], KEYS[5], KEYS[6] + + local amount, tracking_offset = tonumber(ARGV[1]), tonumber(ARGV[2]) + + -- increment to the counter key when not refreshing + if redis.call("exists", refresh_indicator_key) == 0 then + return redis.call("incrby", counter_key, amount) + end + + -- deduplicate and increment to the refresh counter key while refreshing + local found_duplicate = redis.call("getbit", tracking_shard_key, tracking_offset) + if found_duplicate == 1 then + return redis.call("get", refresh_key) + end + + redis.call("setbit", tracking_shard_key, tracking_offset, 1) + redis.call("expire", tracking_shard_key, #{REFRESH_KEYS_TTL.seconds}) + redis.call("sadd", shards_key, tracking_shard_key) + redis.call("expire", shards_key, #{REFRESH_KEYS_TTL.seconds}) + + local found_opposing_change = redis.call("getbit", opposing_tracking_shard_key, tracking_offset) + local increment_without_previous_decrement = amount > 0 and found_opposing_change == 0 + local decrement_with_previous_increment = amount < 0 and found_opposing_change == 1 + local net_change = 0 + + if increment_without_previous_decrement or decrement_with_previous_increment then + net_change = amount + end + + return redis.call("incrby", refresh_key, net_change) + LUA + + def increment(increment) result = redis_state do |redis| - redis.incrby(key, amount) + if Feature.enabled?(:project_statistics_bulk_increment, type: :development) + redis.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)).to_i + else + redis.incrby(key, increment.amount) + end end FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) @@ -41,11 +90,63 @@ module Gitlab result end - def reset! + def bulk_increment(increments) + result = redis_state do |redis| + redis.pipelined do |pipeline| + increments.each do |increment| + if Feature.enabled?(:project_statistics_bulk_increment, type: :development) + pipeline.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)) + else + pipeline.incrby(key, increment.amount) + end + end + end + end + + FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) + + result.last.to_i + end + + LUA_INITIATE_REFRESH_SCRIPT = <<~LUA + local counter_key, refresh_indicator_key = KEYS[1], KEYS[2] + redis.call("del", counter_key) + redis.call("set", refresh_indicator_key, 1, "ex", #{REFRESH_KEYS_TTL.seconds}) + LUA + + def initiate_refresh! counter_record.update!(attribute => 0) redis_state do |redis| - redis.del(key) + redis.eval(LUA_INITIATE_REFRESH_SCRIPT, keys: [key, refresh_indicator_key]) + end + end + + LUA_FINALIZE_REFRESH_SCRIPT = <<~LUA + local counter_key, refresh_key, refresh_indicator_key = KEYS[1], KEYS[2], KEYS[3] + local refresh_amount = redis.call("get", refresh_key) or 0 + + redis.call("incrby", counter_key, refresh_amount) + redis.call("del", refresh_indicator_key, refresh_key) + LUA + + def finalize_refresh + redis_state do |redis| + redis.eval(LUA_FINALIZE_REFRESH_SCRIPT, keys: [key, refresh_key, refresh_indicator_key]) + end + + FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) + ::Counters::CleanupRefreshWorker.perform_async(counter_record.class.name, counter_record.id, attribute) + end + + def cleanup_refresh + redis_state do |redis| + while (shards = redis.spop(shards_key, CLEANUP_BATCH_SIZE)) + redis.del(*shards) + break if shards.size < CLEANUP_BATCH_SIZE + + sleep CLEANUP_INTERVAL_SECONDS + end end end @@ -87,10 +188,67 @@ module Gitlab "#{key}:flushed" end + def refresh_indicator_key + "#{key}:refresh-in-progress" + end + + def refresh_key + "#{key}:refresh" + end + private attr_reader :counter_record, :attribute + def increment_args(increment) + { + keys: [ + key, + refresh_key, + refresh_indicator_key, + tracking_shard_key(increment), + opposing_tracking_shard_key(increment), + shards_key + ], + argv: [ + increment.amount, + tracking_offset(increment) + ] + } + end + + def tracking_shard_key(increment) + positive?(increment) ? positive_shard_key(increment.ref.to_i) : negative_shard_key(increment.ref.to_i) + end + + def opposing_tracking_shard_key(increment) + positive?(increment) ? negative_shard_key(increment.ref.to_i) : positive_shard_key(increment.ref.to_i) + end + + def shards_key + "#{refresh_key}:shards" + end + + def positive_shard_key(ref) + "#{refresh_key}:+:#{shard_number(ref)}" + end + + def negative_shard_key(ref) + "#{refresh_key}:-:#{shard_number(ref)}" + end + + def shard_number(ref) + ref / MAX_BITMAP_OFFSET + end + + def tracking_offset(increment) + increment.ref.to_i % MAX_BITMAP_OFFSET + end + + def positive?(increment) + increment.amount > 0 + end + def remove_flushed_key redis_state do |redis| redis.del(flushed_key) diff --git a/lib/gitlab/counters/legacy_counter.rb b/lib/gitlab/counters/legacy_counter.rb index 06951514ec3..823f9955168 100644 --- a/lib/gitlab/counters/legacy_counter.rb +++ b/lib/gitlab/counters/legacy_counter.rb @@ -11,23 +11,36 @@ module Gitlab @current_value = counter_record.method(attribute).call end - def increment(amount) - updated = counter_record.class.update_counters(counter_record.id, { attribute => amount }) + def increment(increment) + updated = update_counter_record_attribute(increment.amount) if updated == 1 counter_record.execute_after_commit_callbacks - @current_value += amount + @current_value += increment.amount end @current_value end - def reset! - counter_record.update!(attribute => 0) + def bulk_increment(increments) + total_increment = increments.sum(&:amount) + + updated = update_counter_record_attribute(total_increment) + + if updated == 1 + counter_record.execute_after_commit_callbacks + @current_value += total_increment + end + + @current_value end private + def update_counter_record_attribute(amount) + counter_record.class.update_counters(counter_record.id, { attribute => amount }) + end + attr_reader :counter_record, :attribute end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 8eda871770b..8fec5cf3303 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -45,6 +45,7 @@ module Gitlab commit: { # note: commit.id is actually the pipeline id id: commit.id, + name: commit.name, sha: commit.sha, message: commit.git_commit_message, author_name: commit.git_author_name, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 51d5bfcee38..40e2e637114 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -84,7 +84,7 @@ module Gitlab # # TODO: https://gitlab.com/gitlab-org/geo-team/discussions/-/issues/5032 def self.database_base_models_using_load_balancing - @database_base_models_with_gitlab_shared ||= { + @database_base_models_using_load_balancing ||= { # Note that we use ActiveRecord::Base here and not ApplicationRecord. # This is deliberate, as we also use these classes to apply load # balancing to, and the load balancer must be enabled for _all_ models diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index a04ea97117d..417e9f211b9 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -25,7 +25,7 @@ module Gitlab # Note: to be deleted after the minimum PG version is set to 12.0 # Update the documentation together when deleting the method - # https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html#use-ctes-wisely + # https://docs.gitlab.com/ee/development/merge_request_concepts/performance.html#use-ctes-wisely def self.materialized_if_supported materialized_supported? ? 'MATERIALIZED' : '' end diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 2fb4cc8f675..3ae2bb7b3e5 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -4,10 +4,10 @@ module Gitlab module Database module AsyncIndexes class IndexCreator - include ExclusiveLeaseGuard + include IndexingExclusiveLeaseGuard TIMEOUT_PER_ACTION = 1.day - STATEMENT_TIMEOUT = 9.hours + STATEMENT_TIMEOUT = 20.hours def initialize(async_index) @async_index = async_index @@ -47,10 +47,6 @@ module Gitlab TIMEOUT_PER_ACTION end - def lease_key - [super, async_index.connection_db_config.name].join('/') - end - def set_statement_timeout connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) yield diff --git a/lib/gitlab/database/async_indexes/index_destructor.rb b/lib/gitlab/database/async_indexes/index_destructor.rb index fe05872b87a..66955df9d04 100644 --- a/lib/gitlab/database/async_indexes/index_destructor.rb +++ b/lib/gitlab/database/async_indexes/index_destructor.rb @@ -4,7 +4,7 @@ module Gitlab module Database module AsyncIndexes class IndexDestructor - include ExclusiveLeaseGuard + include IndexingExclusiveLeaseGuard TIMEOUT_PER_ACTION = 1.day @@ -53,10 +53,6 @@ module Gitlab TIMEOUT_PER_ACTION end - def lease_key - [super, async_index.connection_db_config.name].join('/') - end - def log_index_info(message) Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index ad747a8131d..f1fc3efae9e 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -49,6 +49,8 @@ module Gitlab def execute_job(tracking_record) job_class = tracking_record.migration_job_class + ApplicationContext.push(feature_category: fetch_feature_category(job_class)) + if job_class < Gitlab::BackgroundMigration::BatchedMigrationJob execute_batched_migration_job(job_class, tracking_record) else @@ -86,6 +88,14 @@ module Gitlab job_instance end + + def fetch_feature_category(job_class) + if job_class.respond_to?(:feature_category) + job_class.feature_category.to_s + else + Gitlab::BackgroundMigration::BatchedMigrationJob::DEFAULT_FEATURE_CATEGORY + end + end end end end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 0f848ed40fb..38558512b6a 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -15,46 +15,12 @@ module Gitlab module Database module GitlabSchema + UnknownSchemaError = Class.new(StandardError) + DICTIONARY_PATH = 'db/docs/' - # These tables are deleted/renamed, but still referenced by migrations. - # This is needed for now, but should be removed in the future - DELETED_TABLES = { - # main tables - 'alerts_service_data' => :gitlab_main, - 'analytics_devops_adoption_segment_selections' => :gitlab_main, - 'analytics_repository_file_commits' => :gitlab_main, - 'analytics_repository_file_edits' => :gitlab_main, - 'analytics_repository_files' => :gitlab_main, - 'audit_events_archived' => :gitlab_main, - 'backup_labels' => :gitlab_main, - 'clusters_applications_fluentd' => :gitlab_main, - 'forked_project_links' => :gitlab_main, - 'issue_milestones' => :gitlab_main, - 'merge_request_milestones' => :gitlab_main, - 'namespace_onboarding_actions' => :gitlab_main, - 'services' => :gitlab_main, - 'terraform_state_registry' => :gitlab_main, - 'tmp_fingerprint_sha256_migration' => :gitlab_main, # used by lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb - 'web_hook_logs_archived' => :gitlab_main, - 'vulnerability_export_registry' => :gitlab_main, - 'vulnerability_finding_fingerprints' => :gitlab_main, - 'vulnerability_export_verification_status' => :gitlab_main, - - # CI tables - 'ci_build_trace_sections' => :gitlab_ci, - 'ci_build_trace_section_names' => :gitlab_ci, - 'ci_daily_report_results' => :gitlab_ci, - 'ci_test_cases' => :gitlab_ci, - 'ci_test_case_failures' => :gitlab_ci, - - # leftovers from early implementation of partitioning - 'audit_events_part_5fc467ac26' => :gitlab_main, - 'web_hook_logs_part_0c5294f417' => :gitlab_main - }.freeze - - def self.table_schemas(tables) - tables.map { |table| table_schema(table) }.to_set + def self.table_schemas(tables, undefined: true) + tables.map { |table| table_schema(table, undefined: undefined) }.to_set end def self.table_schema(name, undefined: true) @@ -69,13 +35,13 @@ module Gitlab # strip partition number of a form `loose_foreign_keys_deleted_records_1` table_name.gsub!(/_[0-9]+$/, '') - # Tables that are properly mapped + # Tables and views that are properly mapped if gitlab_schema = views_and_tables_to_schema[table_name] return gitlab_schema end - # Tables that are deleted, but we still need to reference them - if gitlab_schema = DELETED_TABLES[table_name] + # Tables and views that are deleted, but we still need to reference them + if gitlab_schema = deleted_views_and_tables_to_schema[table_name] return gitlab_schema end @@ -106,29 +72,58 @@ module Gitlab [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] end + def self.deleted_views_path_globs + [Rails.root.join(DICTIONARY_PATH, 'deleted_views', '*.yml')] + end + + def self.deleted_tables_path_globs + [Rails.root.join(DICTIONARY_PATH, 'deleted_tables', '*.yml')] + end + def self.views_and_tables_to_schema @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end - def self.tables_to_schema - @tables_to_schema ||= Dir.glob(self.dictionary_path_globs).each_with_object({}) do |file_path, dic| - data = YAML.load_file(file_path) + def self.table_schema!(name) + self.table_schema(name, undefined: false) || raise( + UnknownSchemaError, + "Could not find gitlab schema for table #{name}: Any new tables must be added to the database dictionary" + ) + end - dic[data['table_name']] = data['gitlab_schema'].to_sym - end + def self.deleted_views_and_tables_to_schema + @deleted_views_and_tables_to_schema ||= self.deleted_tables_to_schema.merge(self.deleted_views_to_schema) end - def self.views_to_schema - @views_to_schema ||= Dir.glob(self.view_path_globs).each_with_object({}) do |file_path, dic| - data = YAML.load_file(file_path) + def self.deleted_tables_to_schema + @deleted_tables_to_schema ||= self.build_dictionary(self.deleted_tables_path_globs) + end - dic[data['view_name']] = data['gitlab_schema'].to_sym - end + def self.deleted_views_to_schema + @deleted_views_to_schema ||= self.build_dictionary(self.deleted_views_path_globs) + end + + def self.tables_to_schema + @tables_to_schema ||= self.build_dictionary(self.dictionary_path_globs) + end + + def self.views_to_schema + @views_to_schema ||= self.build_dictionary(self.view_path_globs) end def self.schema_names @schema_names ||= self.views_and_tables_to_schema.values.to_set end + + private_class_method def self.build_dictionary(path_globs) + Dir.glob(path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + key_name = data['table_name'] || data['view_name'] + + dic[key_name] = data['gitlab_schema'].to_sym + end + end end end end diff --git a/lib/gitlab/database/indexing_exclusive_lease_guard.rb b/lib/gitlab/database/indexing_exclusive_lease_guard.rb new file mode 100644 index 00000000000..fb45de347e6 --- /dev/null +++ b/lib/gitlab/database/indexing_exclusive_lease_guard.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module IndexingExclusiveLeaseGuard + extend ActiveSupport::Concern + include ExclusiveLeaseGuard + + def lease_key + @lease_key ||= "gitlab/database/indexing/actions/#{database_config_name}" + end + + def database_config_name + Gitlab::Database.db_config_name(connection) + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/resolver.rb b/lib/gitlab/database/load_balancing/resolver.rb index a291080cc3d..3e3998cae92 100644 --- a/lib/gitlab/database/load_balancing/resolver.rb +++ b/lib/gitlab/database/load_balancing/resolver.rb @@ -7,8 +7,21 @@ module Gitlab module Database module LoadBalancing class Resolver + FAR_FUTURE_TTL = 100.years.from_now + UnresolvableNameserverError = Class.new(StandardError) + Response = Class.new do + attr_reader :address, :ttl + + def initialize(address:, ttl:) + raise ArgumentError unless ttl.present? && address.present? + + @address = address + @ttl = ttl + end + end + def initialize(nameserver) @nameserver = nameserver end @@ -28,13 +41,14 @@ module Gitlab private def ip_address - IPAddr.new(@nameserver) + # IP addresses are valid forever + Response.new(address: IPAddr.new(@nameserver), ttl: FAR_FUTURE_TTL) rescue IPAddr::InvalidAddressError end def ip_address_from_hosts_file ip = Resolv::Hosts.new.getaddress(@nameserver) - IPAddr.new(ip) + Response.new(address: IPAddr.new(ip), ttl: FAR_FUTURE_TTL) rescue Resolv::ResolvError end @@ -42,7 +56,12 @@ module Gitlab answer = Net::DNS::Resolver.start(@nameserver, Net::DNS::A).answer return if answer.empty? - answer.first.address + raw_response = answer.first + + # Defaults to 30 seconds if there is no TTL present + ttl_in_seconds = raw_response.ttl.presence || 30 + + Response.new(address: answer.first.address, ttl: ttl_in_seconds.seconds.from_now) rescue Net::DNS::Resolver::NoResponseError raise UnresolvableNameserverError, "no response from DNS server(s)" end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 3295301a2d7..5059b3b5c93 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -69,6 +69,7 @@ module Gitlab @use_tcp = use_tcp @load_balancer = load_balancer @max_replica_pools = max_replica_pools + @nameserver_ttl = 1.second.ago # Begin with an expired ttl to trigger a nameserver dns lookup end # rubocop:enable Metrics/ParameterLists @@ -191,8 +192,14 @@ module Gitlab end def resolver - @resolver ||= Net::DNS::Resolver.new( - nameservers: Resolver.new(@nameserver).resolve, + return @resolver if defined?(@resolver) && @nameserver_ttl.future? + + response = Resolver.new(@nameserver).resolve + + @nameserver_ttl = response.ttl + + @resolver = Net::DNS::Resolver.new( + nameservers: response.address, port: @port, use_tcp: @use_tcp ) diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index e3ae2892668..2e08e1ffb42 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -22,37 +22,38 @@ module Gitlab end end - def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) + def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger @dry_run = dry_run + @with_retries = with_retries @table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils .extract_schema_qualified_name(table_name) .identifier end - def table_locked_for_writes?(table_name) + def table_locked_for_writes? query = <<~SQL SELECT COUNT(*) from information_schema.triggers WHERE event_object_table = '#{table_name_without_schema}' - AND trigger_name = '#{write_trigger_name(table_name)}' + AND trigger_name = '#{write_trigger_name}' SQL connection.select_value(query) == EXPECTED_TRIGGER_RECORD_COUNT end def lock_writes - if table_locked_for_writes?(table_name) + if table_locked_for_writes? logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes" return end logger&.info "Database: '#{database_name}', Table: '#{table_name}': Lock Writes".color(:yellow) sql_statement = <<~SQL - CREATE TRIGGER #{write_trigger_name(table_name)} + CREATE TRIGGER #{write_trigger_name} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON #{table_name} FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); @@ -64,7 +65,7 @@ module Gitlab def unlock_writes logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green) sql_statement = <<~SQL - DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; + DROP TRIGGER IF EXISTS #{write_trigger_name} ON #{table_name}; SQL execute_sql_statement(sql_statement) @@ -72,19 +73,23 @@ module Gitlab private - attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema + attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema, :with_retries def execute_sql_statement(sql) if dry_run logger&.info sql - else - with_retries(connection) do + elsif with_retries + raise "Cannot call lock_retries_helper if a transaction is already open" if connection.transaction_open? + + run_with_retries(connection) do connection.execute(sql) end + else + connection.execute(sql) end end - def with_retries(connection, &block) + def run_with_retries(connection, &block) with_statement_timeout_retries do with_lock_retries(connection) do yield @@ -110,11 +115,12 @@ module Gitlab Gitlab::Database::WithLockRetries.new( klass: "gitlab:db:lock_writes", logger: logger || Gitlab::AppLogger, - connection: connection + connection: connection, + allow_savepoints: false # this causes the WithLockRetries to fail if sub-transaction has been detected. ).run(&block) end - def write_trigger_name(table_name) + def write_trigger_name "gitlab_schema_write_trigger_for_#{table_name_without_schema}" end end diff --git a/lib/gitlab/database/loose_foreign_keys.rb b/lib/gitlab/database/loose_foreign_keys.rb index 1338b18a099..6512c672965 100644 --- a/lib/gitlab/database/loose_foreign_keys.rb +++ b/lib/gitlab/database/loose_foreign_keys.rb @@ -22,7 +22,7 @@ module Gitlab { column: config.fetch('column'), on_delete: config.fetch('on_delete').to_sym, - gitlab_schema: GitlabSchema.table_schema(child_table_name) + gitlab_schema: GitlabSchema.table_schema!(child_table_name) } ) end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 4858a96c173..e41107370ec 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -281,6 +281,9 @@ module Gitlab # target_column - The name of the referenced column, defaults to "id". # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # on_update - The action to perform when associated data is updated, + # defaults to nil. This is useful for multi column FKs if + # it's desirable to update one of the columns. # name - The name of the foreign key. # validate - Flag that controls whether the new foreign key will be validated after creation. # If the flag is not set, the constraint will only be enforced for new data. @@ -288,7 +291,8 @@ module Gitlab # order of the ALTER TABLE. This can be useful in situations where the foreign # key creation could deadlock with another process. # - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true, reverse_lock_order: false) + # rubocop: disable Metrics/ParameterLists + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, on_update: nil, target_column: :id, name: nil, validate: true, reverse_lock_order: false) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -298,6 +302,7 @@ module Gitlab options = { column: column, on_delete: on_delete, + on_update: on_update, name: name.presence || concurrent_foreign_key_name(source, column), primary_key: target_column } @@ -306,7 +311,8 @@ module Gitlab warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ - "name: #{options[:name]}, on_delete: #{options[:on_delete]}" + "name: #{options[:name]}, on_update: #{options[:on_update]}, "\ + "on_delete: #{options[:on_delete]}" Gitlab::AppLogger.warn warning_message else @@ -322,6 +328,7 @@ module Gitlab ADD CONSTRAINT #{options[:name]} FOREIGN KEY (#{multiple_columns(options[:column])}) REFERENCES #{target} (#{multiple_columns(target_column)}) + #{on_update_statement(options[:on_update])} #{on_delete_statement(options[:on_delete])} NOT VALID; EOF @@ -343,6 +350,7 @@ module Gitlab end end end + # rubocop: enable Metrics/ParameterLists def validate_foreign_key(source, column, name: nil) fk_name = name || concurrent_foreign_key_name(source, column) @@ -357,10 +365,28 @@ module Gitlab end def foreign_key_exists?(source, target = nil, **options) - foreign_keys(source).any? do |foreign_key| - tables_match?(target.to_s, foreign_key.to_table.to_s) && - options_match?(foreign_key.options, options) + # This if block is necessary because foreign_key_exists? is called in down migrations that may execute before + # the postgres_foreign_keys view had necessary columns added, or even before the view existed. + # In that case, we revert to the previous behavior of this method. + # The behavior in the if block has a bug: it always returns false if the fk being checked has multiple columns. + # This can be removed after init_schema.rb passes 20221122210711_add_columns_to_postgres_foreign_keys.rb + # Tracking issue: https://gitlab.com/gitlab-org/gitlab/-/issues/386796 + if ActiveRecord::Migrator.current_version < 20221122210711 + return foreign_keys(source).any? do |foreign_key| + tables_match?(target.to_s, foreign_key.to_table.to_s) && + options_match?(foreign_key.options, options) + end end + + fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name(source) + + fks = fks.by_referenced_table_name(target) if target + fks = fks.by_name(options[:name]) if options[:name] + fks = fks.by_constrained_columns(options[:column]) if options[:column] + fks = fks.by_referenced_columns(options[:primary_key]) if options[:primary_key] + fks = fks.by_on_delete_action(options[:on_delete]) if options[:on_delete] + + fks.exists? end # Returns the name for a concurrent foreign key. @@ -1278,6 +1304,13 @@ into similar problems in the future (e.g. when new tables are created). "ON DELETE #{on_delete.upcase}" end + def on_update_statement(on_update) + return '' if on_update.blank? + return 'ON UPDATE SET NULL' if on_update == :nullify + + "ON UPDATE #{on_update.upcase}" + end + def create_column_from(table, old, new, type: nil, batch_column_name: :id, type_cast_function: nil, limit: nil) old_col = column_for(table, old) new_type = type || old_col.type diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb index 0aa4b0d01c4..c59139344ea 100644 --- a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -42,7 +42,7 @@ module Gitlab def should_lock_writes_on_table?(table_name) # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables # that should be skipped as they will be removed in a future migration. - return false if Gitlab::Database::GitlabSchema::DELETED_TABLES[table_name] + return false if Gitlab::Database::GitlabSchema.deleted_tables_to_schema[table_name] table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false) @@ -60,12 +60,15 @@ module Gitlab Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) end + # with_retries creates new a transaction. So we set it to false if the connection is + # already has an open transaction, to avoid sub-transactions. def lock_writes_on_table(connection, table_name) database_name = Gitlab::Database.db_config_name(connection) LockWritesManager.new( table_name: table_name, connection: connection, database_name: database_name, + with_retries: !connection.transaction_open?, logger: Logger.new($stdout) ).lock_writes end diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index dbb85bad95c..8975c04e33a 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -44,13 +44,20 @@ module Gitlab jobs.each do |j| break if run_until <= Time.current + meta = migration_meta(j) + instrumentation.observe(version: nil, name: batch_names.next, - connection: connection) do + connection: connection, + meta: meta) do run_job(j) end end end + + def migration_meta(_job) + {} + end end end end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 7c21346007a..8c479d7eda2 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -11,8 +11,8 @@ module Gitlab @result_dir = result_dir end - def observe(version:, name:, connection:, &block) - observation = Observation.new(version: version, name: name, success: false) + def observe(version:, name:, connection:, meta: {}, &block) + observation = Observation.new(version: version, name: name, success: false, meta: meta) per_migration_result_dir = File.join(@result_dir, name) diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 228eea3393c..80388c4dbbb 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -10,6 +10,7 @@ module Gitlab :walltime, :success, :total_database_size_change, + :meta, :query_statistics, keyword_init: true ) diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index a16103f452c..c123d01f327 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -13,7 +13,7 @@ module Gitlab end def jobs_by_migration_name - Gitlab::Database::SharedModel.using_connection(connection) do + set_shared_model_connection do Gitlab::Database::BackgroundMigration::BatchedMigration .executable .where('id > ?', from_id) @@ -70,7 +70,7 @@ module Gitlab end def run_job(job) - Gitlab::Database::SharedModel.using_connection(connection) do + set_shared_model_connection do Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) end end @@ -107,6 +107,16 @@ module Gitlab private attr_reader :from_id + + def set_shared_model_connection(&block) + Gitlab::Database::SharedModel.using_connection(connection, &block) + end + + def migration_meta(job) + set_shared_model_connection do + job.batched_migration.slice(:max_batch_size, :total_tuple_count, :interval) + end + end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index bd8ed677d77..8849191f356 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module PartitioningMigrationHelpers module ForeignKeyHelpers include ::Gitlab::Database::SchemaHelpers + include ::Gitlab::Database::Migrations::LockRetriesHelpers ERROR_SCOPE = 'foreign keys' diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index 241b6f009f7..d3ede45fe86 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -5,17 +5,44 @@ module Gitlab class PostgresForeignKey < SharedModel self.primary_key = :oid + # These values come from the possible confdeltype values in pg_constraint + enum on_delete_action: { + restrict: 'r', + cascade: 'c', + nullify: 'n', + set_default: 'd', + no_action: 'a' + } + scope :by_referenced_table_identifier, ->(identifier) do raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ where(referenced_table_identifier: identifier) end + scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) } + scope :by_constrained_table_identifier, ->(identifier) do raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ where(constrained_table_identifier: identifier) end + + scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) } + + scope :not_inherited, -> { where(is_inherited: false) } + + scope :by_name, ->(name) { where(name: name) } + + scope :by_constrained_columns, ->(cols) { where(constrained_columns: Array.wrap(cols)) } + + scope :by_referenced_columns, ->(cols) { where(referenced_columns: Array.wrap(cols)) } + + scope :by_on_delete_action, ->(on_delete) do + raise ArgumentError, "Invalid on_delete action #{on_delete}" unless on_delete_actions.key?(on_delete) + + where(on_delete_action: on_delete) + end end end end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index eda11fd8382..e4f70ee1745 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -17,7 +17,9 @@ module Gitlab for_identifier(identifier).first! end - scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + scope :for_parent_table, ->(name) do + where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) + end def self.partition_exists?(table_name) where("identifier = concat(current_schema(), '.', ?)", table_name).exists? diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 1280789b30c..6f64d04270f 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -86,11 +86,7 @@ module Gitlab analyzers.each do |analyzer| next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) - if analyzer.raw? - analyzer.analyze(sql) - else - analyzer.analyze(parsed) - end + analyzer.analyze(parsed) rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 9c2c228f869..9a52a4f6e23 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -53,10 +53,6 @@ module Gitlab Thread.current[self.context_key] end - def self.raw? - false - end - def self.enabled? raise NotImplementedError end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb index 3de9e8011fb..c966ae0e105 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb @@ -22,13 +22,16 @@ module Gitlab return unless allowed_schemas invalid_schemas = table_schemas - allowed_schemas - if invalid_schemas.any? - message = "The query tried to access #{tables} (of #{table_schemas.to_a}) " - message += "which is outside of allowed schemas (#{allowed_schemas}) " - message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'" - raise CrossSchemaAccessError, message - end + return if invalid_schemas.empty? + + schema_list = table_schemas.sort.join(',') + + message = "The query tried to access #{tables} (of #{schema_list}) " + message += "which is outside of allowed schemas (#{allowed_schemas}) " + message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'" + + raise CrossSchemaAccessError, message end end end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index dd10e0d7992..713e1f772e3 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -87,15 +87,15 @@ module Gitlab return if tables == ['schema_migrations'] context[:modified_tables_by_db][database].merge(tables) - all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten + all_tables = context[:modified_tables_by_db].values.flat_map(&:to_a) schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) schemas += ApplicationRecord.gitlab_transactions_stack if schemas.many? message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ - "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ + "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." if schemas.any? { |s| s.to_s.start_with?("undefined") } message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ." diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb index b54f3442512..63b4fbb8c1d 100644 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -5,21 +5,19 @@ module Gitlab module QueryAnalyzers class QueryRecorder < Base LOG_PATH = 'query_recorder/' + LIST_PARAMETER_REGEX = %r{\$\d+(?:\s*,\s*\$\d+)+}.freeze + SINGLE_PARAMETER_REGEX = %r{\$\d+}.freeze class << self - def raw? - true - end - def enabled? # Only enable QueryRecorder in CI on database MRs or default branch ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) end - def analyze(sql) + def analyze(parsed) payload = { - sql: sql + normalized: normalize_query(parsed.sql) } log_query(payload) @@ -42,6 +40,12 @@ module Gitlab File.write(log_file, log_line, mode: 'a') end + + def normalize_query(query) + query + .gsub(LIST_PARAMETER_REGEX, '?,?,?') # Replace list parameters with ?,?,? + .gsub(SINGLE_PARAMETER_REGEX, '?') # Replace single parameters with ? + end end end end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index b4f7da999df..eca118a4ff2 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -4,7 +4,7 @@ module Gitlab module Database module Reindexing class Coordinator - include ExclusiveLeaseGuard + include IndexingExclusiveLeaseGuard # Maximum lease time for the global Redis lease # This should be higher than the maximum time for any @@ -20,6 +20,8 @@ module Gitlab end def perform + return if too_late_for_reindexing? + # This obtains a global lease such that there's # only one live reindexing process at a time. try_obtain_lease do @@ -32,26 +34,28 @@ module Gitlab end def drop + return if too_late_for_reindexing? + try_obtain_lease do Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - connection: index.connection, + connection: connection, timing_configuration: REMOVE_INDEX_RETRY_CONFIG, klass: self.class, logger: Gitlab::AppLogger ) retries.run(raise_on_exhaustion: false) do - index.connection.tap do |conn| - conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") - end + connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{full_index_name}") end end end private + delegate :connection, to: :index + def with_notifications(action) notifier.notify_start(action) yield @@ -73,8 +77,18 @@ module Gitlab TIMEOUT_PER_ACTION end - def lease_key - [super, index.connection_db_config.name].join('/') + def full_index_name + [ + connection.quote_table_name(index.schema), + connection.quote_table_name(index.name) + ].join('.') + end + + # We need to check the time explicitly because we execute 4 reindexing + # action per rake invocation and one action can take up to 24 hours. + # This means that it can span for more than the weekend. + def too_late_for_reindexing? + !Time.current.on_weekend? end end end diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb index ece9327b658..e43eddbefc0 100644 --- a/lib/gitlab/database/reindexing/grafana_notifier.rb +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -60,7 +60,9 @@ module Gitlab "Authorization": "Bearer #{@api_key}" } - success = Gitlab::HTTP.post("#{@api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true).success? + success = Gitlab::HTTP.post( + "#{@api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true + ).success? log_error("Response code #{response.code}") unless success diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb index 2d384f2f9e2..ebe245bfadb 100644 --- a/lib/gitlab/database/reindexing/index_selection.rb +++ b/lib/gitlab/database/reindexing/index_selection.rb @@ -12,6 +12,10 @@ module Gitlab # Only consider indexes beyond this size (before reindexing) INDEX_SIZE_MINIMUM = 1.gigabyte + VERY_LARGE_TABLES = %i[ + ci_builds + ].freeze + delegate :each, to: :indexes def initialize(candidates) @@ -30,13 +34,24 @@ module Gitlab # we force a N+1 pattern here and estimate bloat on a per-index # basis. - @indexes ||= candidates - .not_recently_reindexed - .where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM) + @indexes ||= relations_that_need_cleaning_before_deadline .sort_by(&:relative_bloat_level) # forced N+1 .reverse .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT } end + + def relations_that_need_cleaning_before_deadline + relation = candidates.not_recently_reindexed.where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM) + relation = relation.where.not(tablename: VERY_LARGE_TABLES) if too_late_for_very_large_table? + relation + end + + # The reindexing process takes place during the weekends and starting a + # reindexing action on a large table late on Sunday could span during + # Monday. We don't want this because it prevents vacuum from running. + def too_late_for_very_large_table? + !Date.today.saturday? + end end end end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index 2c7ca28942e..d81ff4ff1ae 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -71,19 +71,6 @@ module Gitlab "#{type}_#{hashed_identifier}" end - def with_lock_retries(*args, **kwargs, &block) - raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) - merged_args = { - connection: connection, - klass: self.class, - logger: Gitlab::BackgroundMigration::Logger, - allow_savepoints: true - }.merge(kwargs) - - Gitlab::Database::WithLockRetries.new(**merged_args) - .run(raise_on_exhaustion: raise_on_exhaustion, &block) - end - def assert_not_in_transaction_block(scope:) return unless transaction_open? diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 807ecdb862a..daef0402742 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -40,11 +40,12 @@ module Gitlab table_name: table_name, connection: connection, database_name: database_name, + with_retries: true, logger: logger, dry_run: dry_run ) - unless lock_writes_manager.table_locked_for_writes?(table_name) + unless lock_writes_manager.table_locked_for_writes? raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" end end @@ -81,6 +82,22 @@ module Gitlab sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" logger&.info(sql_statement) connection.execute(sql_statement) unless dry_run + + # Temporarily unlocking writes on the attached partitions of the table. + # Because in some cases they might have been locked for writes as well, when they used to be + # normal tables before being converted into attached partitions. + Gitlab::Database::SharedModel.using_connection(connection) do + table_partitions = Gitlab::Database::PostgresPartition.for_parent_table(table_name_without_schema) + table_partitions.each do |table_partition| + partition_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_partition.identifier) + .identifier + + sql_statement = "SELECT set_config('lock_writes.#{partition_name_without_schema}', 'false', false)" + logger&.info(sql_statement) + connection.execute(sql_statement) unless dry_run + end + end end # We do the truncation in stages to avoid high IO diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index c6ab56e783a..801c1967e0a 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -23,7 +23,9 @@ module Gitlab strong_memoize(:diff_files) do diff_files = super - diff_files.each { |diff_file| highlight_cache.decorate(diff_file) } + Gitlab::Metrics.measure(:diffs_highlight_cache_decorate) do + diff_files.each { |diff_file| highlight_cache.decorate(diff_file) } + end diff_files end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 582c3380869..876a1cbb183 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -116,6 +116,20 @@ module Gitlab process_exception(exception, extra: extra, trackers: [Logger]) end + # This should be used when you want to log the exception and passthrough + # exception handling: rescue and raise to be catched in upper layers of + # the application. + # + # If the exception implements the method `sentry_extra_data` and that method + # returns a Hash, then the return value of that method will be merged into + # `extra`. Exceptions can use this mechanism to provide structured data + # to sentry in addition to their message and back-trace. + def log_and_raise_exception(exception, extra = {}) + process_exception(exception, extra: extra, trackers: [Logger]) + + raise exception + end + private def before_send_raven(event, hint) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 344dd27589c..35b330fa089 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -49,7 +49,7 @@ module Gitlab def self.error_message(key) self.ancestors.each do |cls| - return cls.const_get('ERROR_MESSAGES', false).fetch(key) + return cls.const_get(:ERROR_MESSAGES, false).fetch(key) rescue NameError, KeyError next end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 735c7fcf80c..199257f767d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -37,9 +37,8 @@ module Gitlab @stubs[storage] ||= {} @stubs[storage][name] ||= begin klass = stub_class(name) - addr = stub_address(storage) - creds = stub_creds(storage) - klass.new(addr, creds, interceptors: interceptors, channel_args: channel_args) + channel = create_channel(storage) + klass.new(channel.target, nil, interceptors: interceptors, channel_override: channel) end end end @@ -52,11 +51,29 @@ module Gitlab private_class_method :interceptors def self.channel_args - # These values match the go Gitaly client - # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78 { + # These keepalive values match the go Gitaly client + # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78 'grpc.keepalive_time_ms': 20000, - 'grpc.keepalive_permit_without_calls': 1 + 'grpc.keepalive_permit_without_calls': 1, + # Enable client-side automatic retry. After enabled, gRPC requests will be retried when there are connectivity + # problems with the target host. Only transparent failures, which mean requests fail before leaving clients, are + # eligible. Other cases are configurable via retry policy in service config (below). In theory, we can auto-retry + # read-only RPCs. Gitaly defines a custom field in service proto. Unfortunately, gRPC ruby doesn't support + # descriptor reflection. + # For more information please visit https://github.com/grpc/proposal/blob/master/A6-client-retries.md + 'grpc.enable_retries': 1, + # Service config is a mechanism for grpc to control the behavior of gRPC client. It defines the client-side + # balancing strategy and retry policy. The config receives a raw JSON string. The format is defined here: + # https://github.com/grpc/grpc-proto/blob/master/grpc/service_config/service_config.proto + 'grpc.service_config': { + # By default, gRPC uses pick_first strategy. This strategy establishes one single connection to the first + # target returned by the name resolver. We would like to use round_robin load-balancing strategy so that + # grpc creates multiple subchannels to all targets retrurned by the resolver. Requests are distributed to + # those subchannels in a round-robin fashion. + # More about client-side load-balancing: https://gitlab.com/groups/gitlab-org/-/epics/8971#note_1207008162 + "loadBalancingConfig": [{ "round_robin": {} }] + }.to_json } end private_class_method :channel_args @@ -81,9 +98,20 @@ module Gitlab address(storage).sub(%r{^tcp://|^tls://}, '') end + # Cache gRPC servers by storage. All the client stubs in the same process can share the underlying connection to the + # same host thanks to HTTP2 framing protocol that gRPC is built on top. This method is not thread-safe. It is + # intended to be a part of `stub`, method behind a mutex protection. + def self.create_channel(storage) + @channels ||= {} + @channels[storage] ||= GRPC::ClientStub.setup_channel( + nil, stub_address(storage), stub_creds(storage), channel_args + ) + end + def self.clear_stubs! MUTEX.synchronize do @stubs = nil + @channels = nil end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 98b1d3dceef..74034c4e717 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -215,12 +215,6 @@ module Gitlab consume_list_refs_response(response) end - def pack_refs - request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) - - gitaly_client_call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) - end - def find_refs_by_oid(oid:, limit:, ref_patterns: nil) request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit, ref_patterns: ref_patterns) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index daaf18c711d..203854264ce 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -34,21 +34,6 @@ module Gitlab gitaly_client_call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) end - def garbage_collect(create_bitmap, prune:) - request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) - gitaly_client_call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) - end - - def repack_full(create_bitmap) - request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - gitaly_client_call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) - end - - def repack_incremental - request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) - gitaly_client_call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) - end - def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) response = gitaly_client_call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb index a5e87d3cf7d..4018f425e7c 100644 --- a/lib/gitlab/github_gists_import/importer/gist_importer.rb +++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb @@ -7,6 +7,7 @@ module Gitlab attr_reader :gist, :user FileCountLimitError = Class.new(StandardError) + FILE_COUNT_LIMIT_MESSAGE = 'Snippet maximum file count exceeded' # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`. def initialize(gist, user_id) @@ -76,7 +77,7 @@ module Gitlab def fail_and_track(snippet) remove_snippet_and_repository(snippet) - ServiceResponse.error(message: 'Snippet max file count exceeded').track_exception(as: FileCountLimitError) + ServiceResponse.error(message: FILE_COUNT_LIMIT_MESSAGE).track_exception(as: FileCountLimitError) end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 065410693e5..1c9ca9f43a8 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -264,18 +264,6 @@ module Gitlab private - def collaborations_subquery - each_object(:repos, nil, { affiliation: 'collaborator' }) - .map { |repo| "repo:#{repo[:full_name]}" } - .join(' ') - end - - def organizations_subquery - each_object(:organizations) - .map { |org| "org:#{org[:login]}" } - .join(' ') - end - def with_retry Retriable.retriable(on: CLIENT_CONNECTION_ERROR, on_retry: on_retry) do yield diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb index f6d1c8ed23c..b12df404640 100644 --- a/lib/gitlab/github_import/clients/proxy.rb +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -10,24 +10,24 @@ module Gitlab @client = pick_client(access_token, client_options) end - def repos(search_text, pagination_options) + def repos(search_text, options) return { repos: filtered(client.repos, search_text) } if use_legacy? if use_graphql? - fetch_repos_via_graphql(search_text, pagination_options) + fetch_repos_via_graphql(search_text, options) else - fetch_repos_via_rest(search_text, pagination_options) + fetch_repos_via_rest(search_text, options) end end private - def fetch_repos_via_rest(search_text, pagination_options) - { repos: client.search_repos_by_name(search_text, pagination_options)[:items] } + def fetch_repos_via_rest(search_text, options) + { repos: client.search_repos_by_name(search_text, options)[:items] } end - def fetch_repos_via_graphql(search_text, pagination_options) - response = client.search_repos_by_name_graphql(search_text, pagination_options) + def fetch_repos_via_graphql(search_text, options) + response = client.search_repos_by_name_graphql(search_text, options) { repos: response.dig(:data, :search, :nodes), page_info: response.dig(:data, :search, :pageInfo) diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb index bcd226087e7..b72e5ac7751 100644 --- a/lib/gitlab/github_import/clients/search_repos.rb +++ b/lib/gitlab/github_import/clients/search_repos.rb @@ -14,18 +14,17 @@ module Gitlab end def search_repos_by_name(name, options = {}) + search_query = search_repos_query(name, options) + with_retry do - octokit.search_repositories( - search_repos_query(str: name, type: :name), - options - ).to_h + octokit.search_repositories(search_query, options).to_h end end private def graphql_search_repos_body(name, options) - query = search_repos_query(str: name, type: :name) + query = search_repos_query(name, options) query = "query: \"#{query}\"" first = options[:first].present? ? ", first: #{options[:first]}" : '' after = options[:after].present? ? ", after: \"#{options[:after]}\"" : '' @@ -52,13 +51,49 @@ module Gitlab TEXT end - def search_repos_query(str:, type:, include_collaborations: true, include_orgs: true) - query = "#{str} in:#{type} is:public,private user:#{octokit.user.to_h[:login]}" + def search_repos_query(string, options = {}) + base = "#{string} in:name is:public,private" + + case options[:relation_type] + when 'organization' then organization_repos_query(base, options) + when 'collaborated' then collaborated_repos_query(base) + when 'owned' then owned_repos_query(base) + # TODO: remove after https://gitlab.com/gitlab-org/gitlab/-/issues/385113 get done + else legacy_all_repos_query(base) + end + end + + def organization_repos_query(search_string, options) + "#{search_string} org:#{options[:organization_login]}" + end + + def collaborated_repos_query(search_string) + "#{search_string} #{collaborations_subquery}" + end + + def owned_repos_query(search_string) + "#{search_string} user:#{octokit.user.to_h[:login]}" + end - query = [query, collaborations_subquery].join(' ') if include_collaborations - query = [query, organizations_subquery].join(' ') if include_orgs + def legacy_all_repos_query(search_string) + [ + search_string, + "user:#{octokit.user.to_h[:login]}", + collaborations_subquery, + organizations_subquery + ].join(' ') + end + + def collaborations_subquery + each_object(:repos, nil, { affiliation: 'collaborator' }) + .map { |repo| "repo:#{repo[:full_name]}" } + .join(' ') + end - query + def organizations_subquery + each_object(:organizations) + .map { |org| "org:#{org[:login]}" } + .join(' ') end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb index 801a0840c52..2077e0c6b11 100644 --- a/lib/gitlab/github_import/importer/protected_branch_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -4,7 +4,7 @@ module Gitlab module GithubImport module Importer class ProtectedBranchImporter - attr_reader :protected_branch, :project, :client + attr_reader :project # By default on GitHub, both developers and maintainers can merge # a PR into the protected branch @@ -18,6 +18,7 @@ module Gitlab @protected_branch = protected_branch @project = project @client = client + @user_finder = GithubImport::UserFinder.new(project, client) end def execute @@ -32,11 +33,13 @@ module Gitlab private + attr_reader :protected_branch, :user_finder + def params { name: protected_branch.id, - push_access_levels_attributes: [{ access_level: push_access_level }], - merge_access_levels_attributes: [{ access_level: merge_access_level }], + push_access_levels_attributes: push_access_levels_attributes, + merge_access_levels_attributes: merge_access_levels_attributes, allow_force_push: allow_force_push?, code_owner_approval_required: code_owner_approval_required? } @@ -55,7 +58,7 @@ module Gitlab end def code_owner_approval_required? - return false unless project.licensed_feature_available?(:code_owner_approval_required) + return false unless licensed_feature_available?(:code_owner_approval_required) return protected_branch.require_code_owner_reviews unless protected_on_gitlab? @@ -83,7 +86,7 @@ module Gitlab end def update_project_push_rule - return unless project.licensed_feature_available?(:push_rules) + return unless licensed_feature_available?(:push_rules) return unless protected_branch.required_signatures push_rule = project.push_rule || project.build_push_rule @@ -91,12 +94,34 @@ module Gitlab project.project_setting.update!(push_rule_id: push_rule.id) end - def push_access_level - if protected_branch.required_pull_request_reviews - Gitlab::Access::NO_ACCESS + def push_access_levels_attributes + if allowed_to_push_gitlab_user_ids.present? + @allowed_to_push_gitlab_user_ids.map { |user_id| { user_id: user_id } } + elsif protected_branch.required_pull_request_reviews + [{ access_level: Gitlab::Access::NO_ACCESS }] else - gitlab_access_level_for(:push) + [{ access_level: gitlab_access_level_for(:push) }] + end + end + + def merge_access_levels_attributes + [{ access_level: merge_access_level }] + end + + def allowed_to_push_gitlab_user_ids + return if protected_branch.allowed_to_push_users.empty? || + !licensed_feature_available?(:protected_refs_for_users) + + @allowed_to_push_gitlab_user_ids = [] + + protected_branch.allowed_to_push_users.each do |github_user_data| + gitlab_user_id = user_finder.user_id_for(github_user_data) + next unless gitlab_user_id + + @allowed_to_push_gitlab_user_ids << gitlab_user_id end + + @allowed_to_push_gitlab_user_ids &= project_member_ids end # Gets the strictest merge_access_level between GitHub and GitLab @@ -155,6 +180,14 @@ module Gitlab ProtectedBranch::MergeAccessLevel::GITLAB_DEFAULT_ACCESS_LEVEL end + + def licensed_feature_available?(feature) + project.licensed_feature_available?(feature) + end + + def project_member_ids + project.authorized_users.map(&:id) + end end end end diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index d2a52b64bbf..eb9dd3bc247 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, - :required_pull_request_reviews, :require_code_owner_reviews + :required_pull_request_reviews, :require_code_owner_reviews, :allowed_to_push_users # Builds a Branch Protection info from a GitHub API response. # Resource structure details: @@ -19,6 +19,12 @@ module Gitlab def self.from_api_response(branch_protection, _additional_object_data = {}) branch_name = branch_protection[:url].match(%r{/branches/(\S{1,255})/protection$})[1] + allowed_to_push_users = branch_protection.dig(:required_pull_request_reviews, + :bypass_pull_request_allowances, + :users) + allowed_to_push_users &&= allowed_to_push_users.map do |u| + Representation::User.from_api_response(u) + end hash = { id: branch_name, allow_force_pushes: branch_protection.dig(:allow_force_pushes, :enabled), @@ -26,7 +32,8 @@ module Gitlab required_signatures: branch_protection.dig(:required_signatures, :enabled), required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present?, require_code_owner_reviews: branch_protection.dig(:required_pull_request_reviews, - :require_code_owner_reviews).present? + :require_code_owner_reviews).present?, + allowed_to_push_users: allowed_to_push_users.to_a } new(hash) @@ -34,7 +41,13 @@ module Gitlab # Builds a new Protection using a Hash that was built from a JSON payload. def self.from_json_hash(raw_hash) - new(Representation.symbolize_hash(raw_hash)) + hash = Representation.symbolize_hash(raw_hash) + + hash[:allowed_to_push_users].map! do |u| + Representation::User.from_json_hash(u) + end + + new(hash) end # attributes - A Hash containing the raw Protection details. The keys of this diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 12cdcf445f7..ceef072a710 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -57,6 +57,7 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url gon.time_display_relative = current_user.time_display_relative + gon.use_new_navigation = Feature.enabled?(:super_sidebar_nav, current_user) && current_user&.use_new_navigation end # Initialize gon.features with any flags that should be @@ -67,7 +68,6 @@ module Gitlab push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:integration_slack_app_notifications) - push_frontend_feature_flag(:vue_group_select) push_frontend_feature_flag(:new_fonts, current_user) end diff --git a/lib/gitlab/graphql/deprecations_base.rb b/lib/gitlab/graphql/deprecations_base.rb index 2ee14620907..8a5f07b6ee9 100644 --- a/lib/gitlab/graphql/deprecations_base.rb +++ b/lib/gitlab/graphql/deprecations_base.rb @@ -9,11 +9,11 @@ module Gitlab def self.included(klass) klass.extend(ClassMethods) - klass.const_set('OLD_GRAPHQL_NAME_MAP', klass::DEPRECATIONS.index_by do |d| + klass.const_set(:OLD_GRAPHQL_NAME_MAP, klass::DEPRECATIONS.index_by do |d| klass.map_graphql_name(d.old_name) end.freeze) - klass.const_set('OLD_NAME_MAP', klass::DEPRECATIONS.index_by(&:old_name).freeze) - klass.const_set('NEW_NAME_MAP', klass::DEPRECATIONS.index_by(&:new_name).freeze) + klass.const_set(:OLD_NAME_MAP, klass::DEPRECATIONS.index_by(&:old_name).freeze) + klass.const_set(:NEW_NAME_MAP, klass::DEPRECATIONS.index_by(&:new_name).freeze) end module ClassMethods diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb index 657364abfdf..319c05d6e23 100644 --- a/lib/gitlab/graphql/errors.rb +++ b/lib/gitlab/graphql/errors.rb @@ -8,6 +8,8 @@ module Gitlab ResourceNotAvailable = Class.new(BaseError) MutationError = Class.new(BaseError) LimitError = Class.new(BaseError) + InvalidMembersError = Class.new(StandardError) + InvalidMemberCountError = Class.new(StandardError) end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 96128f432c5..a6ca8323a20 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -64,7 +64,7 @@ module Gitlab assignee_id: merge_request.assignee_ids.first, # This key is deprecated reviewer_ids: merge_request.reviewer_ids, labels: merge_request.labels_hook_attrs, - state: merge_request.state, # This key is deprecated + state: merge_request.state, blocking_discussions_resolved: merge_request.mergeable_discussions_state?, first_contribution: merge_request.first_contribution?, detailed_merge_status: detailed_merge_status diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index dd58f6aca26..b5000777010 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -12,8 +12,6 @@ module Gitlab def intercept_hotlinking?(request) request_accepts = parse_request_accepts(request) - return false unless Feature.enabled?(:repository_archive_hotlinking_interception) - # Block attempts to embed as JS return true if sec_fetch_invalid?(request) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index b05767c7ed4..c6cd5fbfced 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -17,7 +17,8 @@ module Gitlab HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, - Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep + Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep, + Net::HTTPBadResponse ].freeze DEFAULT_TIMEOUT_OPTIONS = { diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 7a42ffca779..31952f75006 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,28 +44,28 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 36, - 'de' => 17, + 'da_DK' => 35, + 'de' => 16, 'en' => 100, 'eo' => 0, - 'es' => 35, + 'es' => 34, 'fil_PH' => 0, - 'fr' => 94, + 'fr' => 98, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 30, + 'ja' => 29, 'ko' => 20, 'nb_NO' => 24, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 57, - 'ro_RO' => 96, + 'ro_RO' => 94, 'ru' => 26, 'si_LK' => 11, - 'tr_TR' => 11, - 'uk' => 52, - 'zh_CN' => 97, + 'tr_TR' => 10, + 'uk' => 54, + 'zh_CN' => 98, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index ed3858d0bf4..77b85fc9f15 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -71,6 +71,8 @@ module Gitlab invalid_subrelations << invalid_record unless invalid_record.persisted? end + + relation_object.save end end end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index af0026b8864..fa179f584eb 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -17,6 +17,10 @@ module Gitlab def self.file_compression_error self.new('File compression/decompression failed') end + + def self.incompatible_import_file_error + self.new('The import file is incompatible') + end end end end diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb deleted file mode 100644 index fa9e765b33a..00000000000 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Group - class LegacyTreeRestorer - include Gitlab::Utils::StrongMemoize - - attr_reader :user - attr_reader :shared - attr_reader :group - - def initialize(user:, shared:, group:, group_hash:) - @user = user - @shared = shared - @group = group - @group_hash = group_hash - end - - def restore - @group_attributes = relation_reader.consume_attributes(nil) - @group_members = relation_reader.consume_relation(nil, 'members') - .map(&:first) - - # We need to remove `name` and `path` as we did consume it in previous pass - @group_attributes.delete('name') - @group_attributes.delete('path') - - @children = @group_attributes.delete('children') - - if members_mapper.map && restorer.restore - @children&.each do |group_hash| - group = create_group(group_hash: group_hash, parent_group: @group) - shared = Gitlab::ImportExport::Shared.new(group) - - self.class.new( - user: @user, - shared: shared, - group: group, - group_hash: group_hash - ).restore - end - end - - return false if @shared.errors.any? - - true - rescue StandardError => e - @shared.error(e) - false - end - - private - - def relation_reader - strong_memoize(:relation_reader) do - if @group_hash.present? - ImportExport::Json::LegacyReader::Hash.new( - @group_hash, - relation_names: reader.group_relation_names) - else - ImportExport::Json::LegacyReader::File.new( - File.join(shared.export_path, 'group.json'), - relation_names: reader.group_relation_names) - end - end - end - - def restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - relation_reader: relation_reader, - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader, - importable: @group, - importable_attributes: @group_attributes, - importable_path: nil - ) - end - - def create_group(group_hash:, parent_group:) - group_params = { - name: group_hash['name'], - path: group_hash['path'], - parent_id: parent_group&.id, - visibility_level: sub_group_visibility_level(group_hash, parent_group) - } - - ::Groups::CreateService.new(@user, group_params).execute - end - - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE - - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) - else - original_visibility_level - end - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new( - exported_members: @group_members, - user: @user, - importable: @group - ) - end - - def relation_factory - Gitlab::ImportExport::Group::RelationFactory - end - - def object_builder - Gitlab::ImportExport::Group::ObjectBuilder - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.legacy_group_config_file - ).to_h - ) - end - end - end - end -end diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb deleted file mode 100644 index 0f74fabeac3..00000000000 --- a/lib/gitlab/import_export/group/legacy_tree_saver.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Group - class LegacyTreeSaver - attr_reader :full_path, :shared - - def initialize(group:, current_user:, shared:, params: {}) - @params = params - @current_user = current_user - @shared = shared - @group = group - @full_path = File.join(@shared.export_path, ImportExport.group_filename) - end - - def save - group_tree = serialize(@group, reader.group_tree) - tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) - - true - rescue StandardError => e - @shared.error(e) - false - end - - private - - def serialize(group, relations_tree) - group_tree = tree_saver.serialize(group, relations_tree) - - group.children.each do |child| - group_tree['children'] ||= [] - group_tree['children'] << serialize(child, relations_tree) - end - - group_tree - rescue StandardError => e - @shared.error(e) - end - - def reader - @reader ||= Gitlab::ImportExport::Reader.new( - shared: @shared, - config: Gitlab::ImportExport::Config.new( - config: Gitlab::ImportExport.legacy_group_config_file - ).to_h - ) - end - - def tree_saver - @tree_saver ||= LegacyRelationTreeSaver.new - end - end - end - end -end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index cc69ed55744..99364996864 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -137,6 +137,7 @@ included_attributes: ci_cd_settings: - :group_runners_enabled - :runner_token_expiration_interval + - :default_git_depth metrics_setting: - :dashboard_timezone - :external_dashboard_url @@ -719,7 +720,6 @@ included_attributes: - :feature_flags_access_level - :releases_access_level - :infrastructure_access_level - - :allow_merge_on_skipped_pipeline - :auto_devops_deploy_strategy - :auto_devops_enabled - :container_registry_enabled @@ -728,13 +728,14 @@ included_attributes: - :merge_method - :merge_requests_enabled - :snippets_enabled - - :squash_option - :topics - :visibility - :wiki_enabled - :build_git_strategy - :build_enabled - :security_and_compliance_enabled + - :allow_merge_on_skipped_pipeline + - :squash_option resource_milestone_events: - :user_id - :action @@ -776,6 +777,7 @@ excluded_attributes: - :wiki_page_hooks_integrations - :deployment_hooks_integrations - :alert_hooks_integrations + - :incident_hooks_integrations - :mirror - :runners_token - :runners_token_encrypted @@ -1071,6 +1073,9 @@ excluded_attributes: - :sequence methods: + project: + - :allow_merge_on_skipped_pipeline + - :squash_option notes: - :type labels: @@ -1179,6 +1184,7 @@ ee: - :reject_unsigned_commits - :commit_committer_check - :regexp_uses_re2 + - :reject_non_dco_commits unprotect_access_levels: - :access_level - :user_id diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 5ec9db00d0a..ad071a4cbd7 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -34,7 +34,7 @@ module Gitlab end def different_version?(version) - Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + Gitlab::VersionInfo.parse(version) != Gitlab::VersionInfo.parse(Gitlab::ImportExport.version) rescue StandardError => e Gitlab::Import::Logger.error( message: 'Import error', diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb index 710c89c6216..5effafc9f5b 100644 --- a/lib/gitlab/memory/reporter.rb +++ b/lib/gitlab/memory/reporter.rb @@ -3,6 +3,8 @@ module Gitlab module Memory class Reporter + COMPRESS_CMD = %w[gzip --fast].freeze + attr_reader :reports_path def initialize(reports_path: nil, logger: Gitlab::AppLogger) @@ -67,29 +69,39 @@ module Gitlab report_file = file_name(report) tmp_file_path = File.join(tmp_dir, report_file) + write_heap_dump_file(report, tmp_file_path) + + File.join(@reports_path, report_file).tap do |report_file_path| + FileUtils.mv(tmp_file_path, report_file_path) + end + end + + def write_heap_dump_file(report, path) io_r, io_w = IO.pipe + err_r, err_w = IO.pipe pid = nil - File.open(tmp_file_path, 'wb') do |file| + status = nil + File.open(path, 'wb') do |file| extras = { in: io_r, out: file, - err: $stderr + err: err_w } - pid = Process.spawn('gzip', '--fast', **extras) + pid = Process.spawn(*COMPRESS_CMD, **extras) io_r.close + err_w.close report.run(io_w) io_w.close - Process.waitpid(pid) + _, status = Process.wait2(pid) end - File.join(@reports_path, report_file).tap do |report_file_path| - FileUtils.mv(tmp_file_path, report_file_path) - end + errors = err_r.read&.strip + err_r.close + raise StandardError, "exit #{status.exitstatus}: #{errors}" if !status&.success? && errors.present? ensure - [io_r, io_w].each(&:close) - + [io_r, io_w, err_r, err_w].each(&:close) # Make sure we don't leave any running processes behind. Gitlab::ProcessManagement.signal(pid, :KILL) if pid end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index aac70a2f6aa..c94dbed1d46 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -68,12 +68,11 @@ module Gitlab monitor end - event_reporter.stopped(log_labels(memwd_reason: @reason).compact) + event_reporter.stopped(log_labels(memwd_reason: @stop_reason).compact) end - def stop(reason: nil) - @reason = reason - @alive = false + def stop + stop_working(reason: 'background task stopped') end private @@ -84,7 +83,7 @@ module Gitlab def monitor if monitors.empty? - stop(reason: 'monitors are not configured') + stop_working(reason: 'monitors are not configured') return end @@ -106,7 +105,7 @@ module Gitlab Gitlab::Memory::Reports::HeapDump.enqueue! - stop(reason: 'successfully handled') if handler.call + stop_working(reason: 'successfully handled') if handler.call end def handler @@ -123,6 +122,13 @@ module Gitlab memwd_sleep_time_s: sleep_time_seconds ) end + + def stop_working(reason:) + return unless @alive + + @stop_reason = reason + @alive = false + end end end end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 0172de8731d..cfdac5264e0 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -23,7 +23,7 @@ module Gitlab # with an explosion in unused metric combinations, but we want the # most common ones to be always present. FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization', - 'code_review', 'continuous_integration', + 'code_review_workflow', 'continuous_integration', 'not_owned', 'source_code_management', FEATURE_CATEGORY_DEFAULT].freeze diff --git a/lib/gitlab/net_http_adapter.rb b/lib/gitlab/net_http_adapter.rb index 2f7557f2bc3..17eb07fff2b 100644 --- a/lib/gitlab/net_http_adapter.rb +++ b/lib/gitlab/net_http_adapter.rb @@ -6,7 +6,7 @@ module Gitlab # Net::HTTP#request usually calls Net::HTTP#connect but the Webmock overwrite doesn't. # This makes sure that, in a test environment, the superclass is the Webmock overwrite. parent_class = if defined?(WebMock) && Rails.env.test? - WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get('@webMockNetHTTP') + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP) else Net::HTTP end diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index 8dde60a73be..8dbd2f41ccb 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -11,5 +11,9 @@ module Gitlab 'https://observe.gitlab.com' end + + def observability_enabled?(user, group) + Gitlab::Observability.observability_url.present? && Ability.allowed?(user, :read_observability, group) + end end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb index be39e52b342..a24d958b7e5 100644 --- a/lib/gitlab/pages/cache_control.rb +++ b/lib/gitlab/pages/cache_control.rb @@ -16,8 +16,8 @@ module Gitlab PAYLOAD_CACHE_KEY = '%{settings_cache_key}_%{settings_hash}' class << self - def for_project(project_id) - new(type: :project, id: project_id) + def for_domain(domain_id) + new(type: :domain, id: domain_id) end def for_namespace(namespace_id) @@ -26,7 +26,7 @@ module Gitlab end def initialize(type:, id:) - raise(ArgumentError, "type must be :namespace or :project") unless %i[namespace project].include?(type) + raise(ArgumentError, "type must be :namespace or :domain") unless %i[namespace domain].include?(type) @type = type @id = id @@ -50,7 +50,9 @@ module Gitlab .map { |hash| payload_cache_key_for(hash) } .push(settings_cache_key) - Rails.cache.delete_multi(keys) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Rails.cache.delete_multi(keys) + end end private diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 199ec16d4df..a21d0228082 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -5,7 +5,8 @@ module Gitlab module CursorBasedKeyset SUPPORTED_ORDERING = { Group => { name: :asc }, - AuditEvent => { id: :desc } + AuditEvent => { id: :desc }, + ::Ci::Build => { id: :desc } }.freeze # Relation types that are enforced in this list diff --git a/lib/gitlab/pagination/keyset/cursor_pager.rb b/lib/gitlab/pagination/keyset/cursor_pager.rb index 0b49aa87a02..d8fa94091ea 100644 --- a/lib/gitlab/pagination/keyset/cursor_pager.rb +++ b/lib/gitlab/pagination/keyset/cursor_pager.rb @@ -10,7 +10,7 @@ module Gitlab @cursor_based_request_context = cursor_based_request_context end - def paginate(relation) + def paginate(relation, _params = {}) @paginator ||= relation.keyset_paginate( per_page: cursor_based_request_context.per_page, cursor: cursor_based_request_context.cursor diff --git a/lib/gitlab/pagination/keyset/pager.rb b/lib/gitlab/pagination/keyset/pager.rb index 6a2ae20f3b8..3fabd454ee3 100644 --- a/lib/gitlab/pagination/keyset/pager.rb +++ b/lib/gitlab/pagination/keyset/pager.rb @@ -10,7 +10,7 @@ module Gitlab @request = request end - def paginate(relation) + def paginate(relation, _params = {}) # Validate assumption: The last two columns must match the page order_by validate_order!(relation) diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 318720c77d1..cbd523389d6 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -11,8 +11,6 @@ module Gitlab # [transformed_scope, true] # true indicates that the new scope was successfully built # [orginal_scope, false] # false indicates that the order values are not supported in this class class SimpleOrderBuilder - NULLS_ORDER_REGEX = /(?<column_name>.*) (?<direction>\bASC\b|\bDESC\b) (?<nullable>\bNULLS LAST\b|\bNULLS FIRST\b)/.freeze - def self.build(scope) new(scope: scope).build end @@ -90,32 +88,6 @@ module Gitlab end end - # This method converts the first order value to a corresponding arel expression - # if the order value uses either NULLS LAST or NULLS FIRST ordering in raw SQL. - # - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/356644 - # We should stop matching raw literals once we switch to using the Arel methods. - def convert_raw_nulls_order! - order_value = order_values.first - - return unless order_value.is_a?(Arel::Nodes::SqlLiteral) - - # Detect NULLS LAST or NULLS FIRST ordering by looking at the raw SQL string. - if matches = order_value.match(NULLS_ORDER_REGEX) - return unless table_column?(matches[:column_name]) - - column_attribute = arel_table[matches[:column_name]] - direction = matches[:direction].downcase.to_sym - nullable = matches[:nullable].downcase.parameterize(separator: '_').to_sym - - # Build an arel order expression for NULLS ordering. - order = direction == :desc ? column_attribute.desc : column_attribute.asc - arel_order_expression = nullable == :nulls_first ? order.nulls_first : order.nulls_last - - order_values[0] = arel_order_expression - end - end - def nullability(order_value, attribute_name) nullable = model_class.columns.find { |column| column.name == attribute_name }.null @@ -206,16 +178,12 @@ module Gitlab def ordered_by_other_column? return unless order_values.one? - convert_raw_nulls_order! - supported_column?(order_values.first) end def ordered_by_other_column_with_tie_breaker? return unless order_values.size == 2 - convert_raw_nulls_order! - return unless supported_column?(order_values.first) tie_breaker_attribute = order_values.second.try(:expr) diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb index 3885a9934d5..49e01eceb5b 100644 --- a/lib/gitlab/phabricator_import.rb +++ b/lib/gitlab/phabricator_import.rb @@ -5,8 +5,7 @@ module Gitlab BaseError = Class.new(StandardError) def self.available? - Feature.enabled?(:phabricator_import) && - Gitlab::CurrentSettings.import_sources.include?('phabricator') + Gitlab::CurrentSettings.import_sources.include?('phabricator') end end end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 9bc0001be81..5394cd115b1 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -81,7 +81,7 @@ module Gitlab ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), - ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/ochorocho/typo3-distribution', 'illustrations/logos/typo3.svg') + ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg') ].freeze end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 14e9e66e037..f782f2802b6 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -252,23 +252,14 @@ module Gitlab desc { _('Promote issue to incident') } explanation { _('Promotes issue to incident') } + execution_message { _('Issue has been promoted to incident') } types Issue condition do - quick_action_target.persisted? && - !quick_action_target.incident? && - current_user.can?(:update_issue, quick_action_target) + !quick_action_target.incident? && + current_user.can?(:"set_#{quick_action_target.issue_type}_metadata", quick_action_target) end command :promote_to_incident do - issue = ::Issues::UpdateService - .new(project: quick_action_target.project, current_user: current_user, params: { issue_type: 'incident' }) - .execute(quick_action_target) - - @execution_message[:promote_to_incident] = - if issue.incident? - _('Issue has been promoted to incident') - else - _('Failed to promote issue to incident') - end + @updates[:issue_type] = "incident" end desc { _('Add customer relation contacts') } diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 8857b544364..ed4f6015603 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -11,6 +11,7 @@ module Gitlab Gitlab::Redis::Cache, Gitlab::Redis::Queues, Gitlab::Redis::RateLimiting, + Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 4f58bee49d0..aa8f390ac10 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -26,7 +26,7 @@ module Gitlab class MethodMissingError < StandardError def message - 'Method missing. Falling back to execute method on the redis secondary store.' + 'Method missing. Falling back to execute method on the redis default store in Rails.env.production.' end end @@ -36,31 +36,64 @@ module Gitlab FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' - SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze + SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze - READ_COMMANDS = %i( - get - mget - smembers - scard - ).freeze - - WRITE_COMMANDS = %i( - set - setnx - setex - sadd - srem + # For ENUMERATOR_CACHE_HIT_VALIDATOR and READ_CACHE_HIT_VALIDATOR, + # we define procs to validate cache hit. The only other acceptable value is nil, + # in the case of errors being raised. + # + # If a command has no empty response, set ->(val) { true } + # + # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands + # + ENUMERATOR_CACHE_HIT_VALIDATOR = { + scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, + zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } + }.freeze + + READ_CACHE_HIT_VALIDATOR = { + exists: ->(val) { val != 0 }, + exists?: ->(val) { val }, + get: ->(val) { !val.nil? }, + hexists: ->(val) { val }, + hget: ->(val) { !val.nil? }, + hgetall: ->(val) { val.is_a?(Hash) && !val.empty? }, + hlen: ->(val) { val != 0 }, + hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? }, + mget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, + scard: ->(val) { val != 0 }, + sismember: ->(val) { val }, + smembers: ->(val) { val.is_a?(Array) && !val.empty? }, + sscan: ->(val) { val != ['0', []] }, + ttl: ->(val) { val != 0 && val != -2 } + }.freeze + + WRITE_COMMANDS = %i[ del + eval + expire flushdb + hdel + hset + incr + incrby + mapped_hmset rpush - eval - ).freeze + sadd + set + setex + setnx + srem + unlink + ].freeze - PIPELINED_COMMANDS = %i( + PIPELINED_COMMANDS = %i[ pipelined multi - ).freeze + ].freeze # To transition between two Redis store, `primary_store` should be the target store, # and `secondary_store` should be the current store. Transition is controlled with feature flags: @@ -81,12 +114,12 @@ module Gitlab end # rubocop:disable GitlabSecurity/PublicSend - READ_COMMANDS.each do |name| - define_method(name) do |*args, &block| + READ_CACHE_HIT_VALIDATOR.each_key do |name| + define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? - read_command(name, *args, &block) + read_command(name, *args, **kwargs, &block) else - default_store.send(name, *args, &block) + default_store.send(name, *args, **kwargs, &block) end end end @@ -101,6 +134,20 @@ module Gitlab end end + ENUMERATOR_CACHE_HIT_VALIDATOR.each_key do |name| + define_method(name) do |*args, **kwargs, &block| + enumerator = if use_primary_and_secondary_stores? + read_command(name, *args, **kwargs) + else + default_store.send(name, *args, **kwargs) + end + + return enumerator if block.nil? + + enumerator.each(&block) + end + end + PIPELINED_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? @@ -170,12 +217,23 @@ module Gitlab extra.merge(command_name: command_name, instance_name: instance_name)) end + def ping(message = nil) + if use_primary_and_secondary_stores? + # Both stores have to response success for the ping to be considered success. + # We assume both stores cannot return different responses (only both "PONG" or both echo the message). + # If either store is not reachable, an Error will be raised anyway thus taking any response works. + [primary_store, secondary_store].map { |store| store.ping(message) }.first + else + default_store.ping(message) + end + end + private # @return [Boolean] def feature_enabled?(prefix) feature_table_exists? && - Feature.enabled?("#{prefix}_#{instance_name.underscore}") && + Feature.enabled?("#{prefix}_#{instance_name.underscore}") && # rubocop:disable Cop/FeatureFlagUsage !same_redis_store? end @@ -193,15 +251,17 @@ module Gitlab def log_method_missing(command_name, *_args) return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) + raise MethodMissingError if Rails.env.test? || Rails.env.development? + log_error(MethodMissingError.new, command_name) increment_method_missing_count(command_name) end - def read_command(command_name, *args, &block) + def read_command(command_name, *args, **kwargs, &block) if @instance - send_command(@instance, command_name, *args, &block) + send_command(@instance, command_name, *args, **kwargs, &block) else - read_one_with_fallback(command_name, *args, &block) + read_one_with_fallback(command_name, *args, **kwargs, &block) end end @@ -213,19 +273,28 @@ module Gitlab end end - def read_one_with_fallback(command_name, *args, &block) + def read_one_with_fallback(command_name, *args, **kwargs, &block) begin - value = send_command(primary_store, command_name, *args, &block) + value = send_command(primary_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) end - value || fallback_read(command_name, *args, &block) + return value if cache_hit?(command_name, value) + + fallback_read(command_name, *args, **kwargs, &block) + end + + def cache_hit?(command, value) + validator = READ_CACHE_HIT_VALIDATOR[command] || ENUMERATOR_CACHE_HIT_VALIDATOR[command] + return false unless validator + + !value.nil? && validator.call(value) end - def fallback_read(command_name, *args, &block) - value = send_command(secondary_store, command_name, *args, &block) + def fallback_read(command_name, *args, **kwargs, &block) + value = send_command(secondary_store, command_name, *args, **kwargs, &block) if value log_error(ReadFromPrimaryError.new, command_name) diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb new file mode 100644 index 00000000000..8bfbfcfea60 --- /dev/null +++ b/lib/gitlab/redis/repository_cache.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class RepositoryCache < ::Gitlab::Redis::Wrapper + class << self + # The data we store on RepositoryCache used to be stored on Cache. + def config_fallback + Cache + end + + def cache_store + @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( + redis: pool, + compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), + namespace: Cache::CACHE_NAMESPACE, + # Cache should not grow forever + expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + ) + end + + private + + def redis + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 0e5389dc995..e5e1e1d4165 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -41,21 +41,6 @@ module Gitlab size end - def _raw_config - return @_raw_config if defined?(@_raw_config) - - @_raw_config = - begin - if filename = config_file_name - ERB.new(File.read(filename)).result.freeze - else - false - end - rescue Errno::ENOENT - false - end - end - def config_file_path(filename) path = File.join(rails_root, 'config', filename) return path if File.file?(path) @@ -67,10 +52,6 @@ module Gitlab File.expand_path('../../..', __dir__) end - def config_fallback? - config_file_name == config_fallback&.config_file_name - end - def config_file_name [ # Instance specific config sources: @@ -91,6 +72,10 @@ module Gitlab ].compact.first end + def redis_yml_path + File.join(rails_root, 'config/redis.yml') + end + def store_name name.demodulize end @@ -212,16 +197,20 @@ module Gitlab end def fetch_config - return false unless self.class._raw_config - - yaml = YAML.safe_load(self.class._raw_config, aliases: true) + redis_yml = read_yaml(self.class.redis_yml_path).fetch(@rails_env, {}) + instance_config_yml = read_yaml(self.class.config_file_name)[@rails_env] + + [ + redis_yml[self.class.store_name.underscore], + instance_config_yml, + self.class.config_fallback && redis_yml[self.class.config_fallback.store_name.underscore] + ].compact.first + end - # If the file has content but it's invalid YAML, `load` returns false - if yaml - yaml.fetch(@rails_env, false) - else - false - end + def read_yaml(path) + YAML.safe_load(ERB.new(File.read(path.to_s)).result, aliases: true) || {} + rescue Errno::ENOENT + {} end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 4f76cce2c7d..828cf65fb82 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -251,6 +251,26 @@ module Gitlab extend self extend Packages + def bulk_import_namespace_path_regex + # This regexp validates the string conforms to rules for a namespace path: + # i.e does not start with a non-alphanueric character except for periods or underscores, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' + @bulk_import_namespace_path_regex ||= %r/^([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+$/i + end + + def group_path_regex + # This regexp validates the string conforms to rules for a group slug: + # i.e does not start with a non-alphanueric character except for periods or underscores, + # contains only alphanumeric characters, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' + @group_path_regex ||= %r/^[.]?[^\W]([.]?[0-9a-z][-_]*)+$/i + end + def project_name_regex # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} # hence the Ruby warning. diff --git a/lib/gitlab/repository_cache.rb b/lib/gitlab/repository_cache.rb index dc8b2467f72..8de2c2fe772 100644 --- a/lib/gitlab/repository_cache.rb +++ b/lib/gitlab/repository_cache.rb @@ -5,7 +5,7 @@ module Gitlab class RepositoryCache attr_reader :repository, :namespace, :backend - def initialize(repository, extra_namespace: nil, backend: Rails.cache) + def initialize(repository, extra_namespace: nil, backend: self.class.store) @repository = repository @namespace = "#{repository.full_path}" @namespace += ":#{repository.project.id}" if repository.project @@ -48,5 +48,14 @@ module Gitlab value end + + def self.store + if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || + Feature.enabled?(:use_primary_store_as_default_for_repository_cache) + Gitlab::Redis::RepositoryCache.cache_store + else + Rails.cache + end + end end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index 1ecdf506208..ea90a341b1e 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -139,8 +139,17 @@ module Gitlab private + def cache + if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || + Feature.enabled?(:use_primary_store_as_default_for_repository_cache) + Gitlab::Redis::RepositoryCache + else + Gitlab::Redis::Cache + end + end + def with(&blk) - Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end # Take a hash and convert both keys and values to strings, for insertion into Redis. diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index baf48fd0dc1..c67ca92af40 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -64,5 +64,20 @@ module Gitlab redis.sscan_each(full_key, match: pattern) end end + + private + + def cache + if Feature.enabled?(:use_primary_and_secondary_stores_for_repository_cache) || + Feature.enabled?(:use_primary_store_as_default_for_repository_cache) + Gitlab::Redis::RepositoryCache + else + Gitlab::Redis::Cache + end + end + + def with(&blk) + cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end end end diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb new file mode 100644 index 00000000000..c77db02061c --- /dev/null +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb @@ -0,0 +1,202 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + module Runner + class RunnerFleetPipelineSeeder + DEFAULT_JOB_COUNT = 400 + + MAX_QUEUE_TIME_IN_SECONDS = 5.minutes.to_i + PIPELINE_CREATION_RANGE_MIN_IN_SECONDS = 2.hours.to_i + PIPELINE_CREATION_RANGE_MAX_IN_SECONDS = 30.days.to_i + PIPELINE_START_RANGE_MAX_IN_SECONDS = 5.minutes.to_i + PIPELINE_FINISH_RANGE_MAX_IN_SECONDS = 1.hour.to_i + + PROJECT_JOB_DISTRIBUTION = [ + { allocation: 70, job_count_default: 10 }, + { allocation: 15, job_count_default: 10 }, + { allocation: 15, job_count_default: 100 } + # remaining jobs on 4th project + ].freeze + + attr_reader :logger + + # Initializes the class + # + # @param [Gitlab::Logger] logger + # @param [Integer] job_count the number of jobs to create across the runners + # @param [Array<Hash>] projects_to_runners list of project IDs to respective runner IDs + def initialize(logger = Gitlab::AppLogger, projects_to_runners:, job_count:) + @logger = logger + @projects_to_runners = projects_to_runners.map do |v| + { project_id: v[:project_id], runners: ::Ci::Runner.id_in(v[:runner_ids]).to_a } + end + @job_count = job_count || DEFAULT_JOB_COUNT + end + + def seed + logger.info(message: 'Starting seed of runner fleet pipelines', job_count: @job_count) + + remaining_job_count = @job_count + PROJECT_JOB_DISTRIBUTION.each_with_index do |d, index| + remaining_job_count = create_pipelines_and_distribute_jobs(remaining_job_count, project_index: index, **d) + end + + while remaining_job_count > 0 + remaining_job_count -= create_pipeline( + job_count: remaining_job_count, + **@projects_to_runners[PROJECT_JOB_DISTRIBUTION.length], + status: random_pipeline_status + ) + end + + logger.info( + message: 'Completed seeding of runner fleet', + job_count: @job_count - remaining_job_count + ) + + nil + end + + private + + def create_pipelines_and_distribute_jobs(remaining_job_count, project_index:, allocation:, job_count_default:) + max_jobs_per_pipeline = [1, @job_count / 3].max + + create_pipelines( + remaining_job_count, + **@projects_to_runners[project_index], + total_jobs: @job_count * allocation / 100, + pipeline_job_count: job_count_default.clamp(1, max_jobs_per_pipeline) + ) + end + + def create_pipelines(remaining_job_count, project_id:, runners:, total_jobs:, pipeline_job_count:) + pipeline_job_count = remaining_job_count if pipeline_job_count > remaining_job_count + return 0 if pipeline_job_count == 0 + + pipeline_count = [1, total_jobs / pipeline_job_count].max + + (1..pipeline_count).each do + remaining_job_count -= create_pipeline( + job_count: pipeline_job_count, + project_id: project_id, + runners: runners, + status: random_pipeline_status + ) + end + + remaining_job_count + end + + def create_pipeline(job_count:, runners:, project_id:, status: 'success', **attrs) + logger.info(message: 'Creating pipeline with builds on project', + status: status, job_count: job_count, project_id: project_id, **attrs) + + raise ArgumentError('runners') unless runners + raise ArgumentError('project_id') unless project_id + + sha = '00000000' + if ::Ci::HasStatus::ALIVE_STATUSES.include?(status) || ::Ci::HasStatus::COMPLETED_STATUSES.include?(status) + created_at = Random.rand(PIPELINE_CREATION_RANGE_MIN_IN_SECONDS..PIPELINE_CREATION_RANGE_MAX_IN_SECONDS) + .seconds.ago + + if ::Ci::HasStatus::STARTED_STATUSES.include?(status) || + ::Ci::HasStatus::COMPLETED_STATUSES.include?(status) + started_at = created_at + Random.rand(1..PIPELINE_START_RANGE_MAX_IN_SECONDS) + if ::Ci::HasStatus::COMPLETED_STATUSES.include?(status) + finished_at = started_at + Random.rand(1..PIPELINE_FINISH_RANGE_MAX_IN_SECONDS) + end + end + end + + pipeline = ::Ci::Pipeline.new( + project_id: project_id, + ref: 'main', + sha: sha, + source: 'api', + status: status, + created_at: created_at, + started_at: started_at, + finished_at: finished_at, + **attrs + ) + pipeline.ensure_project_iid! # allocate an internal_id outside of pipeline creation transaction + pipeline.save! + + if created_at.present? + (1..job_count).each do |index| + create_build(pipeline, runners.sample, job_status(pipeline.status, index, job_count), index) + end + end + + job_count + end + + def create_build(pipeline, runner, job_status, index) + started_at = pipeline.started_at + finished_at = pipeline.finished_at + + max_job_duration = [MAX_QUEUE_TIME_IN_SECONDS, 5, 2].sample + max_job_duration = (finished_at - started_at) if finished_at && max_job_duration > finished_at - started_at + + job_created_at = pipeline.created_at + job_started_at = job_created_at + Random.rand(1..max_job_duration) if started_at + if finished_at + job_finished_at = Random.rand(job_started_at..finished_at) + elsif job_status == 'running' + job_finished_at = job_started_at + Random.rand(1 * 60..PIPELINE_FINISH_RANGE_MAX_IN_SECONDS) + end + + # Do not use the first 2 runner tags ('runner-fleet', "#{registration_prefix}runner"). + # See Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder#additional_runner_args + tags = runner.tags.offset(2).sample(Random.rand(1..5)) # rubocop: disable CodeReuse/ActiveRecord + + build_attrs = { + name: "Fake job #{index}", + scheduling_type: 'dag', + ref: 'main', + status: job_status, + pipeline_id: pipeline.id, + runner_id: runner.id, + project_id: pipeline.project_id, + tag_list: tags, + created_at: job_created_at, + queued_at: job_created_at, + started_at: job_started_at, + finished_at: job_finished_at + } + logger.info(message: 'Creating build', **build_attrs) + + ::Ci::Build.new(importing: true, **build_attrs).tap(&:save!) + end + + def random_pipeline_status + if Random.rand(1..4) == 4 + %w[created pending canceled running].sample + elsif Random.rand(1..3) == 1 + 'success' + else + 'failed' + end + end + + def job_status(pipeline_status, job_index, job_count) + return pipeline_status if %w[created pending success].include?(pipeline_status) + + # Ensure that a failed/canceled pipeline has at least 1 failed/canceled job + if job_index == job_count && ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(pipeline_status) + return pipeline_status + end + + possible_statuses = %w[failed success] + possible_statuses << pipeline_status if %w[canceled running].include?(pipeline_status) + + possible_statuses.sample + end + end + end + end + end +end diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb new file mode 100644 index 00000000000..082d267442c --- /dev/null +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb @@ -0,0 +1,246 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + module Runner + class RunnerFleetSeeder + DEFAULT_USERNAME = 'root' + DEFAULT_PREFIX = 'rf-' + DEFAULT_RUNNER_COUNT = 40 + DEFAULT_JOB_COUNT = DEFAULT_RUNNER_COUNT * 10 + + TAG_LIST = %w[gitlab-org docker ruby 2gb mysql linux shared shell deploy hhvm windows build postgres ios stage android stz front back review-apps pc java scraper test kubernetes staging no-priority osx php nodejs production nvm x86_64 gcc nginx dev unity odoo node sbt amazon xamarin debian gcloud e2e clang composer npm energiency dind flake8 cordova x64 private aws solution ruby2.2 python xcode kube compute mongo runner docker-compose phpunit t-matix docker-machine win server docker-in-docker redis go dotnet win7 area51-1 testing chefdk light osx_10-11 ubuntu gulp jertis gitlab-runner frontendv2 capifony centos7 mac gradle golang docker-builder runrepeat maven centos6 msvc14 amd64 xcode_8-2 macos VS2015 mono osx_10-12 azure-contend-docker msbuild git deployer local development python2.7 eezeeit release ios_9-3 fastlane selenium integration tests review cabinet-dev vs2015 ios_10-2 latex odoo_test quantum-ci prod sqlite heavy icc html-test labs feature alugha ps appivo-server fast web ios_9-2 c# python3 home js xcode_7-3 drupal 7 arm headless php70 gce x86 msvc builder Windows bower mssql pagetest wpf ssh inmobiliabeta.com xcode_7-2 repo laravel testonly gcp online-auth powershell ila-preprod ios_10-1 lossless sharesies backbone javascript fusonic-review autoscale ci ubuntu1604 rails windows10 xcode_8-1 php56 drupal embedded readyselect xamarin.ios XCode-8.1 iOS-10.1 macOS-10.12.1 develop taggun koumoul-internal docker-build iOS angular2 deployment xcode8 lcov test-cluster priv api bundler freebsd x86-64 BOB xcode_8 nuget vinome-backend cq_check fusonic-perf django php7 dy-manager-shell DEV mongodb neadev meteor ANSIBLE ftp master exerica-build server01 exerica-test mother-of-god nodejs-app ansible Golang mpi exploragen shootr Android macos_10-12 win64 ngsrunner @docker images script-maven ayk makepkg Linux ecolint wix xcode_8-0 coverage dreamhost multi ubuntu1404 eyeka jow3an-site repository politibot qt haskellstack arch priviti backend Sisyphus gm-dev dotNet internal support rpi .net buildbot-01 quay.io BOB2 codebnb vs2013 no-reset live 192.168.100.209 failfast-ci ios_10 crm_master_builds Qt packer selenium hub ci-shell rust dyscount-ci-manager-shell kubespray vagrant deployAutomobileBuild 1md k8s behat vinome-frontend development-nanlabs build-backend libvirt build-frontend contend-server windows-x64 chimpAPI ec2-runner kubectl linux-x64 epitech portals kvm ucaya-docker scala desktop buildmacbinaries ghc buildwinbinaries sonarqube deploySteelDistributorsBuild macOS r cpran rubocop binarylane r-packages alpha SIGAC tester area51-2 customer Build qa acegames_central mTaxNativeShell c++ cloveapp-ios smallville portal root lemmy nightly buildlinuxbinaries rundeck taxonic ios_10-0 n0004 data fedora rr-test seedai_master_builds geofence_master_builds].freeze # rubocop:disable Layout/LineLength + + attr_reader :logger + + # Initializes the class + # + # @param [Gitlab::Logger] logger + # @param [Hash] options + # @option options [String] :username username of the user that will create the fleet + # @option options [String] :registration_prefix string to use as prefix in group, project, and runner names + # @option options [Integer] :runner_count number of runners to create across the groups and projects + # @return [Array<Hash>] list of project IDs to respective runner IDs + def initialize(logger = Gitlab::AppLogger, **options) + username = options[:username] || DEFAULT_USERNAME + + @logger = logger + @user = User.find_by_username(username) + @registration_prefix = options[:registration_prefix] || DEFAULT_PREFIX + @runner_count = options[:runner_count] || DEFAULT_RUNNER_COUNT + @groups = {} + @projects = {} + end + + # seed returns an array of hashes of projects to its assigned runners + def seed + return unless within_plan_limits? + + logger.info( + message: 'Starting seed of runner fleet', + user_id: @user.id, + registration_prefix: @registration_prefix, + runner_count: @runner_count + ) + + groups_and_projects = create_groups_and_projects + runner_ids = create_runners(groups_and_projects) + + logger.info( + message: 'Completed seeding of runner fleet', + registration_prefix: @registration_prefix, + groups: @groups.count, + projects: @projects.count, + runner_count: @runner_count + ) + + %i[project_1_1_1_1 project_1_1_2_1 project_2_1_1].map do |project_key| + { project_id: groups_and_projects[project_key].id, runner_ids: runner_ids[project_key] } + end + end + + private + + def within_plan_limits? + plan_limits = Plan.default.actual_limits + + if plan_limits.ci_registered_group_runners < @runner_count + logger.error('The plan limits for group runners is set to ' \ + "#{plan_limits.ci_registered_group_runners} runners. " \ + 'You should raise the plan limits to avoid errors during runner creation') + return false + elsif plan_limits.ci_registered_project_runners < @runner_count + logger.error('The plan limits for project runners is set to ' \ + "#{plan_limits.ci_registered_project_runners} runners. " \ + 'You should raise the plan limits to avoid errors during runner creation') + return false + end + + true + end + + def create_groups_and_projects + root_group_1 = ensure_group(name: 'top-level group 1') + root_group_2 = ensure_group(name: 'top-level group 2') + group_1_1 = ensure_group(name: 'group 1.1', parent_id: root_group_1.id) + group_1_1_1 = ensure_group(name: 'group 1.1.1', parent_id: group_1_1.id) + group_1_1_2 = ensure_group(name: 'group 1.1.2', parent_id: group_1_1.id) + group_2_1 = ensure_group(name: 'group 2.1', parent_id: root_group_2.id) + + { + root_group_1: root_group_1, + root_group_2: root_group_2, + group_1_1: group_1_1, + group_1_1_1: group_1_1_1, + group_1_1_2: group_1_1_2, + project_1_1_1_1: ensure_project(name: 'project 1.1.1.1', namespace_id: group_1_1_1.id), + project_1_1_2_1: ensure_project(name: 'project 1.1.2.1', namespace_id: group_1_1_2.id), + group_2_1: group_2_1, + project_2_1_1: ensure_project(name: 'project 2.1.1', namespace_id: group_2_1.id) + } + end + + def create_runners(gp) + instance_runners = [] + group_1_1_1_runners = [] + group_2_1_runners = [] + project_1_1_1_1_runners = [] + project_1_1_2_1_runners = [] + project_2_1_1_runners = [] + instance_runners << create_runner(name: 'instance runner 1') + project_1_1_1_1_shared_runner_1 = + create_runner(name: 'project 1.1.1.1 shared runner 1', scope: gp[:project_1_1_1_1]) + project_1_1_1_1_runners << project_1_1_1_1_shared_runner_1 + project_1_1_2_1_runners << assign_runner(project_1_1_1_1_shared_runner_1, gp[:project_1_1_2_1]) + project_2_1_1_runners << assign_runner(project_1_1_1_1_shared_runner_1, gp[:project_2_1_1]) + + (3..@runner_count).each do + case Random.rand(0..100) + when 0..30 + runner_name = "group 1.1.1 runner #{1 + group_1_1_1_runners.count}" + group_1_1_1_runners << create_runner(name: runner_name, scope: gp[:group_1_1_1]) + when 31..50 + runner_name = "project 1.1.1.1 runner #{1 + project_1_1_1_1_runners.count}" + project_1_1_1_1_runners << create_runner(name: runner_name, scope: gp[:project_1_1_1_1]) + when 51..99 + runner_name = "project 1.1.2.1 runner #{1 + project_1_1_2_1_runners.count}" + project_1_1_2_1_runners << create_runner(name: runner_name, scope: gp[:project_1_1_2_1]) + else + runner_name = "group 2.1 runner #{1 + group_2_1_runners.count}" + group_2_1_runners << create_runner(name: runner_name, scope: gp[:group_2_1]) + end + end + + { # use only the first 5 runners to assign CI jobs + project_1_1_1_1: + ((instance_runners + project_1_1_1_1_runners).map(&:id) + group_1_1_1_runners.map(&:id)).first(5), + project_1_1_2_1: (instance_runners + project_1_1_2_1_runners).map(&:id).first(5), + project_2_1_1: + ((instance_runners + project_2_1_1_runners).map(&:id) + group_2_1_runners.map(&:id)).first(5) + } + end + + def ensure_group(name:, parent_id: nil, **args) + args[:description] ||= "Runner fleet #{name}" + name = generate_name(name) + + group = ::Group.by_parent(parent_id).find_by_name(name) + group ||= create_group(name: name, path: name.tr(' ', '-'), parent_id: parent_id, **args) + + register_record(group, @groups) + end + + def generate_name(name) + "#{@registration_prefix}#{name}" + end + + def create_group(**args) + logger.info(message: 'Creating group', **args) + + ensure_success(::Groups::CreateService.new(@user, **args).execute) + end + + def ensure_project(name:, namespace_id:, **args) + args[:description] ||= "Runner fleet #{name}" + name = generate_name(name) + + project = ::Project.in_namespace(namespace_id).find_by_name(name) + project ||= create_project(name: name, namespace_id: namespace_id, **args) + + register_record(project, @projects) + end + + def create_project(**args) + logger.info(message: 'Creating project', **args) + + ensure_success(::Projects::CreateService.new(@user, **args).execute) + end + + def register_record(record, records) + return record if record.errors.any? + + records[record.id] = record + end + + def ensure_success(record) + return record unless record.errors.any? + + logger.error(record.errors.full_messages.to_sentence) + raise RuntimeError + end + + def create_runner(name:, scope: nil, **args) + name = generate_name(name) + + scope_name = scope.class.name if scope + logger.info(message: 'Creating runner', scope: scope_name, name: name) + + executor = ::Ci::Runner::EXECUTOR_NAME_TO_TYPES.keys.sample + args.merge!(additional_runner_args(name, executor)) + + runners_token = if scope.nil? + Gitlab::CurrentSettings.runners_registration_token + else + scope.runners_token + end + + response = ::Ci::Runners::RegisterRunnerService.new.execute(runners_token, name: name, **args) + runner = response.payload[:runner] + + ::Ci::Runners::ProcessRunnerVersionUpdateWorker.new.perform(args[:version]) + + if runner && runner.errors.empty? && + Random.rand(0..100) < 70 # % of runners having contacted GitLab instance + runner.heartbeat(args.merge(executor: executor)) + runner.save! + end + + ensure_success(runner) + end + + def additional_runner_args(name, executor) + base_tags = ['runner-fleet', "#{@registration_prefix}runner", executor] + tag_limit = ::Ci::Runner::TAG_LIST_MAX_LENGTH - base_tags.length + + { + tag_list: base_tags + TAG_LIST.sample(Random.rand(1..tag_limit)), + description: "Runner fleet #{name}", + run_untagged: false, + active: Random.rand(1..3) != 1, + version: ::Gitlab::Ci::RunnerReleases.instance.releases.sample.to_s, + ip_address: '127.0.0.1' + } + end + + def assign_runner(runner, project) + result = ::Ci::Runners::AssignRunnerService.new(runner, project, @user).execute + result.track_and_raise_exception + + runner + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index 70798f8c3e8..c49180a6c1c 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -57,8 +57,8 @@ module Gitlab end def clear_memoization! - if instance_variable_defined?('@worker_metadatas') - remove_instance_variable('@worker_metadatas') + if instance_variable_defined?(:@worker_metadatas) + remove_instance_variable(:@worker_metadatas) end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 4bf9fd8470a..1682d62d782 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -241,9 +241,8 @@ module Gitlab deadline = Gitlab::Metrics::System.monotonic_time + time - # we try to finish as early as all jobs finished - # so we retest that in loop - sleep(CHECK_INTERVAL_SECONDS) while enabled? && any_jobs? && Gitlab::Metrics::System.monotonic_time < deadline + # Sleep until thread killed or timeout reached + sleep(CHECK_INTERVAL_SECONDS) while enabled? && Gitlab::Metrics::System.monotonic_time < deadline end def signal_pgroup(signal, explanation) @@ -289,10 +288,6 @@ module Gitlab def pid Process.pid end - - def any_jobs? - @sidekiq_daemon_monitor.jobs.any? - end end end end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index cd5587bbaef..6563968f315 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -81,3 +81,6 @@ module Gitlab end end end + +Gitlab::SQL::Pattern.prepend_mod +Gitlab::SQL::Pattern::ClassMethods.prepend_mod_with('Gitlab::SQL::Pattern::ClassMethods') diff --git a/lib/gitlab/ssh/commit.rb b/lib/gitlab/ssh/commit.rb index bfeefc47f13..d9ac8c1b881 100644 --- a/lib/gitlab/ssh/commit.rb +++ b/lib/gitlab/ssh/commit.rb @@ -16,6 +16,8 @@ module Gitlab commit_sha: @commit.sha, project: @commit.project, key_id: signature.signed_by_key&.id, + key_fingerprint_sha256: signature.key_fingerprint, + user_id: signature.signed_by_key&.user_id, verification_status: signature.verification_status } end diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index a654d5b2ff1..763d89116f1 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -41,6 +41,10 @@ module Gitlab end end + def key_fingerprint + strong_memoize(:key_fingerprint) { signature&.public_key&.fingerprint } + end + private def all_attributes_present? @@ -77,10 +81,6 @@ module Gitlab nil end end - - def key_fingerprint - strong_memoize(:key_fingerprint) { signature&.public_key&.fingerprint } - end end end end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index e9c8e816f18..707f7f3fc0a 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -2,6 +2,8 @@ module Gitlab class SSHPublicKey + include Gitlab::Utils::StrongMemoize + Technology = Struct.new(:name, :key_class, :supported_sizes, :supported_algorithms) # See https://man.openbsd.org/sshd#AUTHORIZED_KEYS_FILE_FORMAT for the list of @@ -15,29 +17,6 @@ module Gitlab Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) ].freeze - BANNED_SSH_KEY_FINGERPRINTS = [ - # https://github.com/rapid7/ssh-badkeys/tree/master/authorized - # banned ssh rsa keys - "SHA256:Z+q4XhSwWY7q0BIDVPR1v/S306FjGBsid7tLq/8kIxM", - "SHA256:uy5wXyEgbRCGsk23+J6f85om7G55Cu3UIPwC7oMZhNQ", - "SHA256:9prMbqhS4QteoFQ1ZRJDqSBLWoHXPyKB0iWR05Ghro4", - "SHA256:1M4RzhMyWuFS/86uPY/ce2prh/dVTHW7iD2RhpquOZA", - - # banned ssh dsa keys - "SHA256:/JLp6z6uGE3BPcs70RQob6QOdEWQ6nDC0xY7ejPOCc0", - "SHA256:whDP3xjKBEettbDuecxtGsfWBST+78gb6McdB9P7jCU", - "SHA256:MEc4HfsOlMqJ3/9QMTmrKn5Xj/yfnMITMW8EwfUfTww", - "SHA256:aPoYT2nPIfhqv6BIlbCCpbDjirBxaDFOtPfZ2K20uWw", - "SHA256:VtjqZ5fiaeoZ3mXOYi49Lk9aO31iT4pahKFP9JPiQPc", - - # other banned ssh keys - # https://github.com/BenBE/kompromat/commit/c8d9a05ea155a1ed609c617d4516f0ac978e8559 - "SHA256:Z+q4XhSwWY7q0BIDVPR1v/S306FjGBsid7tLq/8kIxM", - - # https://www.ctrlu.net/vuln/0006.html - "SHA256:2ewGtK7Dc8XpnfNKShczdc8HSgoEGpoX+MiJkfH2p5I" - ].to_set.freeze - def self.technologies if Gitlab::FIPS.enabled? Gitlab::FIPS::SSH_KEY_TECHNOLOGIES @@ -139,11 +118,21 @@ module Gitlab end def banned? - BANNED_SSH_KEY_FINGERPRINTS.include?(fingerprint_sha256) + return false unless valid? + + banned_ssh_keys.fetch(type.to_s, []).include?(fingerprint_sha256) end private + def banned_ssh_keys + path = Rails.root.join('config/security/banned_ssh_keys.yml') + config = YAML.load_file(path) if File.exist?(path) + + config || {} + end + strong_memoize_attr :banned_ssh_keys + def technology @technology ||= self.class.technology_for_key(key) || raise_unsupported_key_type_error diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 8d816c8d902..b68e1ace658 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -7,6 +7,11 @@ module Gitlab class Aggregate include Gitlab::Usage::TimeFrame + # TODO: define this missing event https://gitlab.com/gitlab-org/gitlab/-/issues/385080 + EVENTS_NOT_DEFINED_YET = %w[ + i_code_review_merge_request_widget_license_compliance_warning + ].freeze + def initialize(recorded_at) @recorded_at = recorded_at end @@ -14,11 +19,12 @@ module Gitlab def calculate_count_for_aggregation(aggregation:, time_frame:) with_validate_configuration(aggregation, time_frame) do source = SOURCES[aggregation[:source]] + events = select_defined_events(aggregation[:events], aggregation[:source]) if aggregation[:operator] == UNION_OF_AGGREGATED_METRICS - source.calculate_metrics_union(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + source.calculate_metrics_union(**time_constraints(time_frame).merge(metric_names: events, recorded_at: recorded_at)) else - source.calculate_metrics_intersections(**time_constraints(time_frame).merge(metric_names: aggregation[:events], recorded_at: recorded_at)) + source.calculate_metrics_intersections(**time_constraints(time_frame).merge(metric_names: events, recorded_at: recorded_at)) end end rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error @@ -71,6 +77,16 @@ module Gitlab { start_date: nil, end_date: nil } end end + + def select_defined_events(events, source) + # Database source metrics get validated inside the PostgresHll class: + # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb#L16 + return events if source != ::Gitlab::Usage::Metrics::Aggregates::REDIS_SOURCE + + events.select do |event| + ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) || EVENTS_NOT_DEFINED_YET.include?(event) + end + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 55da2315e45..0c102f0f386 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -15,7 +15,7 @@ module Gitlab def available?(&block) return @metric_available = block if block - return @metric_available.call if instance_variable_defined?('@metric_available') + return @metric_available.call if instance_variable_defined?(:@metric_available) true end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator.rb new file mode 100644 index 00000000000..db313bc1fbe --- /dev/null +++ b/lib/gitlab/usage/service_ping/legacy_metric_metadata_decorator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module ServicePing + class LegacyMetricMetadataDecorator < SimpleDelegator + attr_reader :duration, :error + + delegate :class, :is_a?, :kind_of?, :nil?, to: :__getobj__ + + def initialize(value, duration, error: nil) + @duration = duration + @error = error + super(value) + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb deleted file mode 100644 index e32dcd3777b..00000000000 --- a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module ServicePing - class LegacyMetricTimingDecorator < SimpleDelegator - attr_reader :duration - - delegate :class, :is_a?, :kind_of?, to: :__getobj__ - - def initialize(value, duration) - @duration = duration - super(value) - end - end - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 24f6cc725f6..c105288fff0 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -344,6 +344,11 @@ module Gitlab def jira_usage # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 # so we can just check for subdomains of atlassian.net + jira_integration_data_hash = jira_integration_data + if jira_integration_data_hash.nil? + return { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } + end + results = { projects_jira_server_active: 0, projects_jira_cloud_active: 0, @@ -351,14 +356,10 @@ module Gitlab projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - jira_integration_data_hash = jira_integration_data results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active] results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] results - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } end # rubocop: enable CodeReuse/ActiveRecord @@ -602,13 +603,18 @@ module Gitlab } end - def with_duration + def with_metadata result = nil + error = nil + duration = Benchmark.realtime do result = yield + rescue StandardError => e + error = e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end - ::Gitlab::Usage::ServicePing::LegacyMetricTimingDecorator.new(result, duration) + ::Gitlab::Usage::ServicePing::LegacyMetricMetadataDecorator.new(result, duration, error: error) end private diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 4486ca53966..0d15475eebb 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,7 +5,7 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self - def with_duration + def with_metadata yield end diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 7e78363dae5..eb44b7ddd95 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -23,7 +23,7 @@ module Gitlab # def enabled? # Feature.enabled?(:some_feature) # end - # strong_memoize_attr :enabled?, :enabled + # strong_memoize_attr :enabled? # def strong_memoize(name) key = ivar(name) @@ -46,20 +46,20 @@ module Gitlab end def strong_memoized?(name) - instance_variable_defined?(ivar(name)) + key = ivar(StrongMemoize.normalize_key(name)) + instance_variable_defined?(key) end def clear_memoization(name) - key = ivar(name) + key = ivar(StrongMemoize.normalize_key(name)) remove_instance_variable(key) if instance_variable_defined?(key) end module StrongMemoizeClassMethods - def strong_memoize_attr(method_name, member_name = nil) - member_name ||= method_name + def strong_memoize_attr(method_name) + member_name = StrongMemoize.normalize_key(method_name) - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :do_strong_memoize, self, method_name, member_name) + StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend end end @@ -83,7 +83,14 @@ module Gitlab end end - class <<self + class << self + def normalize_key(key) + return key unless key.end_with?('!', '?') + + # Replace invalid chars like `!` and `?` with allowed Unicode codeparts. + key.to_s.tr('!?', "\uFF01\uFF1F") + end + private def do_strong_memoize(klass, method_name, member_name) diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 0b818b99ac7..fab8617bcda 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -44,7 +44,7 @@ module Gitlab DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 - def with_duration + def with_metadata yield end @@ -55,7 +55,7 @@ module Gitlab end def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil, start_at: Time.current) - with_duration do + with_metadata do if batch Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) else @@ -68,7 +68,7 @@ module Gitlab end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do if batch Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) else @@ -81,7 +81,7 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter .new(relation, column) .execute(batch_size: batch_size, start: start, finish: finish) @@ -96,7 +96,7 @@ module Gitlab end def sum(relation, column, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) rescue ActiveRecord::StatementInvalid => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) @@ -105,7 +105,7 @@ module Gitlab end def average(relation, column, batch_size: nil, start: nil, finish: nil) - with_duration do + with_metadata do Gitlab::Database::BatchCount.batch_average(relation, column, batch_size: batch_size, start: start, finish: finish) rescue ActiveRecord::StatementInvalid => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) @@ -119,7 +119,7 @@ module Gitlab # # rubocop: disable CodeReuse/ActiveRecord def histogram(relation, column, buckets:, bucket_size: buckets.size) - with_duration do + with_metadata do # Using lambda to avoid exposing histogram specific methods parameters_valid = lambda do error_message = @@ -184,7 +184,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def add(*args) - with_duration do + with_metadata do break -1 if args.any?(&:negative?) args.sum @@ -195,7 +195,7 @@ module Gitlab end def alt_usage_data(value = nil, fallback: FALLBACK, &block) - with_duration do + with_metadata do if block yield else @@ -208,7 +208,7 @@ module Gitlab end def redis_usage_data(counter = nil, &block) - with_duration do + with_metadata do if block redis_usage_counter(&block) elsif counter.present? @@ -218,7 +218,7 @@ module Gitlab end def with_prometheus_client(fallback: {}, verify: true) - with_duration do + with_metadata do client = prometheus_client(verify: verify) break fallback unless client @@ -257,7 +257,7 @@ module Gitlab # rubocop: disable UsageData/LargeTable: def jira_integration_data - with_duration do + with_metadata do data = { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb index 61de003c28d..0351c9b30b3 100644 --- a/lib/gitlab/version_info.rb +++ b/lib/gitlab/version_info.rb @@ -7,11 +7,14 @@ module Gitlab attr_reader :major, :minor, :patch VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze + # To mitigate ReDoS, limit the length of the version string we're + # willing to check + MAX_VERSION_LENGTH = 128 def self.parse(str, parse_suffix: false) if str.is_a?(self) str - elsif str && m = str.match(VERSION_REGEX) + elsif str && str.length <= MAX_VERSION_LENGTH && m = str.match(VERSION_REGEX) VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) else VersionInfo.new diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index e37bd1f7606..5eef4fd0e4e 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -15,13 +15,6 @@ module GoogleApi class Client < GoogleApi::Auth SCOPE = 'https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/service.management' LEAST_TOKEN_LIFE_TIME = 10.minutes - CLUSTER_MASTER_AUTH_USERNAME = 'admin' - CLUSTER_IPV4_CIDR_BLOCK = '/16' - CLUSTER_OAUTH_SCOPES = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring" - ].freeze ROLES_LIST = %w[roles/iam.serviceAccountUser roles/artifactregistry.admin roles/cloudbuild.builds.builder roles/run.admin roles/storage.admin roles/cloudsql.client roles/browser].freeze REVOKE_URL = 'https://oauth2.googleapis.com/revoke' @@ -59,36 +52,6 @@ module GoogleApi true end - def projects_zones_clusters_get(project_id, zone, cluster_id) - service = Google::Apis::ContainerV1::ContainerService.new - service.authorization = access_token - - service.get_zone_cluster(project_id, zone, cluster_id, options: user_agent_header) - end - - def projects_zones_clusters_create(project_id, zone, cluster_name, cluster_size, machine_type:, legacy_abac:, enable_addons: []) - service = Google::Apis::ContainerV1beta1::ContainerService.new - service.authorization = access_token - - cluster_options = make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons) - - request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(**cluster_options) - - service.create_cluster(project_id, zone, request_body, options: user_agent_header) - end - - def projects_zones_operations(project_id, zone, operation_id) - service = Google::Apis::ContainerV1::ContainerService.new - service.authorization = access_token - - service.get_zone_operation(project_id, zone, operation_id, options: user_agent_header) - end - - def parse_operation_id(self_link) - m = self_link.match(%r{projects/.*/zones/.*/operations/(.*)}) - m[1] if m - end - def list_projects result = [] @@ -210,38 +173,6 @@ module GoogleApi service.enable_service(name) end - def make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons) - { - cluster: { - name: cluster_name, - initial_node_count: cluster_size, - node_config: { - machine_type: machine_type, - oauth_scopes: CLUSTER_OAUTH_SCOPES - }, - master_auth: { - client_certificate_config: { - issue_client_certificate: true - } - }, - legacy_abac: { - enabled: legacy_abac - }, - ip_allocation_policy: { - use_ip_aliases: true, - cluster_ipv4_cidr_block: CLUSTER_IPV4_CIDR_BLOCK - }, - addons_config: make_addons_config(enable_addons) - } - } - end - - def make_addons_config(enable_addons) - enable_addons.index_with do |addon| - { disabled: false } - end - end - def token_life_time(expires_at) DateTime.strptime(expires_at, '%s').to_time.utc - Time.now.utc end diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 9449e51b053..7d2f825e119 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -44,6 +44,7 @@ module ObjectStorage GetURL: get_url, StoreURL: store_url, DeleteURL: delete_url, + SkipDelete: false, MultipartUpload: multipart_upload_hash, CustomPutHeaders: true, PutHeaders: upload_options diff --git a/lib/sidebars/groups/menus/observability_menu.rb b/lib/sidebars/groups/menus/observability_menu.rb index 656142375af..d85efb1a002 100644 --- a/lib/sidebars/groups/menus/observability_menu.rb +++ b/lib/sidebars/groups/menus/observability_menu.rb @@ -6,9 +6,8 @@ module Sidebars class ObservabilityMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - add_item(dashboards_menu_item) add_item(explore_menu_item) - add_item(manage_menu_item) + add_item(datasources_menu_item) end override :title @@ -23,14 +22,14 @@ module Sidebars override :render? def render? - can?(context.current_user, :read_observability, context.group) + Gitlab::Observability.observability_enabled?(context.current_user, context.group) end private def dashboards_menu_item ::Sidebars::MenuItem.new( - title: _('Dashboards'), + title: s_('Observability|Dashboards'), link: group_observability_dashboards_path(context.group), active_routes: { path: 'groups/observability#dashboards' }, item_id: :dashboards @@ -39,16 +38,25 @@ module Sidebars def explore_menu_item ::Sidebars::MenuItem.new( - title: _('Explore'), + title: s_('Observability|Explore telemetry data'), link: group_observability_explore_path(context.group), active_routes: { path: 'groups/observability#explore' }, item_id: :explore ) end + def datasources_menu_item + ::Sidebars::MenuItem.new( + title: s_('Observability|Data sources'), + link: group_observability_datasources_path(context.group), + active_routes: { path: 'groups/observability#datasources' }, + item_id: :datasources + ) + end + def manage_menu_item ::Sidebars::MenuItem.new( - title: _('Manage Dashboards'), + title: s_('Observability|Manage dashboards'), link: group_observability_manage_path(context.group), active_routes: { path: 'groups/observability#manage' }, item_id: :manage diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index ede195a8e59..5b81f22c796 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -15,6 +15,7 @@ module Sidebars add_item(ci_cd_menu_item) add_item(applications_menu_item) add_item(packages_and_registries_menu_item) + add_item(usage_quotas_menu_item) return true elsif Gitlab.ee? && can?(context.current_user, :change_push_rules, context.group) # Push Rules are the only group setting that can also be edited by maintainers. @@ -115,6 +116,22 @@ module Sidebars ) end + def usage_quotas_menu_item + return ::Sidebars::NilMenuItem.new(item_id: :usage_quotas) unless usage_quotas_menu_enabled? + + ::Sidebars::MenuItem.new( + title: s_('UsageQuota|Usage Quotas'), + link: group_usage_quotas_path(context.group), + active_routes: { path: 'usage_quotas#index' }, + item_id: :usage_quotas + ) + end + + # overriden in ee/lib/ee/sidebars/groups/menus/settings_menu.rb + def usage_quotas_menu_enabled? + context.group.usage_quotas_enabled? + end + def packages_and_registries_menu_item unless context.group.packages_feature_enabled? return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index 5f789748288..4d4e65e9795 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -10,6 +10,10 @@ module Sidebars add_item(feature_flags_menu_item) add_item(releases_menu_item) + if Feature.enabled?(:show_pages_in_deployments_menu, context.current_user, type: :experiment) + add_item(pages_menu_item) + end + true end @@ -74,6 +78,19 @@ module Sidebars container_html_options: { class: 'shortcuts-deployments-releases' } ) end + + def pages_menu_item + unless context.project.pages_available? && context.current_user&.can?(:update_pages, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :pages) + end + + ::Sidebars::MenuItem.new( + title: _('Pages'), + link: project_pages_path(context.project), + active_routes: { path: 'pages#show' }, + item_id: :pages + ) + end end end end diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 11d5f4d59c7..eea32d8b626 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -16,7 +16,11 @@ module Sidebars add_item(merge_requests_menu_item) add_item(ci_cd_menu_item) add_item(packages_and_registries_menu_item) - add_item(pages_menu_item) + + if Feature.disabled?(:show_pages_in_deployments_menu, context.current_user, type: :experiment) + add_item(pages_menu_item) + end + add_item(monitor_menu_item) add_item(usage_quotas_menu_item) diff --git a/lib/sidebars/your_work/menus/activity_menu.rb b/lib/sidebars/your_work/menus/activity_menu.rb new file mode 100644 index 00000000000..d39c9bfda9c --- /dev/null +++ b/lib/sidebars/your_work/menus/activity_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class ActivityMenu < ::Sidebars::Menu + override :link + def link + activity_dashboard_path + end + + override :title + def title + _('Activity') + end + + override :sprite_icon + def sprite_icon + 'history' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { path: 'dashboard#activity' } + end + end + end + end +end diff --git a/lib/sidebars/your_work/menus/groups_menu.rb b/lib/sidebars/your_work/menus/groups_menu.rb new file mode 100644 index 00000000000..fd50b9b4b50 --- /dev/null +++ b/lib/sidebars/your_work/menus/groups_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class GroupsMenu < ::Sidebars::Menu + override :link + def link + dashboard_groups_path + end + + override :title + def title + _('Groups') + end + + override :sprite_icon + def sprite_icon + 'group' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { controller: ['groups', 'dashboard/groups'] } + end + end + end + end +end diff --git a/lib/sidebars/your_work/menus/issues_menu.rb b/lib/sidebars/your_work/menus/issues_menu.rb new file mode 100644 index 00000000000..6046b78e54e --- /dev/null +++ b/lib/sidebars/your_work/menus/issues_menu.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class IssuesMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :link + def link + issues_dashboard_path(assignee_username: @context.current_user.username) + end + + override :title + def title + _('Issues') + end + + override :sprite_icon + def sprite_icon + 'issues' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { path: 'dashboard#issues' } + end + + override :has_pill? + def has_pill? + pill_count > 0 + end + + override :pill_count + def pill_count + context.current_user.assigned_open_issues_count + end + strong_memoize_attr :pill_count + end + end + end +end diff --git a/lib/sidebars/your_work/menus/merge_requests_menu.rb b/lib/sidebars/your_work/menus/merge_requests_menu.rb new file mode 100644 index 00000000000..695c2ffdf46 --- /dev/null +++ b/lib/sidebars/your_work/menus/merge_requests_menu.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class MergeRequestsMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :link + def link + merge_requests_dashboard_path(assignee_username: @context.current_user.username) + end + + override :title + def title + _('Merge requests') + end + + override :sprite_icon + def sprite_icon + 'merge-request' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { path: 'dashboard#merge_requests' } + end + + override :has_pill? + def has_pill? + pill_count > 0 + end + + override :pill_count + def pill_count + context.current_user.assigned_open_merge_requests_count + end + strong_memoize_attr :pill_count + end + end + end +end diff --git a/lib/sidebars/your_work/menus/milestones_menu.rb b/lib/sidebars/your_work/menus/milestones_menu.rb new file mode 100644 index 00000000000..9b643afeec5 --- /dev/null +++ b/lib/sidebars/your_work/menus/milestones_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class MilestonesMenu < ::Sidebars::Menu + override :link + def link + dashboard_milestones_path + end + + override :title + def title + _('Milestones') + end + + override :sprite_icon + def sprite_icon + 'clock' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { controller: 'dashboard/milestones' } + end + end + end + end +end diff --git a/lib/sidebars/your_work/menus/projects_menu.rb b/lib/sidebars/your_work/menus/projects_menu.rb new file mode 100644 index 00000000000..e8b2a1d7869 --- /dev/null +++ b/lib/sidebars/your_work/menus/projects_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class ProjectsMenu < ::Sidebars::Menu + override :link + def link + dashboard_projects_path + end + + override :title + def title + _('Projects') + end + + override :sprite_icon + def sprite_icon + 'project' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { controller: ['root', 'projects', 'dashboard/projects'] } + end + end + end + end +end diff --git a/lib/sidebars/your_work/menus/snippets_menu.rb b/lib/sidebars/your_work/menus/snippets_menu.rb new file mode 100644 index 00000000000..c7c591f03cd --- /dev/null +++ b/lib/sidebars/your_work/menus/snippets_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class SnippetsMenu < ::Sidebars::Menu + override :link + def link + dashboard_snippets_path + end + + override :title + def title + _('Snippets') + end + + override :sprite_icon + def sprite_icon + 'snippet' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { controller: :snippets } + end + end + end + end +end diff --git a/lib/sidebars/your_work/menus/todos_menu.rb b/lib/sidebars/your_work/menus/todos_menu.rb new file mode 100644 index 00000000000..d37ffadb579 --- /dev/null +++ b/lib/sidebars/your_work/menus/todos_menu.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + module Menus + class TodosMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :link + def link + dashboard_todos_path + end + + override :title + def title + _('To-Do List') + end + + override :sprite_icon + def sprite_icon + 'todo-done' + end + + override :render? + def render? + !!context.current_user + end + + override :active_routes + def active_routes + { path: 'dashboard/todos#index' } + end + + override :has_pill? + def has_pill? + pill_count > 0 + end + + override :pill_count + def pill_count + context.current_user.todos_pending_count + end + strong_memoize_attr :pill_count + end + end + end +end diff --git a/lib/sidebars/your_work/panel.rb b/lib/sidebars/your_work/panel.rb new file mode 100644 index 00000000000..215a2a2da09 --- /dev/null +++ b/lib/sidebars/your_work/panel.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Sidebars + module YourWork + class Panel < ::Sidebars::Panel + override :configure_menus + def configure_menus + add_menus + end + + override :aria_label + def aria_label + _('Your work') + end + + override :render_raw_scope_menu_partial + def render_raw_scope_menu_partial + "shared/nav/your_work_scope_header" + end + + private + + def add_menus + add_menu(Sidebars::YourWork::Menus::ProjectsMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::GroupsMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::IssuesMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::MergeRequestsMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::TodosMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::MilestonesMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::SnippetsMenu.new(context)) + add_menu(Sidebars::YourWork::Menus::ActivityMenu.new(context)) + end + end + end +end diff --git a/lib/system_check/ldap_check.rb b/lib/system_check/ldap_check.rb index 3d71edbc256..62f803a60fc 100644 --- a/lib/system_check/ldap_check.rb +++ b/lib/system_check/ldap_check.rb @@ -42,7 +42,7 @@ module SystemCheck end end end - rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e + rescue Errno::ECONNREFUSED => e $stdout.puts "Could not connect to the LDAP server: #{e.message}".color(:red) end end diff --git a/lib/tasks/contracts/merge_requests.rake b/lib/tasks/contracts/merge_requests.rake index 61823f0cf1a..5a6186d393d 100644 --- a/lib/tasks/contracts/merge_requests.rake +++ b/lib/tasks/contracts/merge_requests.rake @@ -14,15 +14,16 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_batch_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end Pact::VerificationTask.new(:get_diffs_metadata) do |pact| pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_metadata_helper.rb" + pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -31,14 +32,14 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/merge_requests/show/get_discussions_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end desc 'Run all merge request contract tests' task 'test:merge_requests', :contract_merge_requests do |_t, arg| - errors = %w[diffs_batch diffs_metadata discussions].each_with_object([]) do |task, err| + errors = %w[get_diffs_batch get_diffs_metadata get_discussions].each_with_object([]) do |task, err| Rake::Task["contracts:merge_requests:pact:verify:#{task}"].execute rescue StandardError, SystemExit err << "contracts:merge_requests:pact:verify:#{task}" diff --git a/lib/tasks/contracts/pipeline_schedules.rake b/lib/tasks/contracts/pipeline_schedules.rake index b4c87d2e3c9..f3e65b94940 100644 --- a/lib/tasks/contracts/pipeline_schedules.rake +++ b/lib/tasks/contracts/pipeline_schedules.rake @@ -14,7 +14,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipeline_schedules/edit/put_edit_a_pipeline_schedule_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake index 55a7baa4539..13c973f1358 100644 --- a/lib/tasks/contracts/pipelines.rake +++ b/lib/tasks/contracts/pipelines.rake @@ -14,7 +14,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/new/post_create_a_new_pipeline_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -23,7 +23,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/index/get_list_project_pipelines_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -32,7 +32,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/show/get_pipeline_header_data_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -41,7 +41,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/show/delete_pipeline_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index f0264456201..9c92aa5eb28 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -407,7 +407,12 @@ namespace :gitlab do Rails.application.eager_load! tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables } - classes = tables.index_with { [] } + + views = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.views } + + sources = tables + views + + classes = sources.index_with { [] } Gitlab::Database.database_base_models.each do |_, model_class| model_class @@ -421,12 +426,13 @@ namespace :gitlab do version = Gem::Version.new(File.read('VERSION')) milestone = version.release.segments[0..1].join('.') - tables.each do |table_name| - file = File.join(DB_DOCS_PATH, "#{table_name}.yml") + sources.each do |source_name| + file = dictionary_file_path(source_name, views) + key_name = "#{data_source_type(source_name, views)}_name" table_metadata = { - 'table_name' => table_name, - 'classes' => classes[table_name]&.sort&.uniq, + key_name => source_name, + 'classes' => classes[source_name]&.sort&.uniq, 'feature_categories' => [], 'description' => nil, 'introduced_by_url' => nil, @@ -438,12 +444,12 @@ namespace :gitlab do existing_metadata = YAML.safe_load(File.read(file)) - if existing_metadata['table_name'] != table_metadata['table_name'] - existing_metadata['table_name'] = table_metadata['table_name'] + if existing_metadata[key_name] != table_metadata[key_name] + existing_metadata[key_name] = table_metadata[key_name] outdated = true end - if existing_metadata['classes'].difference(table_metadata['classes']).any? + if existing_metadata['classes'].sort != table_metadata['classes'].sort existing_metadata['classes'] = table_metadata['classes'] outdated = true end @@ -455,6 +461,20 @@ namespace :gitlab do end end + private + + def data_source_type(source_name, views) + return 'view' if views.include?(source_name) + + 'table' + end + + def dictionary_file_path(source_name, views) + sub_directory = views.include?(source_name) ? 'views' : '' + + File.join(DB_DOCS_PATH, sub_directory, "#{source_name}.yml") + end + # Temporary disable this, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85760#note_998452069 # Rake::Task['db:migrate'].enhance do # Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake index a856aa77abc..212d60a7231 100644 --- a/lib/tasks/gitlab/db/lock_writes.rake +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -15,6 +15,7 @@ namespace :gitlab do table_name: table_name, connection: connection, database_name: database_name, + with_retries: true, logger: Logger.new($stdout), dry_run: ENV['DRY_RUN'] == 'true' ) @@ -39,6 +40,7 @@ namespace :gitlab do table_name: table_name, connection: connection, database_name: database_name, + with_retries: true, logger: Logger.new($stdout) ) diff --git a/lib/tasks/gitlab/security/update_banned_ssh_keys.rake b/lib/tasks/gitlab/security/update_banned_ssh_keys.rake new file mode 100644 index 00000000000..b3f8bb16ef9 --- /dev/null +++ b/lib/tasks/gitlab/security/update_banned_ssh_keys.rake @@ -0,0 +1,72 @@ +# frozen_string_literal: true +# Update banned SSH keys from a Git repository +# +# This task: +# - Reads banned SSH keys from a Git repository, and updates default key set at config/security/banned_ssh_keys.yml +# - Stops uploading new keys if YAML file size is greater than 2 MB. +# - Caution: The task adds all the files with suffix of .pub, and does NOT check the key's contents. +# +# @param git_url - Remote Git URL. +# @param output_file - Update keys to an output file. Default is config/security/banned_ssh_keys.yml. +# +# @example +# bundle exec rake "gitlab:security:update_banned_ssh_keys[https://github.com/rapid7/ssh-badkeys]" +# +MAX_CONFIG_SIZE = 2.megabytes.freeze + +namespace :gitlab do + namespace :security do + desc 'GitLab | Security | Update banned_ssh_keys config file from a remote Git repository' + task :update_banned_ssh_keys, [:git_url, :output_file] => :gitlab_environment do |_t, args| + require 'yaml' + require 'git' + require 'find' + require_relative '../../../../config/environment' + logger = Logger.new($stdout) + begin + exit 0 unless Rails.env.test? || Rails.env.development? + name = args.git_url.rpartition('/').last.delete_suffix('.git') + tmp_path = Dir.mktmpdir + logger.info "start to clone the git repository at #{tmp_path}/#{name}" + Git.clone(args.git_url, name, path: tmp_path) + logger.info "Git clone finished. Next, add bad keys to config/security/banned_ssh_keys.yml." + + path = args.output_file || Rails.root.join('config/security/banned_ssh_keys.yml') + config_size = File.size?(path) || 0 + exit 0 if config_size > MAX_CONFIG_SIZE + + config = (YAML.load_file(path) if File.exist?(path)) || {} + + Find.find("#{tmp_path}/#{name}") do |path| + next unless path.end_with?('.pub') + + if config_size > MAX_CONFIG_SIZE + logger.info "banned_ssh_keys.yml has grown too large - halting execution" + break + end + + logger.info "update bad SSH keys in #{path}" + keys = File.readlines(path, chomp: true) + keys.each do |key| + pub = Gitlab::SSHPublicKey.new(key) + + type = pub.type.to_s + config[type] = [] unless config.key?(type) + + next if config[type].include?(pub.fingerprint_sha256) + + config[type].append(pub.fingerprint_sha256) + config_size += pub.fingerprint_sha256.size + end + end + rescue StandardError => e + logger.error "Exception: #{e.message}" + logger.debug e.backtrace + exit 1 + end + + logger.info "finish writing." + File.open(path, 'w') { |file| file.write(config.to_yaml) } + end + end +end diff --git a/lib/tasks/gitlab/seed/runner_fleet.rake b/lib/tasks/gitlab/seed/runner_fleet.rake new file mode 100644 index 00000000000..c0b79269c75 --- /dev/null +++ b/lib/tasks/gitlab/seed/runner_fleet.rake @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Seed database with: +# 1. 2 root groups, one with 2 sub-groups and another with 1 sub-group +# 1. 1 project in each of the sub-groups +# 1. 1 instance runner, 1 shared project runner, and group/project runners in some groups/projects +# 1. Successful and failed pipelines assigned to the first 5 available runners of each group/project +# 1. 1 pipeline on one group runner with the remaining jobs +# +# @param username - user creating subgroups (i.e. GitLab admin) +# @param registration_prefix - prefix used for the group, project, and runner names +# @param runner_count - total number of runners to create (default: 40) +# @param job_count - total number of jobs to create and assign to runners (default: 400) +# +# @example +# bundle exec rake "gitlab:seed:runner_fleet[root, rf-]" +# +namespace :gitlab do + namespace :seed do + desc 'Seed groups with sub-groups/projects/runners/jobs for Runner Fleet testing' + task :runner_fleet, + [:username, :registration_prefix, :runner_count, :job_count] => :gitlab_environment do |_t, args| + timings = Benchmark.measure do + projects_to_runners = Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder.new( + Gitlab::AppLogger, + username: args.username, + registration_prefix: args.registration_prefix, + runner_count: args.runner_count&.to_i + ).seed + + Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder.new( + projects_to_runners: projects_to_runners, + job_count: args.job_count&.to_i + ).seed + end + + puts "Seed finished. Timings: #{timings}" + end + end +end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index ec2ea623e02..b3559bde988 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -78,7 +78,7 @@ namespace :tw do CodeOwnerRule.new('Tutorials', '@kpaizee'), CodeOwnerRule.new('Utilization', '@fneill'), CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'), - CodeOwnerRule.new('Workspace', '@lciutacu') + CodeOwnerRule.new('Organization', '@lciutacu') ].freeze ERRORS_EXCLUDED_FILES = [ |