diff options
Diffstat (limited to 'lib')
294 files changed, 10983 insertions, 1812 deletions
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index 4aebd9c0d40..d6c212a9886 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -112,7 +112,7 @@ module API helpers do def clusterable_instance - Clusters::Instance.new + ::Clusters::Instance.new end def clusters_for_current_user diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index d595b5b2e09..99be30809d2 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -5,7 +5,7 @@ module API class PlanLimits < ::API::Base before { authenticated_as_admin! } - feature_category :not_owned + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned helpers do def current_plan(name) diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb index 05eb7f8222b..9be432046a5 100644 --- a/lib/api/admin/sidekiq.rb +++ b/lib/api/admin/sidekiq.rb @@ -5,7 +5,7 @@ module API class Sidekiq < ::API::Base before { authenticated_as_admin! } - feature_category :not_owned + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned namespace 'admin' do namespace 'sidekiq' do diff --git a/lib/api/alert_management_alerts.rb b/lib/api/alert_management_alerts.rb new file mode 100644 index 00000000000..88230c86247 --- /dev/null +++ b/lib/api/alert_management_alerts.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module API + class AlertManagementAlerts < ::API::Base + feature_category :incident_management + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :alert_iid, type: Integer, desc: 'The IID of the Alert' + end + + resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/alert_management_alerts/:alert_iid/metric_images' do + post 'authorize' do + authorize!(:upload_alert_management_metric_image, find_project_alert(request.params[:alert_iid])) + + require_gitlab_workhorse! + ::Gitlab::Workhorse.verify_api_request!(request.headers) + status 200 + content_type ::Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + + params = { + has_length: false, + maximum_size: ::AlertManagement::MetricImage::MAX_FILE_SIZE.to_i + } + + ::MetricImageUploader.workhorse_authorize(**params) + end + + desc 'Upload a metric image for an alert' do + success Entities::MetricImage + end + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The image file to be uploaded' + optional :url, type: String, desc: 'The url to view more metric info' + optional :url_text, type: String, desc: 'A description of the image or URL' + end + post do + require_gitlab_workhorse! + bad_request!('File is too large') if max_file_size_exceeded? + + alert = find_project_alert(params[:alert_iid]) + + authorize!(:upload_alert_management_metric_image, alert) + + upload = ::AlertManagement::MetricImages::UploadService.new( + alert, + current_user, + params.slice(:file, :url, :url_text) + ).execute + + if upload.success? + present upload.payload[:metric], + with: Entities::MetricImage, + current_user: current_user, + project: user_project + else + render_api_error!(upload.message, upload.http_status) + end + end + + desc 'Metric Images for alert' + get do + alert = find_project_alert(params[:alert_iid]) + + if can?(current_user, :read_alert_management_metric_image, alert) + present alert.metric_images.order_created_at_asc, with: Entities::MetricImage + else + render_api_error!('Alert not found', 404) + end + end + + desc 'Update a metric image for an alert' do + success Entities::MetricImage + end + params do + requires :metric_image_id, type: Integer, desc: 'The ID of metric image' + optional :url, type: String, desc: 'The url to view more metric info' + optional :url_text, type: String, desc: 'A description of the image or URL' + end + put ':metric_image_id' do + alert = find_project_alert(params[:alert_iid]) + + authorize!(:update_alert_management_metric_image, alert) + + render_api_error!('Feature not available', 403) unless alert.metric_images_available? + + metric_image = alert.metric_images.find_by_id(params[:metric_image_id]) + + render_api_error!('Metric image not found', 404) unless metric_image + + if metric_image.update(params.slice(:url, :url_text)) + present metric_image, with: Entities::MetricImage, current_user: current_user, project: user_project + else + unprocessable_entity!('Metric image could not be updated') + end + end + + desc 'Remove a metric image for an alert' do + success Entities::MetricImage + end + params do + requires :metric_image_id, type: Integer, desc: 'The ID of metric image' + end + delete ':metric_image_id' do + alert = find_project_alert(params[:alert_iid]) + + authorize!(:destroy_alert_management_metric_image, alert) + + render_api_error!('Feature not available', 403) unless alert.metric_images_available? + + metric_image = alert.metric_images.find_by_id(params[:metric_image_id]) + + render_api_error!('Metric image not found', 404) unless metric_image + + if metric_image.destroy + no_content! + else + unprocessable_entity!('Metric image could not be deleted') + end + end + end + end + + helpers do + def find_project_alert(iid, project_id = nil) + project = project_id ? find_project!(project_id) : user_project + + ::AlertManagement::AlertsFinder.new(current_user, project, { iid: [iid] }).execute.first + end + + def max_file_size_exceeded? + params[:file].size > ::AlertManagement::MetricImage::MAX_FILE_SIZE + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 5100ec9ec9d..4dca47efdf2 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -80,6 +80,10 @@ module API Gitlab::UsageDataCounters::JetBrainsPluginActivityUniqueCounter.track_api_request_when_trackable(user_agent: request&.user_agent, user: @current_user) end + after do + Gitlab::UsageDataCounters::GitLabCliActivityUniqueCounter.track_api_request_when_trackable(user_agent: request&.user_agent, user: @current_user) + end + # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } @@ -159,6 +163,7 @@ module API mount ::API::Admin::InstanceClusters mount ::API::Admin::PlanLimits mount ::API::Admin::Sidekiq + mount ::API::AlertManagementAlerts mount ::API::Appearance mount ::API::Applications mount ::API::Avatar @@ -178,6 +183,7 @@ module API mount ::API::Ci::SecureFiles mount ::API::Ci::Triggers mount ::API::Ci::Variables + mount ::API::Clusters::Agents mount ::API::Commits mount ::API::CommitStatuses mount ::API::ContainerRegistryEvent @@ -317,7 +323,7 @@ module API end end - route :any, '*path', feature_category: :not_owned do + route :any, '*path', feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned error!('404 Not Found', 404) end end diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index c732da17166..53967e0af5d 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -10,7 +10,7 @@ module API def bulk_imports @bulk_imports ||= ::BulkImports::ImportsFinder.new( user: current_user, - status: params[:status] + params: params ).execute end @@ -22,7 +22,7 @@ module API @bulk_import_entities ||= ::BulkImports::EntitiesFinder.new( user: current_user, bulk_import: bulk_import, - status: params[:status] + params: params ).execute end @@ -70,6 +70,8 @@ module API end params do use :pagination + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return GitLab Migrations sorted in created by `asc` or `desc` order.' optional :status, type: String, values: BulkImport.all_human_statuses, desc: 'Return GitLab Migrations with specified status' end @@ -82,13 +84,15 @@ module API end params do use :pagination + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return GitLab Migrations sorted in created by `asc` or `desc` order.' optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses, desc: "Return all GitLab Migrations' entities with specified status" end get :entities do entities = ::BulkImports::EntitiesFinder.new( user: current_user, - status: params[:status] + params: params ).execute present paginate(entities), with: Entities::BulkImports::Entity diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 43ed35b99fd..173cfc9a59a 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -104,10 +104,7 @@ module API def set_application_context return unless current_job - Gitlab::ApplicationContext.push( - user: -> { current_job.user }, - project: -> { current_job.project } - ) + Gitlab::ApplicationContext.push(job: current_job) end def track_ci_minutes_usage!(_build, _runner) diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index 9f59eea5013..0800993602b 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -28,7 +28,7 @@ module API requires :job, type: String, desc: 'The name for the job' end route_setting :authentication, job_token_allowed: true - get ':id/jobs/artifacts/:ref_name/download', + get ':id/jobs/artifacts/:ref_name/download', urgency: :low, requirements: { ref_name: /.+/ } do authorize_download_artifacts! @@ -87,7 +87,7 @@ module API requires :artifact_path, type: String, desc: 'Artifact path' end route_setting :authentication, job_token_allowed: true - get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + get ':id/jobs/:job_id/artifacts/*artifact_path', urgency: :low, format: false do authorize_download_artifacts! build = find_build!(params[:job_id]) @@ -100,7 +100,11 @@ module API bad_request! unless path.valid? - send_artifacts_entry(build.artifacts_file, path) + # This endpoint is being used for Artifact Browser feature that renders the content via pages. + # Since Content-Type is controlled by Rails and Workhorse, if a wrong + # content-type is sent, it could cause a regression on pages rendering. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/357078 for more information. + legacy_send_artifacts_entry(build.artifacts_file, path) end desc 'Keep the artifacts to prevent them from being deleted' do @@ -140,8 +144,6 @@ module API desc 'Expire the artifacts files from a project' delete ':id/artifacts' do - not_found! unless Feature.enabled?(:bulk_expire_project_artifacts, default_enabled: :yaml) - authorize_destroy_artifacts! ::Ci::JobArtifacts::DeleteProjectArtifactsService.new(project: user_project).execute diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index d9d0da2e4d1..86897eb61ae 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -114,11 +114,14 @@ module API build = find_build!(params[:job_id]) authorize!(:update_build, build) - break forbidden!('Job is not retryable') unless build.retryable? - build = ::Ci::Build.retry(build, current_user) + response = ::Ci::RetryJobService.new(@project, current_user).execute(build) - present build, with: Entities::Ci::Job + if response.success? + present response[:job], with: Entities::Ci::Job + else + forbidden!('Job is not retryable') + end end desc 'Erase job (remove artifacts and the trace)' do @@ -194,7 +197,7 @@ module API pipeline = current_authenticated_job.pipeline project = current_authenticated_job.project - agent_authorizations = Clusters::AgentAuthorizationsFinder.new(project).execute + agent_authorizations = ::Clusters::AgentAuthorizationsFinder.new(project).execute project_groups = project.group&.self_and_ancestor_ids&.map { |id| { id: id } } || [] user_access_level = project.team.max_member_access(current_user.id) roles_in_project = Gitlab::Access.sym_options_with_owner diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 2d7a437ca08..8d2c58dabdf 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -146,7 +146,7 @@ module API use :pagination end - get ':id/pipelines/:pipeline_id/bridges', feature_category: :pipeline_authoring do + get ':id/pipelines/:pipeline_id/bridges', urgency: :low, feature_category: :pipeline_authoring do authorize!(:read_build, user_project) pipeline = user_project.all_pipelines.find(params[:pipeline_id]) diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index d5b21e2ef29..ee39bdfd90c 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -54,6 +54,7 @@ module API resource do before do + read_only_feature_flag_enabled? authorize! :admin_secure_files, user_project end @@ -97,6 +98,10 @@ module API def feature_flag_enabled? service_unavailable! unless Feature.enabled?(:ci_secure_files, user_project, default_enabled: :yaml) end + + def read_only_feature_flag_enabled? + service_unavailable! if Feature.enabled?(:ci_secure_files_read_only, user_project, type: :ops, default_enabled: :yaml) + end end end end diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb index 9c04d5e9923..ec9951aba0d 100644 --- a/lib/api/ci/variables.rb +++ b/lib/api/ci/variables.rb @@ -23,7 +23,7 @@ module API params do use :pagination end - get ':id/variables' do + get ':id/variables', urgency: :low do variables = user_project.variables present paginate(variables), with: Entities::Ci::Variable end diff --git a/lib/api/clusters/agents.rb b/lib/api/clusters/agents.rb new file mode 100644 index 00000000000..6c1bf21b952 --- /dev/null +++ b/lib/api/clusters/agents.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module API + module Clusters + class Agents < ::API::Base + include PaginationParams + + before { authenticate! } + + feature_category :kubernetes_management + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'List agents' do + detail 'This feature was introduced in GitLab 14.10.' + success Entities::Clusters::Agent + end + params do + use :pagination + end + get ':id/cluster_agents' do + authorize! :read_cluster, user_project + + agents = ::Clusters::AgentsFinder.new(user_project, current_user).execute + + present paginate(agents), with: Entities::Clusters::Agent + end + + desc 'Get single agent' do + detail 'This feature was introduced in GitLab 14.10.' + success Entities::Clusters::Agent + end + params do + requires :agent_id, type: Integer, desc: 'The ID of an agent' + end + get ':id/cluster_agents/:agent_id' do + authorize! :read_cluster, user_project + + agent = user_project.cluster_agents.find(params[:agent_id]) + + present agent, with: Entities::Clusters::Agent + end + + desc 'Add an agent to a project' do + detail 'This feature was introduced in GitLab 14.10.' + success Entities::Clusters::Agent + end + params do + requires :name, type: String, desc: 'The name of the agent' + end + post ':id/cluster_agents' do + authorize! :create_cluster, user_project + + params = declared_params(include_missing: false) + + result = ::Clusters::Agents::CreateService.new(user_project, current_user).execute(name: params[:name]) + + bad_request!(result[:message]) if result[:status] == :error + + present result[:cluster_agent], with: Entities::Clusters::Agent + end + + desc 'Delete an agent' do + detail 'This feature was introduced in GitLab 14.10.' + end + params do + requires :agent_id, type: Integer, desc: 'The ID of an agent' + end + delete ':id/cluster_agents/:agent_id' do + authorize! :admin_cluster, user_project + + agent = user_project.cluster_agents.find(params.delete(:agent_id)) + + destroy_conditionally!(agent) + end + end + end + end +end diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 0e6e04d2645..c311b34a697 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -113,10 +113,6 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - unauthorized_user_project! - end - desc 'Composer packages endpoint for registering packages' namespace ':id/packages/composer' do route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true @@ -150,8 +146,11 @@ module API requires :sha, type: String, desc: 'Shasum of current json' requires :package_name, type: String, file_path: true, desc: 'The Composer package name' end + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get 'archives/*package_name' do - metadata = unauthorized_user_project + authorize_read_package!(authorized_user_project) + + metadata = authorized_user_project .packages .composer .with_name(params[:package_name]) @@ -161,9 +160,9 @@ module API not_found! unless metadata - track_package_event('pull_package', :composer, project: unauthorized_user_project, namespace: unauthorized_user_project.namespace) + track_package_event('pull_package', :composer, project: authorized_user_project, namespace: authorized_user_project.namespace) - send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true + send_git_archive authorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true end end end diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index 465c5f4112b..db51d4380d0 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -40,6 +40,9 @@ module API expose :password_authentication_enabled_for_web, as: :signin_enabled expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services expose :asset_proxy_allowlist, as: :asset_proxy_whitelist + + # This field is deprecated and always returns true + expose(:housekeeping_bitmaps_enabled) { |_settings, _options| true } end end end diff --git a/lib/api/entities/award_emoji.rb b/lib/api/entities/award_emoji.rb index da9a183bf39..40dc38b1900 100644 --- a/lib/api/entities/award_emoji.rb +++ b/lib/api/entities/award_emoji.rb @@ -8,6 +8,7 @@ module API expose :user, using: Entities::UserBasic expose :created_at, :updated_at expose :awardable_id, :awardable_type + expose :url end end end diff --git a/lib/api/entities/basic_release_details.rb b/lib/api/entities/basic_release_details.rb new file mode 100644 index 00000000000..d13080f32f4 --- /dev/null +++ b/lib/api/entities/basic_release_details.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + class BasicReleaseDetails < Grape::Entity + include ::API::Helpers::Presentable + + expose :name + expose :tag, as: :tag_name + expose :description + expose :created_at + expose :released_at + expose :upcoming_release?, as: :upcoming_release + end + end +end diff --git a/lib/api/entities/ci/job_request/artifacts.rb b/lib/api/entities/ci/job_request/artifacts.rb index 4b09db40504..d1fb7d330b9 100644 --- a/lib/api/entities/ci/job_request/artifacts.rb +++ b/lib/api/entities/ci/job_request/artifacts.rb @@ -6,7 +6,7 @@ module API module JobRequest class Artifacts < Grape::Entity expose :name - expose :untracked + expose :untracked, expose_nil: false expose :paths expose :exclude, expose_nil: false expose :when diff --git a/lib/api/entities/clusters/agent.rb b/lib/api/entities/clusters/agent.rb index 3b4538b81c2..140b680f5e8 100644 --- a/lib/api/entities/clusters/agent.rb +++ b/lib/api/entities/clusters/agent.rb @@ -5,7 +5,10 @@ module API module Clusters class Agent < Grape::Entity expose :id + expose :name expose :project, with: Entities::ProjectIdentity, as: :config_project + expose :created_at + expose :created_by_user_id end end end diff --git a/lib/api/entities/commit_with_link.rb b/lib/api/entities/commit_with_link.rb index a135cc19480..23efaca34d5 100644 --- a/lib/api/entities/commit_with_link.rb +++ b/lib/api/entities/commit_with_link.rb @@ -29,7 +29,7 @@ module API end expose :signature_html, if: { type: :full } do |commit| - render('projects/commit/_signature', signature: commit.signature) if commit.has_signature? + ::CommitPresenter.new(commit).signature_html end expose :prev_commit_id, if: { type: :full } do |commit| @@ -50,12 +50,6 @@ module API pipelines_project_commit_path(pipeline_project, commit.id, ref: pipeline_ref) end - - def render(*args) - return unless request.respond_to?(:render) && request.render.respond_to?(:call) - - request.render.call(*args) - end end end end diff --git a/lib/api/entities/issue.rb b/lib/api/entities/issue.rb index e2506cc596e..f87ef093cd8 100644 --- a/lib/api/entities/issue.rb +++ b/lib/api/entities/issue.rb @@ -35,6 +35,10 @@ module API issue end + expose :severity, + format_with: :upcase, + documentation: { type: "String", desc: "One of #{::IssuableSeverity.severities.keys.map(&:upcase)}" } + # Calculating the value of subscribed field triggers Markdown # processing. We can't do that for multiple issues / merge # requests in a single API request. diff --git a/lib/api/entities/member.rb b/lib/api/entities/member.rb index 87f03adba31..7ce1e73a043 100644 --- a/lib/api/entities/member.rb +++ b/lib/api/entities/member.rb @@ -6,6 +6,7 @@ module API expose :user, merge: true, using: UserBasic expose :access_level expose :created_at + expose :created_by, with: UserBasic, expose_nil: false expose :expires_at end end diff --git a/lib/api/entities/merge_request_changes.rb b/lib/api/entities/merge_request_changes.rb index 488f33dfb93..a1e8b5ae00a 100644 --- a/lib/api/entities/merge_request_changes.rb +++ b/lib/api/entities/merge_request_changes.rb @@ -24,7 +24,7 @@ module API end def expose_raw_diffs? - options[:access_raw_diffs] || ::Feature.enabled?(:mrc_api_use_raw_diffs_from_gitaly, options[:project]) + options[:access_raw_diffs] end end end diff --git a/lib/api/entities/metric_image.rb b/lib/api/entities/metric_image.rb new file mode 100644 index 00000000000..fd5e3a62e40 --- /dev/null +++ b/lib/api/entities/metric_image.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class MetricImage < Grape::Entity + expose :id, :created_at, :filename, :file_path, :url, :url_text + end + end +end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 8f9a8add938..60cc5167c41 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -85,8 +85,11 @@ module API end expose :mr_default_target_self, if: -> (project) { project.forked? } + expose :import_url, if: -> (project, options) { Ability.allowed?(options[:current_user], :admin_project, project) } do |project| + project[:import_url] + end + expose :import_type, if: -> (project, options) { Ability.allowed?(options[:current_user], :admin_project, project) } expose :import_status - expose :import_error, if: lambda { |_project, options| options[:user_can_admin_project] } do |project| project.import_state&.last_error end diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index 056b54674f1..2403c907f7f 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -2,20 +2,14 @@ module API module Entities - class Release < Grape::Entity + class Release < BasicReleaseDetails include ::API::Helpers::Presentable - expose :name - expose :tag, as: :tag_name, if: ->(_, _) { can_download_code? } - expose :description expose :description_html, if: -> (_, options) { options[:include_html_description] } do |entity| MarkupHelper.markdown_field(entity, :description, current_user: options[:current_user]) end - expose :created_at - expose :released_at expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } expose :commit, using: Entities::Commit, if: ->(_, _) { can_download_code? } - expose :upcoming_release?, as: :upcoming_release expose :milestones, using: Entities::MilestoneWithStats, if: -> (release, _) { release.milestones.present? && can_read_milestone? } do |release, _| diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb index e148a5c45b5..f9c1a646a4f 100644 --- a/lib/api/entities/user_with_admin.rb +++ b/lib/api/entities/user_with_admin.rb @@ -5,6 +5,7 @@ module API class UserWithAdmin < UserPublic expose :admin?, as: :is_admin expose :note + expose :namespace_id end end end diff --git a/lib/api/entities/wiki_attachment.rb b/lib/api/entities/wiki_attachment.rb index e622dea04dd..03a6cc8d644 100644 --- a/lib/api/entities/wiki_attachment.rb +++ b/lib/api/entities/wiki_attachment.rb @@ -16,11 +16,11 @@ module API end def filename - object.file_name + object[:file_name] end def secure_url - object.file_path + object[:file_path] end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index c032b80e39b..19b48c1e3cf 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -131,7 +131,7 @@ module API environment = user_project.environments.find(params[:environment_id]) authorize! :stop_environment, environment - environment.stop_with_action!(current_user) + environment.stop_with_actions!(current_user) status 200 present environment, with: Entities::Environment, current_user: current_user diff --git a/lib/api/files.rb b/lib/api/files.rb index 39b3904ec90..41a8e899614 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -24,7 +24,8 @@ module API file_content_encoding: attrs[:encoding], author_email: attrs[:author_email], author_name: attrs[:author_name], - last_commit_sha: attrs[:last_commit_id] + last_commit_sha: attrs[:last_commit_id], + execute_filemode: attrs[:execute_filemode] } end @@ -65,7 +66,8 @@ module API ref: params[:ref], blob_id: @blob.id, commit_id: @commit.id, - last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path], literal_pathspec: true) + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path], literal_pathspec: true), + execute_filemode: @blob.executable? } end @@ -83,6 +85,7 @@ module API requires :content, type: String, desc: 'File content' optional :encoding, type: String, values: %w[base64], desc: 'File encoding' optional :last_commit_id, type: String, desc: 'Last known commit id for this file' + optional :execute_filemode, type: Boolean, desc: 'Enable / Disable the executable flag on the file path' end end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index f0c0182a02f..5754eceda97 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -3,8 +3,6 @@ module API class GroupExport < ::API::Base before do - not_found! unless Feature.enabled?(:group_import_export, user_group, default_enabled: true) - authorize! :admin_group, user_group end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 5fbf222be5d..0ed14476c61 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,10 +7,10 @@ module API before { authenticate_non_get! } - feature_category :subgroups - helpers Helpers::GroupsHelpers + feature_category :subgroups, ['/groups/:id/custom_attributes', '/groups/:id/custom_attributes/:key'] + helpers do params :statistics_params do optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' @@ -181,7 +181,7 @@ module API use :group_list_params use :with_custom_attributes end - get do + get feature_category: :subgroups do groups = find_groups(declared_params(include_missing: false), params[:id]) present_groups_with_pagination_strategies params, groups end @@ -196,7 +196,7 @@ module API use :optional_params end - post do + post feature_category: :subgroups do parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? if parent_group authorize! :create_subgroup, parent_group @@ -229,7 +229,7 @@ module API use :optional_update_params use :optional_update_params_ee end - put ':id' do + put ':id', feature_category: :subgroups do group = find_group!(params[:id]) group.preload_shared_group_links @@ -249,7 +249,8 @@ module API use :with_custom_attributes optional :with_projects, type: Boolean, default: true, desc: 'Omit project details' end - get ":id" do + # TODO: Set higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/357841 + get ":id", feature_category: :subgroups, urgency: :low do group = find_group!(params[:id]) group.preload_shared_group_links @@ -265,7 +266,7 @@ module API end desc 'Remove a group.' - delete ":id" do + delete ":id", feature_category: :subgroups do group = find_group!(params[:id]) authorize! :admin_group, group check_subscription! group @@ -300,7 +301,8 @@ module API use :with_custom_attributes use :optional_projects_params end - get ":id/projects" do + # TODO: Set higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/211498 + get ":id/projects", feature_category: :subgroups, urgency: :low do finder_options = { only_owned: !params[:with_shared], include_subgroups: params[:include_subgroups], @@ -334,7 +336,7 @@ module API use :pagination use :with_custom_attributes end - get ":id/projects/shared" do + get ":id/projects/shared", feature_category: :subgroups do projects = find_group_projects(params, { only_shared: true }) present_projects(params, projects) @@ -347,7 +349,7 @@ module API use :group_list_params use :with_custom_attributes end - get ":id/subgroups" do + get ":id/subgroups", feature_category: :subgroups, urgency: :low do groups = find_groups(declared_params(include_missing: false), params[:id]) present_groups params, groups end @@ -359,7 +361,7 @@ module API use :group_list_params use :with_custom_attributes end - get ":id/descendant_groups" do + get ":id/descendant_groups", feature_category: :subgroups do finder_params = declared_params(include_missing: false).merge(include_parent_descendants: true) groups = find_groups(finder_params, params[:id]) present_groups params, groups @@ -371,7 +373,7 @@ module API params do requires :project_id, type: String, desc: 'The ID or path of the project' end - post ":id/projects/:project_id", requirements: { project_id: /.+/ } do + post ":id/projects/:project_id", requirements: { project_id: /.+/ }, feature_category: :projects do authenticated_as_admin! group = find_group!(params[:id]) group.preload_shared_group_links @@ -391,7 +393,7 @@ module API desc: 'The ID of the target group to which the group needs to be transferred to.'\ 'If not provided, the source group will be promoted to a root group.' end - post ':id/transfer' do + post ':id/transfer', feature_category: :subgroups do group = find_group!(params[:id]) authorize! :admin_group, group @@ -415,7 +417,7 @@ module API requires :group_access, type: Integer, values: Gitlab::Access.all_values, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end - post ":id/share" do + post ":id/share", feature_category: :subgroups do shared_group = find_group!(params[:id]) shared_with_group = find_group!(params[:group_id]) @@ -438,7 +440,7 @@ module API requires :group_id, type: Integer, desc: 'The ID of the shared group' end # rubocop: disable CodeReuse/ActiveRecord - delete ":id/share/:group_id" do + delete ":id/share/:group_id", feature_category: :subgroups do shared_group = find_group!(params[:id]) link = shared_group.shared_with_group_links.find_by(shared_with_group_id: params[:group_id]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index de9d42bdce7..e4a7f2213ae 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -705,8 +705,16 @@ module API body '' end + # Deprecated. Use `send_artifacts_entry` instead. + def legacy_send_artifacts_entry(file, entry) + header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) + + body '' + end + def send_artifacts_entry(file, entry) header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) + header(*Gitlab::Workhorse.detect_content_type) body '' end diff --git a/lib/api/integrations.rb b/lib/api/integrations.rb index ff1d88e35f0..71c55704ddf 100644 --- a/lib/api/integrations.rb +++ b/lib/api/integrations.rb @@ -6,7 +6,7 @@ module API integrations = Helpers::IntegrationsHelpers.integrations integration_classes = Helpers::IntegrationsHelpers.integration_classes - if Rails.env.development? + if Gitlab.dev_or_test_env? integrations['mock-ci'] = [ { required: true, diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 9c527f28d44..2ab5d482295 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -189,7 +189,7 @@ module API present actor.user, with: Entities::UserSafe end - get '/check', feature_category: :not_owned do + get '/check', feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned { api_version: API.version, gitlab_version: Gitlab::VERSION, diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index df887a83c4f..59bc917a602 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -54,7 +54,7 @@ module API def check_agent_token unauthorized! unless agent_token - Clusters::AgentTokens::TrackUsageService.new(agent_token).execute + ::Clusters::AgentTokens::TrackUsageService.new(agent_token).execute end end @@ -91,9 +91,9 @@ module API requires :agent_config, type: JSON, desc: 'Configuration for the Agent' end post '/' do - agent = Clusters::Agent.find(params[:agent_id]) + agent = ::Clusters::Agent.find(params[:agent_id]) - Clusters::Agents::RefreshAuthorizationService.new(agent, config: params[:agent_config]).execute + ::Clusters::Agents::RefreshAuthorizationService.new(agent, config: params[:agent_config]).execute no_content! end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index d78576b5d5b..75f63a5d98f 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -20,19 +20,25 @@ module API success Entities::Invitation end params do - requires :email, types: [String, Array[String]], email_or_email_list: true, desc: 'The email address to invite, or multiple emails separated by comma' requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :email, types: [String, Array[String]], email_or_email_list: true, desc: 'The email address to invite, or multiple emails separated by comma' + optional :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end post ":id/invitations" do - params[:source] = find_source(source_type, params[:id]) + ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/354016') - authorize_admin_source!(source_type, params[:source]) + bad_request!('Must provide either email or user_id as a parameter') if params[:email].blank? && params[:user_id].blank? - ::Members::InviteService.new(current_user, params).execute + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + + create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id], source: source }) + + ::Members::InviteService.new(current_user, create_service_params).execute end desc 'Get a list of group or project invitations viewable by the authenticated user' do diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 98451afb12d..0e93a4adb65 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -67,14 +67,16 @@ module API requires :issue_link_id, type: Integer, desc: 'The ID of an issue link' end delete ':id/issues/:issue_iid/links/:issue_link_id' do - issue_link = IssueLink.find(declared_params[:issue_link_id]) + issue = find_project_issue(params[:issue_iid]) + issue_link = IssueLink + .for_source_or_target(issue) + .find(declared_params[:issue_link_id]) - find_project_issue(params[:issue_iid]) find_project_issue(issue_link.target.iid.to_s, issue_link.target.project_id.to_s) result = ::IssueLinks::DestroyService - .new(issue_link, current_user) - .execute + .new(issue_link, current_user) + .execute if result[:status] == :success present issue_link, with: Entities::IssueLink diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 6de78c81cac..f65ecf3b4a6 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -21,7 +21,7 @@ module API optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' optional :include_jobs, type: Boolean, desc: 'Whether or not to include CI jobs in the response' end - post '/lint' do + post '/lint', urgency: :low do unauthorized! unless can_lint_ci? result = Gitlab::Ci::Lint.new(project: nil, current_user: current_user) diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index de612ff8321..c465087c4a2 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -2,7 +2,7 @@ module API class Markdown < ::API::Base - feature_category :not_owned + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned params do requires :text, type: String, desc: "The markdown text to render" diff --git a/lib/api/members.rb b/lib/api/members.rb index 4798edc4ddf..01e859c94c4 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -7,6 +7,7 @@ module API before { authenticate! } feature_category :authentication_and_authorization + urgency :low helpers ::API::Helpers::MembersHelpers @@ -100,8 +101,6 @@ module API end post ":id/members" do - ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/333434') - source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb index 0989340b3ea..c6406bf61df 100644 --- a/lib/api/metrics/dashboard/annotations.rb +++ b/lib/api/metrics/dashboard/annotations.rb @@ -12,7 +12,7 @@ module API ANNOTATIONS_SOURCES = [ { class: ::Environment, resource: :environments, create_service_param_key: :environment }, - { class: Clusters::Cluster, resource: :clusters, create_service_param_key: :cluster } + { class: ::Clusters::Cluster, resource: :clusters, create_service_param_key: :cluster } ].freeze ANNOTATIONS_SOURCES.each do |annotations_source| diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index d2468fb1c2e..1f3516e0667 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -6,8 +6,6 @@ module API before { authenticate! } - feature_category :subgroups - helpers do params :optional_list_params_ee do # EE::API::Namespaces would override this helper @@ -32,7 +30,7 @@ module API use :pagination use :optional_list_params_ee end - get do + get feature_category: :subgroups do owned_only = params[:owned_only] == true namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only) @@ -54,7 +52,7 @@ module API params do requires :id, type: String, desc: "Namespace's ID or path" end - get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups do user_namespace = find_namespace!(params[:id]) present user_namespace, with: Entities::Namespace, current_user: current_user @@ -67,7 +65,7 @@ module API requires :namespace, type: String, desc: "Namespace's path" optional :parent_id, type: Integer, desc: "The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered." end - get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups do namespace_path = params[:namespace] exists = Namespace.without_project_namespaces.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists? diff --git a/lib/api/notes.rb b/lib/api/notes.rb index b260f5289b3..c12b3bf5562 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -112,7 +112,7 @@ module API requires :noteable_id, type: Integer, desc: 'The ID of the noteable' requires :note_id, type: Integer, desc: 'The ID of a note' optional :body, type: String, allow_blank: false, desc: 'The content of a note' - optional :confidential, type: Boolean, desc: 'Confidentiality note flag' + optional :confidential, type: Boolean, desc: '[Deprecated in 14.10] No longer allowed to update confidentiality of notes' end put ":id/#{noteables_str}/:noteable_id/notes/:note_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 7d28394e034..420eabb41db 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -5,7 +5,7 @@ module API class NotificationSettings < ::API::Base before { authenticate! } - feature_category :users + feature_category :team_planning helpers ::API::Helpers::MembersHelpers diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb index 69b47f9420d..e8829216336 100644 --- a/lib/api/project_events.rb +++ b/lib/api/project_events.rb @@ -8,6 +8,9 @@ module API feature_category :users + # TODO: Set higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/357839 + urgency :low + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 843f72c0e1d..8b27d8d2163 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -25,7 +25,7 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - check_rate_limit! :project_download_export, scope: [current_user, user_project] + check_rate_limit! :project_download_export, scope: [current_user, user_project.namespace] if user_project.export_file_exists? if user_project.export_archive_exists? diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index fae170d638b..bd8faefa803 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -135,8 +135,6 @@ module API success Entities::ProjectImportStatus end post 'remote-import' do - not_found! unless ::Feature.enabled?(:import_project_from_remote_file, default_enabled: :yaml) - check_rate_limit! :project_import, scope: [current_user, :project_import] response = ::Import::GitlabProjects::CreateProjectService.new( diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index a80e45637dc..14792730eae 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -38,7 +38,7 @@ module API params do use :pagination end - get ":id/snippets" do + get ":id/snippets", urgency: :low do authenticate! present paginate(snippets_for_current_user), with: Entities::ProjectSnippet, current_user: current_user diff --git a/lib/api/projects.rb b/lib/api/projects.rb index d772079372c..9f7b3f9b088 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -20,6 +20,7 @@ module API projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = projects.with_statistics if params[:statistics] projects = projects.joins(:statistics) if params[:order_by].include?('project_statistics') # rubocop: disable CodeReuse/ActiveRecord + projects = projects.created_by(current_user).imported.with_import_state if params[:imported] lang = params[:with_programming_language] projects = projects.with_programming_language(lang) if lang @@ -125,6 +126,7 @@ module API optional :search_namespaces, type: Boolean, desc: "Include ancestor namespaces when matching search criteria" optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :imported, type: Boolean, default: false, desc: 'Limit by imported by authenticated user' optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' @@ -212,7 +214,7 @@ module API use :statistics_params use :with_custom_attributes end - get ":user_id/projects", feature_category: :projects do + get ":user_id/projects", feature_category: :projects, urgency: :default do user = find_user(params[:user_id]) not_found!('User') unless user @@ -249,7 +251,8 @@ module API use :statistics_params use :with_custom_attributes end - get feature_category: :projects do + # TODO: Set higher urgency https://gitlab.com/gitlab-org/gitlab/-/issues/211495 + get feature_category: :projects, urgency: :low do present_projects load_projects end @@ -338,7 +341,8 @@ module API optional :license, type: Boolean, default: false, desc: 'Include project license data' end - get ":id", feature_category: :projects do + # TODO: Set higher urgency https://gitlab.com/gitlab-org/gitlab/-/issues/357622 + get ":id", feature_category: :projects, urgency: :default do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, @@ -609,7 +613,8 @@ module API params do requires :project_id, type: Integer, desc: 'The ID of the source project to import the members from.' end - post ":id/import_project_members/:project_id", feature_category: :experimentation_expansion do + post ":id/import_project_members/:project_id", feature_category: :projects do + ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/355916') authorize! :admin_project, user_project source_project = Project.find_by_id(params[:project_id]) @@ -628,7 +633,7 @@ module API desc 'Workhorse authorize the file upload' do detail 'This feature was introduced in GitLab 13.11' end - post ':id/uploads/authorize', feature_category: :not_owned do + post ':id/uploads/authorize', feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned require_gitlab_workhorse! status 200 @@ -640,7 +645,7 @@ module API params do requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded' end - post ":id/uploads", feature_category: :not_owned do + post ":id/uploads", feature_category: :not_owned do # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned log_if_upload_exceed_max_size(user_project, params[:file]) service = UploadService.new(user_project, params[:file]) diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index aabecb43653..35f555e16b5 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -14,6 +14,7 @@ module API Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects_relation, options[:current_user]).execute if options[:current_user] Preloaders::SingleHierarchyProjectGroupPlansPreloader.new(projects_relation).execute if options[:single_hierarchy] + preload_groups(projects_relation) if options[:with] == Entities::Project projects_relation end @@ -40,6 +41,25 @@ module API def repositories_for_preload(projects_relation) projects_relation.map(&:repository) end + + # For all projects except those in a user namespace, the `namespace` + # and `group` are identical. Preload the group when it's not a user namespace. + def preload_groups(projects_relation) + return unless Feature.enabled?(:group_projects_api_preload_groups) + + group_projects = projects_for_group_preload(projects_relation) + groups = group_projects.map(&:namespace) + + Preloaders::GroupRootAncestorPreloader.new(groups).execute + + group_projects.each do |project| + project.group = project.namespace + end + end + + def projects_for_group_preload(projects_relation) + projects_relation.select { |project| project.namespace.type == Group.sti_name } + end end end end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 7b89a177fd9..9e085a91a7c 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -8,16 +8,48 @@ module API .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) RELEASE_CLI_USER_AGENT = 'GitLab-release-cli' - before { authorize_read_releases! } + feature_category :release_orchestration - after { track_release_event } + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before { authorize_read_group_releases! } - feature_category :release_orchestration + desc 'Get a list of releases for projects in this group.' do + success Entities::Release + end + params do + requires :id, type: Integer, desc: 'The ID of the group to get releases for' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order by released_at' + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + + use :pagination + end + get ":id/releases" do + not_found! unless Feature.enabled?(:group_releases_finder_inoperator) + + finder_options = { + sort: params[:sort] + } + + strict_params = declared_params(include_missing: false) + releases = find_group_releases(finder_options) + + present_group_releases(strict_params, releases) + end + end params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before { authorize_read_releases! } + + after { track_release_event } + desc 'Get a project releases' do detail 'This feature was introduced in GitLab 11.7.' named 'get_releases' @@ -162,6 +194,10 @@ module API end helpers do + def authorize_read_group_releases! + authorize! :read_release, user_group + end + def authorize_create_release! authorize! :create_release, user_project end @@ -220,6 +256,22 @@ module API Gitlab::Tracking.event(options[:for].name, options[:route_options][:named], project: user_project, user: current_user, **event_context) end + + def find_group_releases(finder_options) + ::Releases::GroupReleasesFinder + .new(user_group, current_user, finder_options) + .execute(preload: true) + end + + def present_group_releases(params, releases) + options = { + with: params[:simple] ? Entities::BasicReleaseDetails : Entities::Release, + current_user: current_user + } + + # GroupReleasesFinder has already ordered the data for us + present paginate(releases, skip_default_order: true), options + end end end end diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index 83096772d32..8de155312fb 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -25,6 +25,18 @@ module API with: Entities::RemoteMirror end + desc 'Get a single remote mirror' do + success Entities::RemoteMirror + end + params do + requires :mirror_id, type: String, desc: 'The ID of a remote mirror' + end + get ':id/remote_mirrors/:mirror_id' do + mirror = user_project.remote_mirrors.find(params[:mirror_id]) + + present mirror, with: Entities::RemoteMirror + end + desc 'Create remote mirror for a project' do success Entities::RemoteMirror end @@ -73,6 +85,29 @@ module API render_api_error!(result[:message], result[:http_status]) end end + + desc 'Delete a single remote mirror' do + detail 'This feature was introduced in GitLab 14.10' + end + params do + requires :mirror_id, type: String, desc: 'The ID of a remote mirror' + end + delete ':id/remote_mirrors/:mirror_id' do + mirror = user_project.remote_mirrors.find(params[:mirror_id]) + + destroy_conditionally!(mirror) do + mirror_params = declared_params(include_missing: false).merge(_destroy: 1) + mirror_params[:id] = mirror_params.delete(:mirror_id) + update_params = { remote_mirrors_attributes: mirror_params } + + # Note: We are using the update service to be consistent with how the controller handles deletion + result = ::Projects::UpdateService.new(user_project, current_user, update_params).execute + + if result[:status] != :success + render_api_error!(result[:message], 400) + end + end + end end end end diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index e52f8fd9111..2ba109b7092 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -27,6 +27,28 @@ module API present paginate(tokens), with: Entities::ResourceAccessToken, resource: resource end + desc 'Get an access token for the specified resource by ID' do + detail 'This feature was introduced in GitLab 14.10.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :token_id, type: String, desc: "The ID of the token" + end + get ":id/access_tokens/:token_id" do + resource = find_source(source_type, params[:id]) + + next unauthorized! unless current_user.can?(:read_resource_access_tokens, resource) + + token = find_token(resource, params[:token_id]) + + if token.nil? + next not_found!("Could not find #{source_type} access token with token_id: #{params[:token_id]}") + end + + resource.members.load + present token, with: Entities::ResourceAccessToken, resource: resource + end + desc 'Revoke a resource access token' do detail 'This feature was introduced in GitLab 13.9.' end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index b256432fbf1..774ab472f2d 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -4,7 +4,7 @@ module API class Settings < ::API::Base before { authenticated_as_admin! } - feature_category :not_owned + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned helpers Helpers::SettingsHelpers @@ -83,7 +83,6 @@ module API optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' given housekeeping_enabled: ->(val) { val } do - requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." @@ -182,7 +181,7 @@ module API optional :group_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for group runners, in seconds' optional :project_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for project runners, in seconds' - ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| + Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", type: Integer, values: KeyRestrictionValidator.supported_key_restrictions(type), diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index 680363d036e..c30b9d7583a 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -6,7 +6,7 @@ module API class SidekiqMetrics < ::API::Base before { authenticated_as_admin! } - feature_category :not_owned + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned helpers do def queue_metrics diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 9a3c68bc854..496532a15b2 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -184,7 +184,7 @@ module API params do use :raw_file_params end - get ":id/files/:ref/:file_path/raw", requirements: { file_path: API::NO_SLASH_URL_PART_REGEX } do + get ":id/files/:ref/:file_path/raw", urgency: :low, requirements: { file_path: API::NO_SLASH_URL_PART_REGEX } do snippet = snippets.find_by_id(params.delete(:id)) not_found!('Snippet') unless snippet&.repo_exists? @@ -200,7 +200,7 @@ module API get ":id/user_agent_detail" do authenticated_as_admin! - snippet = Snippet.find_by_id!(params[:id]) + snippet = Snippet.find(params[:id]) break not_found!('UserAgentDetail') unless snippet.user_agent_detail diff --git a/lib/api/users.rb b/lib/api/users.rb index 0f710e0a307..b26611cfe03 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -89,6 +89,7 @@ module API optional :created_before, type: DateTime, desc: 'Return users created before the specified time' optional :without_projects, type: Boolean, default: false, desc: 'Filters only users without projects' optional :exclude_internal, as: :non_internal, type: Boolean, default: false, desc: 'Filters only non internal users' + optional :without_project_bots, type: Boolean, default: false, desc: 'Filters users without project bots' optional :admins, type: Boolean, default: false, desc: 'Filters only admin users' all_or_none_of :extern_uid, :provider @@ -98,7 +99,7 @@ module API use :optional_index_params_ee end # rubocop: disable CodeReuse/ActiveRecord - get feature_category: :users do + get feature_category: :users, urgency: :default do authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present? unless current_user&.admin? @@ -120,8 +121,11 @@ module API users = reorder_users(users) entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic - users = users.preload(:identities, :u2f_registrations) if entity == Entities::UserWithAdmin - users = users.preload(:identities, :webauthn_registrations) if entity == Entities::UserWithAdmin + + if entity == Entities::UserWithAdmin + users = users.preload(:identities, :u2f_registrations, :webauthn_registrations, :namespace) + end + users, options = with_custom_attributes(users, { with: entity, current_user: current_user }) users = users.preload(:user_detail) @@ -139,7 +143,7 @@ module API use :with_custom_attributes end # rubocop: disable CodeReuse/ActiveRecord - get ":id", feature_category: :users do + get ":id", feature_category: :users, urgency: :medium do forbidden!('Not authorized!') unless current_user unless current_user.admin? @@ -164,7 +168,7 @@ module API params do requires :user_id, type: String, desc: 'The ID or username of the user' end - get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users do + get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users, urgency: :high do user = find_user(params[:user_id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -915,7 +919,7 @@ module API desc 'Get the currently authenticated user' do success Entities::UserPublic end - get feature_category: :users do + get feature_category: :users, urgency: :medium do entity = if current_user.admin? Entities::UserWithAdmin @@ -1090,7 +1094,7 @@ module API requires :credit_card_mask_number, type: String, desc: 'The last 4 digits of credit card number' requires :credit_card_type, type: String, desc: 'The credit card network name' end - put ":user_id/credit_card_validation", feature_category: :users do + put ":user_id/credit_card_validation", feature_category: :purchase do authenticated_as_admin! user = find_user(params[:user_id]) diff --git a/lib/api/validations/validators/limit.rb b/lib/api/validations/validators/limit.rb index e8f894849a5..7e11f1d77cc 100644 --- a/lib/api/validations/validators/limit.rb +++ b/lib/api/validations/validators/limit.rb @@ -7,7 +7,7 @@ module API def validate_param!(attr_name, params) value = params[attr_name] - return if value.size <= @option + return if value.nil? || value.size <= @option raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], diff --git a/lib/api/version.rb b/lib/api/version.rb index 86eb34ca589..bdce88ab827 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -9,7 +9,7 @@ module API before { authenticate! } - feature_category :not_owned + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned METADATA_QUERY = <<~EOF { diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index e90d88940a5..12dbf4792d6 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -12,7 +12,7 @@ module API params :common_wiki_page_params do optional :format, type: String, - values: Wiki::MARKUPS.values.map(&:to_s), + values: Wiki::VALID_USER_MARKUPS.keys.map(&:to_s), default: 'markdown', desc: 'Format of a wiki page. Available formats are markdown, rdoc, asciidoc and org' end @@ -48,7 +48,7 @@ module API optional :version, type: String, desc: 'The version hash of a wiki page' optional :render_html, type: Boolean, default: false, desc: 'Render content to HTML' end - get ':id/wikis/:slug' do + get ':id/wikis/:slug', urgency: :low do authorize! :read_wiki, container present wiki_page(params[:version]), with: Entities::WikiPage, render_html: params[:render_html] @@ -136,7 +136,7 @@ module API if result[:status] == :success status(201) - present OpenStruct.new(result[:result]), with: Entities::WikiAttachment + present result[:result], with: Entities::WikiAttachment else render_api_error!(result[:message], 400) end diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb deleted file mode 100644 index 4ef76b0aaf3..00000000000 --- a/lib/backup/artifacts.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Artifacts < Backup::Files - def initialize(progress) - super(progress, 'artifacts', JobArtifactUploader.root, excludes: ['tmp']) - end - - override :human_name - def human_name - _('artifacts') - end - end -end diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb deleted file mode 100644 index fbf932e3f6b..00000000000 --- a/lib/backup/builds.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Builds < Backup::Files - def initialize(progress) - super(progress, 'builds', Settings.gitlab_ci.builds_path) - end - - override :human_name - def human_name - _('builds') - end - end -end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index afc84a4b913..3cbe3cf7d88 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -25,7 +25,7 @@ module Backup end override :dump - def dump(db_file_name) + def dump(db_file_name, backup_id) FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe @@ -134,11 +134,6 @@ module Backup MSG end - override :human_name - def human_name - _('database') - end - protected def database diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 7fa07e40cee..55b10c008fb 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -9,19 +9,18 @@ module Backup DEFAULT_EXCLUDE = 'lost+found' - attr_reader :name, :excludes + attr_reader :excludes - def initialize(progress, name, app_files_dir, excludes: []) + def initialize(progress, app_files_dir, excludes: []) super(progress) - @name = name @app_files_dir = app_files_dir @excludes = [DEFAULT_EXCLUDE].concat(excludes) end # Copy files from public/files to backup/files override :dump - def dump(backup_tarball) + def dump(backup_tarball, backup_id) FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) @@ -55,7 +54,7 @@ module Backup override :restore def restore(backup_tarball) - backup_existing_files_dir + backup_existing_files_dir(backup_tarball) cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] status_list, output = run_pipeline!(cmd_list, in: backup_tarball) @@ -73,11 +72,13 @@ module Backup end end - def backup_existing_files_dir + def backup_existing_files_dir(backup_tarball) + name = File.basename(backup_tarball, '.tar.gz') + timestamped_files_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}.#{Time.now.to_i}") if File.exist?(app_files_realpath) # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory + # repositories.<timestamp> directory FileUtils.mkdir_p(timestamped_files_path, mode: 0700) files = Dir.glob(File.join(app_files_realpath, "*"), File::FNM_DOTMATCH) - [File.join(app_files_realpath, "."), File.join(app_files_realpath, "..")] begin diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index b688ff7f13b..93342e789e9 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -9,16 +9,14 @@ module Backup # @param [StringIO] progress IO interface to output progress # @param [Integer] max_parallelism max parallelism when running backups # @param [Integer] storage_parallelism max parallelism per storage (is affected by max_parallelism) - # @param [String] backup_id unique identifier for the backup def initialize(progress, max_parallelism: nil, storage_parallelism: nil, incremental: false, backup_id: nil) @progress = progress @max_parallelism = max_parallelism @storage_parallelism = storage_parallelism @incremental = incremental - @backup_id = backup_id end - def start(type, backup_repos_path) + def start(type, backup_repos_path, backup_id: nil) raise Error, 'already started' if started? command = case type @@ -37,7 +35,7 @@ module Backup args += ['-layout', 'pointer'] if type == :create args += ['-incremental'] if @incremental - args += ['-id', @backup_id] if @backup_id + args += ['-id', backup_id] if backup_id end end @@ -68,10 +66,6 @@ module Backup schedule_backup_job(repository, always_create: repo_type.project?) end - def parallel_enqueue? - false - end - private # Schedule a new backup job through a non-blocking JSON based pipe protocol @@ -104,6 +98,8 @@ module Backup end def bin_path + raise Error, 'gitaly-backup binary not found and gitaly_backup_path is not configured' unless Gitlab.config.backup.gitaly_backup_path.present? + File.absolute_path(Gitlab.config.backup.gitaly_backup_path) end end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb deleted file mode 100644 index 89ed27cfa13..00000000000 --- a/lib/backup/gitaly_rpc_backup.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -module Backup - # Backup and restores repositories using the gitaly RPC - class GitalyRpcBackup - def initialize(progress) - @progress = progress - end - - def start(type, backup_repos_path) - raise Error, 'already started' if @type - - @type = type - @backup_repos_path = backup_repos_path - case type - when :create - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) - when :restore - # no op - else - raise Error, "unknown backup type: #{type}" - end - end - - def finish! - @type = nil - end - - def enqueue(container, repository_type) - backup_restore = BackupRestore.new( - progress, - repository_type.repository_for(container), - @backup_repos_path - ) - - case @type - when :create - backup_restore.backup - when :restore - backup_restore.restore(always_create: repository_type.project?) - else - raise Error, 'not started' - end - end - - def parallel_enqueue? - true - end - - private - - attr_reader :progress - - class BackupRestore - attr_accessor :progress, :repository, :backup_repos_path - - def initialize(progress, repository, backup_repos_path) - @progress = progress - @repository = repository - @backup_repos_path = backup_repos_path - end - - def backup - progress.puts " * #{display_repo_path} ... " - - if repository.empty? - progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) - return - end - - FileUtils.mkdir_p(repository_backup_path) - - repository.bundle_to_disk(path_to_bundle) - repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] backing up #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - def restore(always_create: false) - progress.puts " * #{display_repo_path} ... " - - repository.remove rescue nil - - if File.exist?(path_to_bundle) - repository.create_from_bundle(path_to_bundle) - restore_custom_hooks - elsif always_create - repository.create_repository - end - - progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - - rescue StandardError => e - progress.puts "[Failed] restoring #{display_repo_path}".color(:red) - progress.puts "Error #{e}".color(:red) - end - - private - - def display_repo_path - "#{repository.full_path} (#{repository.disk_path})" - end - - def repository_backup_path - @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) - end - - def path_to_bundle - @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') - end - - def restore_custom_hooks - return unless File.exist?(custom_hooks_tar) - - repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) - end - - def custom_hooks_tar - File.join(repository_backup_path, "custom_hooks.tar") - end - end - end -end diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb deleted file mode 100644 index e92f235a2d7..00000000000 --- a/lib/backup/lfs.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Lfs < Backup::Files - def initialize(progress) - super(progress, 'lfs', Settings.lfs.storage_path) - end - - override :human_name - def human_name - _('lfs objects') - end - end -end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index cb5fd959bc9..403b2d9f16c 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -5,74 +5,43 @@ module Backup FILE_NAME_SUFFIX = '_gitlab_backup.tar' MANIFEST_NAME = 'backup_information.yml' + # pages used to deploy tmp files to this path + # if some of these files are still there, we don't need them in the backup + LEGACY_PAGES_TMP_PATH = '@pages.tmp' + TaskDefinition = Struct.new( + :enabled, # `true` if the task can be used. Treated as `true` when not specified. + :human_name, # Name of the task used for logging. :destination_path, # Where the task should put its backup file/dir. :destination_optional, # `true` if the destination might not exist on a successful backup. :cleanup_path, # Path to remove after a successful backup. Uses `destination_path` when not specified. :task, keyword_init: true - ) + ) do + def enabled? + enabled.nil? || enabled + end + end attr_reader :progress def initialize(progress, definitions: nil) @progress = progress - max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i - max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i - force = ENV['force'] == 'yes' - incremental = Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false) + @incremental = Feature.feature_flags_available? && + Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml) && + Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false) - @definitions = definitions || { - 'db' => TaskDefinition.new( - destination_path: 'db/database.sql.gz', - cleanup_path: 'db', - task: Database.new(progress, force: force) - ), - 'repositories' => TaskDefinition.new( - destination_path: 'repositories', - destination_optional: true, - task: Repositories.new(progress, - strategy: repository_backup_strategy(incremental), - max_concurrency: max_concurrency, - max_storage_concurrency: max_storage_concurrency) - ), - 'uploads' => TaskDefinition.new( - destination_path: 'uploads.tar.gz', - task: Uploads.new(progress) - ), - 'builds' => TaskDefinition.new( - destination_path: 'builds.tar.gz', - task: Builds.new(progress) - ), - 'artifacts' => TaskDefinition.new( - destination_path: 'artifacts.tar.gz', - task: Artifacts.new(progress) - ), - 'pages' => TaskDefinition.new( - destination_path: 'pages.tar.gz', - task: Pages.new(progress) - ), - 'lfs' => TaskDefinition.new( - destination_path: 'lfs.tar.gz', - task: Lfs.new(progress) - ), - 'terraform_state' => TaskDefinition.new( - destination_path: 'terraform_state.tar.gz', - task: TerraformState.new(progress) - ), - 'registry' => TaskDefinition.new( - destination_path: 'registry.tar.gz', - task: Registry.new(progress) - ), - 'packages' => TaskDefinition.new( - destination_path: 'packages.tar.gz', - task: Packages.new(progress) - ) - }.freeze + @definitions = definitions || build_definitions end def create + if incremental? + unpack + read_backup_information + verify_backup_version + end + @definitions.keys.each do |task_name| run_create_task(task_name) end @@ -88,34 +57,33 @@ module Backup remove_old end - progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need these files to restore a backup.\n" \ "Please back them up manually.".color(:red) - progress.puts "Backup task is done." + puts_time "Backup #{backup_id} is done." end def run_create_task(task_name) definition = @definitions[task_name] build_backup_information - puts_time "Dumping #{definition.task.human_name} ... ".color(:blue) - unless definition.task.enabled - puts_time "[DISABLED]".color(:cyan) + unless definition.enabled? + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan) return end if skipped?(task_name) - puts_time "[SKIPPED]".color(:cyan) + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "[SKIPPED]".color(:cyan) return end - definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path)) - - puts_time "done".color(:green) + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), backup_id) + puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "done".color(:green) rescue Backup::DatabaseBackupError, Backup::FileBackupError => e - progress.puts "#{e.message}" + puts_time "Dumping #{definition.human_name} failed: #{e.message}".color(:red) end def restore @@ -136,21 +104,21 @@ module Backup remove_tmp - puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ - "and are not included in this backup. You will need to restore these files manually.".color(:red) - puts "Restore task is done." + puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ + "and are not included in this backup. You will need to restore these files manually.".color(:red) + puts_time "Restore task is done." end def run_restore_task(task_name) definition = @definitions[task_name] - puts_time "Restoring #{definition.task.human_name} ... ".color(:blue) - - unless definition.task.enabled - puts_time "[DISABLED]".color(:cyan) + unless definition.enabled? + puts_time "Restoring #{definition.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan) return end + puts_time "Restoring #{definition.human_name} ... ".color(:blue) + warning = definition.task.pre_restore_warning if warning.present? puts_time warning.color(:red) @@ -159,7 +127,7 @@ module Backup definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path)) - puts_time "done".color(:green) + puts_time "Restoring #{definition.human_name} ... ".color(:blue) + "done".color(:green) warning = definition.task.post_restore_warning if warning.present? @@ -174,6 +142,86 @@ module Backup private + def build_definitions + { + 'db' => TaskDefinition.new( + human_name: _('database'), + destination_path: 'db/database.sql.gz', + cleanup_path: 'db', + task: build_db_task + ), + 'repositories' => TaskDefinition.new( + human_name: _('repositories'), + destination_path: 'repositories', + destination_optional: true, + task: build_repositories_task + ), + 'uploads' => TaskDefinition.new( + human_name: _('uploads'), + destination_path: 'uploads.tar.gz', + task: build_files_task(File.join(Gitlab.config.uploads.storage_path, 'uploads'), excludes: ['tmp']) + ), + 'builds' => TaskDefinition.new( + human_name: _('builds'), + destination_path: 'builds.tar.gz', + task: build_files_task(Settings.gitlab_ci.builds_path) + ), + 'artifacts' => TaskDefinition.new( + human_name: _('artifacts'), + destination_path: 'artifacts.tar.gz', + task: build_files_task(JobArtifactUploader.root, excludes: ['tmp']) + ), + 'pages' => TaskDefinition.new( + human_name: _('pages'), + destination_path: 'pages.tar.gz', + task: build_files_task(Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) + ), + 'lfs' => TaskDefinition.new( + human_name: _('lfs objects'), + destination_path: 'lfs.tar.gz', + task: build_files_task(Settings.lfs.storage_path) + ), + 'terraform_state' => TaskDefinition.new( + human_name: _('terraform states'), + destination_path: 'terraform_state.tar.gz', + task: build_files_task(Settings.terraform_state.storage_path, excludes: ['tmp']) + ), + 'registry' => TaskDefinition.new( + enabled: Gitlab.config.registry.enabled, + human_name: _('container registry images'), + destination_path: 'registry.tar.gz', + task: build_files_task(Settings.registry.path) + ), + 'packages' => TaskDefinition.new( + human_name: _('packages'), + destination_path: 'packages.tar.gz', + task: build_files_task(Settings.packages.storage_path, excludes: ['tmp']) + ) + }.freeze + end + + def build_db_task + force = Gitlab::Utils.to_boolean(ENV['force'], default: false) + + Database.new(progress, force: force) + end + + def build_repositories_task + max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence + max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence + strategy = Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) + + Repositories.new(progress, strategy: strategy) + end + + def build_files_task(app_files_dir, excludes: []) + Files.new(progress, app_files_dir, excludes: excludes) + end + + def incremental? + @incremental + end + def read_backup_information @backup_information ||= YAML.load_file(File.join(backup_path, MANIFEST_NAME)) end @@ -209,103 +257,104 @@ module Backup def pack Dir.chdir(backup_path) do # create archive - progress.print "Creating backup archive: #{tar_file} ... " + puts_time "Creating backup archive: #{tar_file} ... ".color(:blue) # Set file permissions on open to prevent chmod races. tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) - progress.puts "done".color(:green) + puts_time "Creating backup archive: #{tar_file} ... ".color(:blue) + 'done'.color(:green) else - puts "creating archive #{tar_file} failed".color(:red) + puts_time "Creating archive #{tar_file} failed".color(:red) raise Backup::Error, 'Backup failed' end end end def upload - progress.print "Uploading backup archive to remote storage #{remote_directory} ... " - connection_settings = Gitlab.config.backup.upload.connection - if connection_settings.blank? - progress.puts "skipped".color(:yellow) + if connection_settings.blank? || skipped?('remote') + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "[SKIPPED]".color(:cyan) return end + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + directory = connect_to_remote_directory upload = directory.files.create(create_attributes) if upload if upload.respond_to?(:encryption) && upload.encryption - progress.puts "done (encrypted with #{upload.encryption})".color(:green) + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "done (encrypted with #{upload.encryption})".color(:green) else - progress.puts "done".color(:green) + puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "done".color(:green) end else - puts "uploading backup to #{remote_directory} failed".color(:red) + puts_time "Uploading backup to #{remote_directory} failed".color(:red) raise Backup::Error, 'Backup failed' end end def cleanup - progress.print "Deleting tmp directories ... " + puts_time "Deleting tar staging files ... ".color(:blue) remove_backup_path(MANIFEST_NAME) @definitions.each do |_, definition| remove_backup_path(definition.cleanup_path || definition.destination_path) end + + puts_time "Deleting tar staging files ... ".color(:blue) + 'done'.color(:green) end def remove_backup_path(path) - return unless File.exist?(File.join(backup_path, path)) + absolute_path = File.join(backup_path, path) + return unless File.exist?(absolute_path) - FileUtils.rm_rf(File.join(backup_path, path)) - progress.puts "done".color(:green) + puts_time "Cleaning up #{absolute_path}" + FileUtils.rm_rf(absolute_path) end def remove_tmp # delete tmp inside backups - progress.print "Deleting backups/tmp ... " + puts_time "Deleting backups/tmp ... ".color(:blue) - if FileUtils.rm_rf(File.join(backup_path, "tmp")) - progress.puts "done".color(:green) - else - puts "deleting backups/tmp failed".color(:red) - end + FileUtils.rm_rf(File.join(backup_path, "tmp")) + puts_time "Deleting backups/tmp ... ".color(:blue) + "done".color(:green) end def remove_old # delete backups - progress.print "Deleting old backups ... " keep_time = Gitlab.config.backup.keep_time.to_i - if keep_time > 0 - removed = 0 - - Dir.chdir(backup_path) do - backup_file_list.each do |file| - # For backward compatibility, there are 3 names the backups can have: - # - 1495527122_gitlab_backup.tar - # - 1495527068_2017_05_23_gitlab_backup.tar - # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar - matched = backup_file?(file) - next unless matched - - timestamp = matched[1].to_i - - if Time.at(timestamp) < (Time.now - keep_time) - begin - FileUtils.rm(file) - removed += 1 - rescue StandardError => e - progress.puts "Deleting #{file} failed: #{e.message}".color(:red) - end + if keep_time <= 0 + puts_time "Deleting old backups ... ".color(:blue) + "[SKIPPED]".color(:cyan) + return + end + + puts_time "Deleting old backups ... ".color(:blue) + removed = 0 + + Dir.chdir(backup_path) do + backup_file_list.each do |file| + # For backward compatibility, there are 3 names the backups can have: + # - 1495527122_gitlab_backup.tar + # - 1495527068_2017_05_23_gitlab_backup.tar + # - 1495527097_2017_05_23_9.3.0-pre_gitlab_backup.tar + matched = backup_file?(file) + next unless matched + + timestamp = matched[1].to_i + + if Time.at(timestamp) < (Time.now - keep_time) + begin + FileUtils.rm(file) + removed += 1 + rescue StandardError => e + puts_time "Deleting #{file} failed: #{e.message}".color(:red) end end end - - progress.puts "done. (#{removed} removed)".color(:green) - else - progress.puts "skipping".color(:yellow) end + + puts_time "Deleting old backups ... ".color(:blue) + "done. (#{removed} removed)".color(:green) end def verify_backup_version @@ -327,7 +376,7 @@ module Backup def unpack if ENV['BACKUP'].blank? && non_tarred_backup? - progress.puts "Non tarred backup found in #{backup_path}, using that" + puts_time "Non tarred backup found in #{backup_path}, using that" return false end @@ -335,15 +384,22 @@ module Backup Dir.chdir(backup_path) do # check for existing backups in the backup dir if backup_file_list.empty? - progress.puts "No backups found in #{backup_path}" - progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + puts_time "No backups found in #{backup_path}" + puts_time "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 elsif backup_file_list.many? && ENV["BACKUP"].nil? - progress.puts 'Found more than one backup:' + puts_time 'Found more than one backup:' # print list of available backups - progress.puts " " + available_timestamps.join("\n ") - progress.puts 'Please specify which one you want to restore:' - progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + puts_time " " + available_timestamps.join("\n ") + + if incremental? + puts_time 'Please specify which one you want to create an incremental backup for:' + puts_time 'rake gitlab:backup:create INCREMENTAL=true BACKUP=timestamp_of_backup' + else + puts_time 'Please specify which one you want to restore:' + puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' + end + exit 1 end @@ -354,16 +410,16 @@ module Backup end unless File.exist?(tar_file) - progress.puts "The backup file #{tar_file} does not exist!" + puts_time "The backup file #{tar_file} does not exist!" exit 1 end - progress.print 'Unpacking backup ... ' + puts_time 'Unpacking backup ... '.color(:blue) if Kernel.system(*%W(tar -xf #{tar_file})) - progress.puts 'done'.color(:green) + puts_time 'Unpacking backup ... '.color(:blue) + 'done'.color(:green) else - progress.puts 'unpacking backup failed'.color(:red) + puts_time 'Unpacking backup failed'.color(:red) exit 1 end end @@ -375,11 +431,12 @@ module Backup end def skipped?(item) - backup_information[:skipped] && backup_information[:skipped].include?(item) + ENV.fetch('SKIP', '').include?(item) || + backup_information[:skipped] && backup_information[:skipped].include?(item) end def enabled_task?(task_name) - @definitions[task_name].task.enabled + @definitions[task_name].enabled? end def backup_file?(file) @@ -441,11 +498,15 @@ module Backup end def tar_file - @tar_file ||= if ENV['BACKUP'].present? - File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX - else - "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}#{FILE_NAME_SUFFIX}" - end + @tar_file ||= "#{backup_id}#{FILE_NAME_SUFFIX}" + end + + def backup_id + @backup_id ||= if ENV['BACKUP'].present? + File.basename(ENV['BACKUP']) + else + "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}" + end end def create_attributes @@ -481,16 +542,6 @@ module Backup Gitlab.config.backup.upload.connection&.provider&.downcase == 'google' end - def repository_backup_strategy(incremental) - if !Feature.feature_flags_available? || Feature.enabled?(:gitaly_backup, default_enabled: :yaml) - max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence - max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence - Backup::GitalyBackup.new(progress, incremental: incremental, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - else - Backup::GitalyRpcBackup.new(progress) - end - end - def puts_time(msg) progress.puts "#{Time.now} -- #{msg}" Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") diff --git a/lib/backup/packages.rb b/lib/backup/packages.rb deleted file mode 100644 index 9384e007162..00000000000 --- a/lib/backup/packages.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Packages < Backup::Files - def initialize(progress) - super(progress, 'packages', Settings.packages.storage_path, excludes: ['tmp']) - end - - override :human_name - def human_name - _('packages') - end - end -end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb deleted file mode 100644 index ebed6820724..00000000000 --- a/lib/backup/pages.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Pages < Backup::Files - # pages used to deploy tmp files to this path - # if some of these files are still there, we don't need them in the backup - LEGACY_PAGES_TMP_PATH = '@pages.tmp' - - def initialize(progress) - super(progress, 'pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH]) - end - - override :human_name - def human_name - _('pages') - end - end -end diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb deleted file mode 100644 index 68ea635034d..00000000000 --- a/lib/backup/registry.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Registry < Backup::Files - def initialize(progress) - super(progress, 'registry', Settings.registry.path) - end - - override :human_name - def human_name - _('container registry images') - end - - override :enabled - def enabled - Gitlab.config.registry.enabled - end - end -end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 3633ebd661e..11bed84e356 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -6,50 +6,17 @@ module Backup class Repositories < Task extend ::Gitlab::Utils::Override - def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1) + def initialize(progress, strategy:) super(progress) @strategy = strategy - @max_concurrency = max_concurrency - @max_storage_concurrency = max_storage_concurrency end override :dump - def dump(path) - strategy.start(:create, path) - - # gitaly-backup is designed to handle concurrency on its own. So we want - # to avoid entering the buggy concurrency code here when gitaly-backup - # is enabled. - if (max_concurrency <= 1 && max_storage_concurrency <= 1) || !strategy.parallel_enqueue? - return enqueue_consecutive - end - - if max_concurrency < 1 || max_storage_concurrency < 1 - puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) - exit 1 - end - - check_valid_storages! - - semaphore = Concurrent::Semaphore.new(max_concurrency) - errors = Queue.new - - threads = Gitlab.config.repositories.storages.keys.map do |storage| - Thread.new do - Rails.application.executor.wrap do - enqueue_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) - rescue StandardError => e - errors << e - end - end - end - - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end + def dump(path, backup_id) + strategy.start(:create, path, backup_id: backup_id) + enqueue_consecutive - raise errors.pop unless errors.empty? ensure strategy.finish! end @@ -66,26 +33,9 @@ module Backup restore_object_pools end - override :human_name - def human_name - _('repositories') - end - private - attr_reader :strategy, :max_concurrency, :max_storage_concurrency - - def check_valid_storages! - repository_storage_klasses.each do |klass| - if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? - raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}" - end - end - end - - def repository_storage_klasses - [ProjectRepository, SnippetRepository] - end + attr_reader :strategy def enqueue_consecutive enqueue_consecutive_projects @@ -102,50 +52,6 @@ module Backup Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } end - def enqueue_storage(storage, semaphore, max_storage_concurrency:) - errors = Queue.new - queue = InterlockSizedQueue.new(1) - - threads = Array.new(max_storage_concurrency) do - Thread.new do - Rails.application.executor.wrap do - while container = queue.pop - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - semaphore.acquire - end - - begin - enqueue_container(container) - rescue StandardError => e - errors << e - break - ensure - semaphore.release - end - end - end - end - end - - enqueue_records_for_storage(storage, queue, errors) - - raise errors.pop unless errors.empty? - ensure - queue.close - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - end - - def enqueue_container(container) - case container - when Project - enqueue_project(container) - when Snippet - enqueue_snippet(container) - end - end - def enqueue_project(project) strategy.enqueue(project, Gitlab::GlRepository::PROJECT) strategy.enqueue(project, Gitlab::GlRepository::WIKI) @@ -156,32 +62,10 @@ module Backup strategy.enqueue(snippet, Gitlab::GlRepository::SNIPPET) end - def enqueue_records_for_storage(storage, queue, errors) - records_to_enqueue(storage).each do |relation| - relation.find_each(batch_size: 100) do |project| - break unless errors.empty? - - queue.push(project) - end - end - end - - def records_to_enqueue(storage) - [projects_in_storage(storage), snippets_in_storage(storage)] - end - - def projects_in_storage(storage) - project_relation.id_in(ProjectRepository.for_repository_storage(storage).select(:project_id)) - end - def project_relation Project.includes(:route, :group, namespace: :owner) end - def snippets_in_storage(storage) - Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id)) - end - def restore_object_pools PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." @@ -216,24 +100,6 @@ module Backup Snippet.id_in(invalid_snippets).delete_all end - - class InterlockSizedQueue < SizedQueue - extend ::Gitlab::Utils::Override - - override :pop - def pop(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - - override :push - def push(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - end end end diff --git a/lib/backup/task.rb b/lib/backup/task.rb index 15cd2aa64d3..776c19130a7 100644 --- a/lib/backup/task.rb +++ b/lib/backup/task.rb @@ -6,13 +6,11 @@ module Backup @progress = progress end - # human readable task name used for logging - def human_name - raise NotImplementedError - end - # dump task backup to `path` - def dump(path) + # + # @param [String] path fully qualified backup task destination + # @param [String] backup_id unique identifier for the backup + def dump(path, backup_id) raise NotImplementedError end @@ -29,11 +27,6 @@ module Backup def post_restore_warning end - # returns `true` when the task should be used - def enabled - true - end - private attr_reader :progress diff --git a/lib/backup/terraform_state.rb b/lib/backup/terraform_state.rb deleted file mode 100644 index 05f61d248be..00000000000 --- a/lib/backup/terraform_state.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class TerraformState < Backup::Files - def initialize(progress) - super(progress, 'terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp']) - end - - override :human_name - def human_name - _('terraform states') - end - end -end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb deleted file mode 100644 index 700f2af4415..00000000000 --- a/lib/backup/uploads.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Backup - class Uploads < Backup::Files - def initialize(progress) - super(progress, 'uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp']) - end - - override :human_name - def human_name - _('uploads') - end - end -end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 4e350a59fa0..3b00d1a9824 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -39,6 +39,9 @@ module Banzai allowlist[:attributes][:all].delete('name') allowlist[:attributes]['a'].push('name') + allowlist[:attributes]['img'].push('data-diagram') + allowlist[:attributes]['img'].push('data-diagram-src') + # Allow any protocol in `a` elements # and then remove links with unsafe protocols allowlist[:protocols].delete('a') diff --git a/lib/banzai/filter/custom_emoji_filter.rb b/lib/banzai/filter/custom_emoji_filter.rb index a5f1a22c483..ae95c7f66b6 100644 --- a/lib/banzai/filter/custom_emoji_filter.rb +++ b/lib/banzai/filter/custom_emoji_filter.rb @@ -8,8 +8,7 @@ module Banzai IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call - return doc unless context[:project] - return doc unless Feature.enabled?(:custom_emoji, context[:project]) + return doc unless resource_parent doc.xpath('descendant-or-self::text()').each do |node| content = node.to_html @@ -50,12 +49,12 @@ module Banzai def has_custom_emoji? strong_memoize(:has_custom_emoji) do - namespace&.custom_emoji&.any? + CustomEmoji.for_resource(resource_parent).any? end end - def namespace - context[:project].namespace.root_ancestor + def resource_parent + context[:project] || context[:group] end def custom_emoji_candidates @@ -63,7 +62,8 @@ module Banzai end def all_custom_emoji - @all_custom_emoji ||= namespace.custom_emoji.by_name(custom_emoji_candidates).index_by(&:name) + @all_custom_emoji ||= + CustomEmoji.for_resource(resource_parent).by_name(custom_emoji_candidates).index_by(&:name) end end end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index 44acc7805b4..60881b5f511 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -27,6 +27,13 @@ module Banzai # make sure the original non-proxied src carries over to the link link['data-canonical-src'] = img['data-canonical-src'] if img['data-canonical-src'] + if img['data-diagram'] && img['data-diagram-src'] + link['data-diagram'] = img['data-diagram'] + link['data-diagram-src'] = img['data-diagram-src'] + img.remove_attribute('data-diagram') + img.remove_attribute('data-diagram-src') + end + link.children = if link_replaces_image img['alt'] || img['data-src'] || img['src'] else diff --git a/lib/banzai/filter/kroki_filter.rb b/lib/banzai/filter/kroki_filter.rb index 9aa2afce5a8..845c7f2bc0a 100644 --- a/lib/banzai/filter/kroki_filter.rb +++ b/lib/banzai/filter/kroki_filter.rb @@ -25,11 +25,19 @@ module Banzai diagram_type = node.parent['lang'] diagram_src = node.content image_src = create_image_src(diagram_type, diagram_format, diagram_src) - lazy_load = diagram_src.length > MAX_CHARACTER_LIMIT - other_attrs = lazy_load ? "hidden" : "" + img_tag = Nokogiri::HTML::DocumentFragment.parse(%(<img src="#{image_src}" />)) + img_tag = img_tag.children.first - img_tag = Nokogiri::HTML::DocumentFragment.parse(%(<img class="js-render-kroki" src="#{image_src}" #{other_attrs} />)) - node.parent.replace(img_tag) + unless img_tag.nil? + lazy_load = diagram_src.length > MAX_CHARACTER_LIMIT + img_tag.set_attribute('hidden', '') if lazy_load + img_tag.set_attribute('class', 'js-render-kroki') + + img_tag.set_attribute('data-diagram', node.parent['lang']) + img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(node.content)}") + + node.parent.replace(img_tag) + end end doc diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 68a99702d6f..cbcd547120d 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -15,8 +15,14 @@ module Banzai doc.xpath(lang_tag).each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) - node.parent.replace(img_tag) + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})).css('img').first + + unless img_tag.nil? + img_tag.set_attribute('data-diagram', 'plantuml') + img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(node.content)}") + + node.parent.replace(img_tag) + end end doc diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index 408e6dc685d..f5cf1833304 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -180,7 +180,7 @@ module Banzai parts.pop if uri_type(request_path) != :tree - path.sub!(%r{\A\./}, '') + path.delete_prefix!('./') while path.start_with?('../') parts.pop diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index df8151b3296..5e7c2f64c92 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -15,11 +15,11 @@ module Banzai # Must always be before the SanitizationFilter to prevent XSS attacks Filter::SpacedLinkFilter, Filter::SanitizationFilter, + Filter::KrokiFilter, Filter::AssetProxyFilter, Filter::SyntaxHighlightFilter, Filter::MathFilter, Filter::ColorFilter, - Filter::KrokiFilter, Filter::MermaidFilter, Filter::VideoLinkFilter, Filter::AudioLinkFilter, diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index aa9221cceee..0f4def3b17a 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -30,6 +30,8 @@ module BulkImports pipeline_class: self.class.name, message: "Entity #{entity.status_name}" ) + + context.portable.try(:after_import) end private diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index bc27220391d..97a423b6ea9 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -47,7 +47,7 @@ module BulkImports end def project_entities_pipeline - if project_pipeline_available? && ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml) + if project_pipeline_available? && feature_flag_enabled? { project_entities: { pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline, @@ -62,6 +62,18 @@ module BulkImports def project_pipeline_available? @bulk_import.source_version_info >= BulkImport.min_gl_version_for_project_migration end + + def feature_flag_enabled? + destination_namespace = @bulk_import_entity.destination_namespace + + if destination_namespace.present? + root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor + + ::Feature.enabled?(:bulk_import_projects, root_ancestor, default_enabled: :yaml) + else + ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml) + end + end end end end diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index 9c19e9ea60b..6cf394c5df0 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -2,10 +2,13 @@ module BulkImports class Stage - def initialize(bulk_import) - raise(ArgumentError, 'Expected an argument of type ::BulkImport') unless bulk_import.is_a?(::BulkImport) + def initialize(bulk_import_entity) + unless bulk_import_entity.is_a?(::BulkImports::Entity) + raise(ArgumentError, 'Expected an argument of type ::BulkImports::Entity') + end - @bulk_import = bulk_import + @bulk_import_entity = bulk_import_entity + @bulk_import = bulk_import_entity.bulk_import end def pipelines diff --git a/lib/container_registry/base_client.rb b/lib/container_registry/base_client.rb index 22d4510fe71..bb9422ae048 100644 --- a/lib/container_registry/base_client.rb +++ b/lib/container_registry/base_client.rb @@ -37,14 +37,24 @@ module ContainerRegistry class << self private - def with_dummy_client(return_value_if_disabled: nil) + def with_dummy_client(return_value_if_disabled: nil, token_config: { type: :full_access_token, path: nil }) registry_config = Gitlab.config.registry unless registry_config.enabled && registry_config.api_url.present? return return_value_if_disabled end - token = Auth::ContainerRegistryAuthenticationService.access_token([], []) - yield new(registry_config.api_url, token: token) + yield new(registry_config.api_url, token: token_from(token_config)) + end + + def token_from(config) + case config[:type] + when :full_access_token + Auth::ContainerRegistryAuthenticationService.access_token([], []) + when :nested_repositories_token + return unless config[:path] + + Auth::ContainerRegistryAuthenticationService.pull_nested_repositories_access_token(config[:path]) + end end end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index 3cd7003d1f8..0cd8f8509f6 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -5,10 +5,12 @@ module ContainerRegistry include Gitlab::Utils::StrongMemoize JSON_TYPE = 'application/json' + CANCEL_RESPONSE_STATUS_HEADER = 'status' IMPORT_RESPONSES = { 200 => :already_imported, 202 => :ok, + 400 => :bad_request, 401 => :unauthorized, 404 => :not_found, 409 => :already_being_imported, @@ -25,6 +27,12 @@ module ContainerRegistry end end + def self.deduplicated_size(path) + with_dummy_client(token_config: { type: :nested_repositories_token, path: path }) do |client| + client.repository_details(path, sizing: :self_with_descendants)['size_bytes'] + end + end + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#compliance-check def supports_gitlab_api? strong_memoize(:supports_gitlab_api) do @@ -50,18 +58,38 @@ module ContainerRegistry IMPORT_RESPONSES.fetch(response.status, :error) end + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#cancel-repository-import + def cancel_repository_import(path, force: false) + response = with_import_token_faraday do |faraday_client| + faraday_client.delete(import_url_for(path)) do |req| + req.params['force'] = true if force + end + end + + status = IMPORT_RESPONSES.fetch(response.status, :error) + actual_state = response.body[CANCEL_RESPONSE_STATUS_HEADER] + + { status: status, migration_state: actual_state } + end + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-import-status def import_status(path) with_import_token_faraday do |faraday_client| - body_hash = response_body(faraday_client.get(import_url_for(path))) - body_hash['status'] || 'error' + response = faraday_client.get(import_url_for(path)) + + # Temporary solution for https://gitlab.com/gitlab-org/gitlab/-/issues/356085#solutions + # this will trigger a `retry_pre_import` + break 'pre_import_failed' unless response.success? + + body_hash = response_body(response) + body_hash&.fetch('status') || 'error' end end - def repository_details(path, with_size: false) + def repository_details(path, sizing: nil) with_token_faraday do |faraday_client| req = faraday_client.get("/gitlab/v1/repositories/#{path}/") do |req| - req.params['size'] = 'self' if with_size + req.params['size'] = sizing if sizing end break {} unless req.success? diff --git a/lib/container_registry/migration.rb b/lib/container_registry/migration.rb index b03c94e5ebf..005ef880034 100644 --- a/lib/container_registry/migration.rb +++ b/lib/container_registry/migration.rb @@ -2,6 +2,17 @@ module ContainerRegistry module Migration + # Some container repositories do not have a plan associated with them, they will be imported with + # the free tiers + FREE_TIERS = ['free', 'early_adopter', nil].freeze + PREMIUM_TIERS = %w[premium bronze silver premium_trial].freeze + ULTIMATE_TIERS = %w[ultimate gold ultimate_trial].freeze + PLAN_GROUPS = { + 'free' => FREE_TIERS, + 'premium' => PREMIUM_TIERS, + 'ultimate' => ULTIMATE_TIERS + }.freeze + class << self delegate :container_registry_import_max_tags_count, to: ::Gitlab::CurrentSettings delegate :container_registry_import_max_retries, to: ::Gitlab::CurrentSettings @@ -28,9 +39,9 @@ module ContainerRegistry def self.enqueue_waiting_time return 0 if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_fast) - return 6.hours if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_slow) + return 165.minutes if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_slow) - 1.hour + 45.minutes end def self.capacity @@ -46,8 +57,12 @@ module ContainerRegistry 0 end - def self.target_plan - Plan.find_by_name(target_plan_name) + def self.target_plans + PLAN_GROUPS[target_plan_name] + end + + def self.all_plans? + Feature.enabled?(:container_registry_migration_phase2_all_plans) end end end diff --git a/lib/error_tracking/sentry_client.rb b/lib/error_tracking/sentry_client.rb index 8d1bcec032d..6a341ddbe86 100644 --- a/lib/error_tracking/sentry_client.rb +++ b/lib/error_tracking/sentry_client.rb @@ -59,10 +59,8 @@ module ErrorTracking end end - def http_request - response = handle_request_exceptions do - yield - end + def http_request(&block) + response = handle_request_exceptions(&block) handle_response(response) end @@ -86,9 +84,7 @@ module ErrorTracking end def handle_response(response) - unless response.code.between?(200, 204) - raise_error "Sentry response status code: #{response.code}" - end + raise_error "Sentry response status code: #{response.code}" unless response.code.between?(200, 204) { body: response.parsed_response, headers: response.headers } end diff --git a/lib/error_tracking/sentry_client/event.rb b/lib/error_tracking/sentry_client/event.rb index 93449344d6c..5343eb7df57 100644 --- a/lib/error_tracking/sentry_client/event.rb +++ b/lib/error_tracking/sentry_client/event.rb @@ -15,14 +15,14 @@ module ErrorTracking stack_trace = parse_stack_trace(event) Gitlab::ErrorTracking::ErrorEvent.new( - issue_id: event.dig('groupID'), - date_received: event.dig('dateReceived'), + issue_id: event['groupID'], + date_received: event['dateReceived'], stack_trace_entries: stack_trace ) end def parse_stack_trace(event) - exception_entry = event.dig('entries')&.detect { |h| h['type'] == 'exception' } + exception_entry = event['entries']&.detect { |h| h['type'] == 'exception' } return [] unless exception_entry exception_values = exception_entry.dig('data', 'values') diff --git a/lib/error_tracking/sentry_client/issue.rb b/lib/error_tracking/sentry_client/issue.rb index 65da072ef8d..d0e6bd783f3 100644 --- a/lib/error_tracking/sentry_client/issue.rb +++ b/lib/error_tracking/sentry_client/issue.rb @@ -54,9 +54,7 @@ module ErrorTracking end def list_issue_sentry_query(issue_status:, limit:, sort: nil, search_term: '', cursor: nil) - unless SENTRY_API_SORT_VALUE_MAP.key?(sort) - raise BadRequestError, 'Invalid value for sort param' - end + raise BadRequestError, 'Invalid value for sort param' unless SENTRY_API_SORT_VALUE_MAP.key?(sort) { query: "is:#{issue_status} #{search_term}".strip, @@ -69,7 +67,8 @@ module ErrorTracking def validate_size(issues) return if Gitlab::Utils::DeepSize.new(issues).valid? - raise ResponseInvalidSizeError, "Sentry API response is too big. Limit is #{Gitlab::Utils::DeepSize.human_default_max_size}." + message = "Sentry API response is too big. Limit is #{Gitlab::Utils::DeepSize.human_default_max_size}." + raise ResponseInvalidSizeError, message end def get_issue(issue_id:) @@ -117,7 +116,7 @@ module ErrorTracking end def map_to_errors(issues) - issues.map(&method(:map_to_error)) + issues.map { map_to_error(_1) } end def map_to_error(issue) @@ -142,7 +141,7 @@ module ErrorTracking end def map_to_detailed_error(issue) - Gitlab::ErrorTracking::DetailedError.new({ + Gitlab::ErrorTracking::DetailedError.new( id: issue.fetch('id'), first_seen: issue.fetch('firstSeen', nil), last_seen: issue.fetch('lastSeen', nil), @@ -169,7 +168,7 @@ module ErrorTracking last_release_short_version: issue.dig('lastRelease', 'shortVersion'), last_release_version: issue.dig('lastRelease', 'version'), integrated: false - }) + ) end def extract_tags(issue) diff --git a/lib/error_tracking/sentry_client/pagination_parser.rb b/lib/error_tracking/sentry_client/pagination_parser.rb index 362a5d098f7..c6a42a6def2 100644 --- a/lib/error_tracking/sentry_client/pagination_parser.rb +++ b/lib/error_tracking/sentry_client/pagination_parser.rb @@ -3,7 +3,7 @@ module ErrorTracking class SentryClient module PaginationParser - PATTERN = /rel=\"(?<direction>\w+)\";\sresults=\"(?<results>\w+)\";\scursor=\"(?<cursor>.+)\"/.freeze + PATTERN = /rel="(?<direction>\w+)";\sresults="(?<results>\w+)";\scursor="(?<cursor>.+)"/.freeze def self.parse(headers) links = headers['link'].to_s.split(',') diff --git a/lib/error_tracking/sentry_client/projects.rb b/lib/error_tracking/sentry_client/projects.rb index 9b8daa226b0..a06b44cf29d 100644 --- a/lib/error_tracking/sentry_client/projects.rb +++ b/lib/error_tracking/sentry_client/projects.rb @@ -18,7 +18,7 @@ module ErrorTracking end def map_to_projects(projects) - projects.map(&method(:map_to_project)) + projects.map { map_to_project(_1) } end def map_to_project(project) @@ -28,7 +28,7 @@ module ErrorTracking id: project.fetch('id', nil), name: project.fetch('name'), slug: project.fetch('slug'), - status: project.dig('status'), + status: project['status'], organization_name: organization.fetch('name'), organization_id: organization.fetch('id', nil), organization_slug: organization.fetch('slug') diff --git a/lib/error_tracking/sentry_client/repo.rb b/lib/error_tracking/sentry_client/repo.rb index 3baa7e69be6..4333ca9b3d9 100644 --- a/lib/error_tracking/sentry_client/repo.rb +++ b/lib/error_tracking/sentry_client/repo.rb @@ -23,7 +23,7 @@ module ErrorTracking end def map_to_repos(repos) - repos.map(&method(:map_to_repo)) + repos.map { map_to_repo(_1) } end def map_to_repo(repo) diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 915ab355508..8833207dd1d 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +# rubocop: disable CodeReuse/ActiveRecord class EventFilter include Gitlab::Utils::StrongMemoize @@ -24,7 +25,6 @@ class EventFilter filter == key.to_s end - # rubocop: disable CodeReuse/ActiveRecord def apply_filter(events) case filter when PUSH @@ -34,9 +34,9 @@ class EventFilter when COMMENTS events.commented_action when TEAM - events.where(action: [:joined, :left, :expired]) + events.where(action: Event::TEAM_ACTIONS) when ISSUE - events.where(action: [:created, :updated, :closed, :reopened], target_type: 'Issue') + events.where(action: Event::ISSUE_ACTIONS, target_type: 'Issue') when WIKI wiki_events(events) when DESIGNS @@ -45,10 +45,157 @@ class EventFilter events end end - # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable Metrics/CyclomaticComplexity + # This method build specialized in-operator optimized queries based on different + # filter parameters. All queries will benefit from the index covering the following columns: + # author_id target_type action id + # + # More context: https://docs.gitlab.com/ee/development/database/efficient_in_operator_queries.html#the-inoperatoroptimization-module + def in_operator_query_builder_params(user_ids) + case filter + when ALL + in_operator_params(array_scope_ids: user_ids) + when PUSH + # Here we need to add an order hint column to force the correct index usage. + # Without the order hint, the following conditions will use the `index_events_on_author_id_and_id` + # index which is not as efficient as the `index_events_for_followed_users` index. + # > target_type IS NULL AND action = 5 AND author_id = X ORDER BY id DESC + # + # The order hint adds an extra order by column which doesn't affect the result but forces the planner + # to use the correct index: + # > target_type IS NULL AND action = 5 AND author_id = X ORDER BY target_type DESC, id DESC + in_operator_params( + array_scope_ids: user_ids, + scope: Event.where(target_type: nil).pushed_action, + order_hint_column: :target_type + ) + when MERGED + in_operator_params( + array_scope_ids: user_ids, + scope: Event.where(target_type: MergeRequest.to_s).merged_action + ) + when COMMENTS + in_operator_params( + array_scope_ids: user_ids, + scope: Event.commented_action, + in_column: :target_type, + in_values: [Note, *Note.descendants].map(&:name) # To make the query efficient we need to list all Note classes + ) + when TEAM + in_operator_params( + array_scope_ids: user_ids, + scope: Event.where(target_type: nil), + order_hint_column: :target_type, + in_column: :action, + in_values: Event.actions.values_at(*Event::TEAM_ACTIONS) + ) + when ISSUE + in_operator_params( + array_scope_ids: user_ids, + scope: Event.where(target_type: Issue.name), + in_column: :action, + in_values: Event.actions.values_at(*Event::ISSUE_ACTIONS) + ) + when WIKI + in_operator_params( + array_scope_ids: user_ids, + scope: Event.for_wiki_page, + in_column: :action, + in_values: Event.actions.values_at(*Event::WIKI_ACTIONS) + ) + when DESIGNS + in_operator_params( + array_scope_ids: user_ids, + scope: Event.for_design, + in_column: :action, + in_values: Event.actions.values_at(*Event::DESIGN_ACTIONS) + ) + else + in_operator_params(array_scope_ids: user_ids) + end + end + # rubocop: enable Metrics/CyclomaticComplexity private + def in_operator_params(array_scope_ids:, scope: nil, in_column: nil, in_values: nil, order_hint_column: nil) + base_scope = Event.all + base_scope = base_scope.merge(scope) if scope + + order = { id: :desc } + finder_query = -> (id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } + + if order_hint_column.present? + order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: order_hint_column, + order_expression: Event.arel_table[order_hint_column].desc, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Event.arel_table[:id].desc + ) + ]) + + finder_query = -> (_order_hint, id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } + end + + base_scope = base_scope.reorder(order) + + array_params = in_operator_array_params( + array_scope_ids: array_scope_ids, + scope: base_scope, + in_column: in_column, + in_values: in_values + ) + + array_params.merge( + scope: base_scope, + finder_query: finder_query + ) + end + + # This method builds the array_ parameters + # without in_column parameter: uses one IN filter: author_id + # with in_column: two IN filters: author_id, (target_type OR action) + def in_operator_array_params(scope:, array_scope_ids:, in_column: nil, in_values: nil) + if in_column + # Builds Carthesian product of the in_values and the array_scope_ids (in this case: user_ids). + # The process is described here: https://docs.gitlab.com/ee/development/database/efficient_in_operator_queries.html#multiple-in-queries + # VALUES ((array_scope_ids[0], in_values[0]), (array_scope_ids[1], in_values[0]) ...) + cartesian = array_scope_ids.product(in_values) + user_with_column_list = Arel::Nodes::ValuesList.new(cartesian) + + as = "array_ids(id, #{Event.connection.quote_column_name(in_column)})" + from = Arel::Nodes::Grouping.new(user_with_column_list).as(as) + { + array_scope: User.select(:id, in_column).from(from), + array_mapping_scope: -> (author_id_expression, in_column_expression) do + Event + .merge(scope) + .where(Event.arel_table[:author_id].eq(author_id_expression)) + .where(Event.arel_table[in_column].eq(in_column_expression)) + end + } + else + # Builds a simple query to represent the array_scope_ids + # VALUES ((array_scope_ids[0]), (array_scope_ids[2])...) + array_ids_list = Arel::Nodes::ValuesList.new(array_scope_ids.map { |id| [id] }) + from = Arel::Nodes::Grouping.new(array_ids_list).as('array_ids(id)') + { + array_scope: User.select(:id).from(from), + array_mapping_scope: -> (author_id_expression) do + Event + .merge(scope) + .where(Event.arel_table[:author_id].eq(author_id_expression)) + end + } + end + end + def wiki_events(events) events.for_wiki_page end @@ -61,5 +208,6 @@ class EventFilter [ALL, PUSH, MERGED, ISSUE, COMMENTS, TEAM, WIKI, DESIGNS] end end +# rubocop: enable CodeReuse/ActiveRecord EventFilter.prepend_mod_with('EventFilter') diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index d172df4920f..06160b55f5c 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -50,9 +50,8 @@ module ExpandVariables # Convert hash array to variables if variables.is_a?(Array) - variables = variables.reduce({}) do |hash, variable| + variables = variables.each_with_object({}) do |variable, hash| hash[variable[:key]] = variable[:value] - hash end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index d93067c7e2f..b10330914ca 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -16,6 +16,8 @@ module Gitlab :client_id, :caller_id, :remote_ip, + :job_id, + :pipeline_id, :related_class, :feature_category ].freeze @@ -28,6 +30,7 @@ module Gitlab Attribute.new(:runner, ::Ci::Runner), Attribute.new(:caller_id, String), Attribute.new(:remote_ip, String), + Attribute.new(:job, ::Ci::Build), Attribute.new(:related_class, String), Attribute.new(:feature_category, String) ].freeze @@ -73,14 +76,16 @@ module Gitlab def to_lazy_hash {}.tap do |hash| - hash[:user] = -> { username } if set_values.include?(:user) - hash[:project] = -> { project_path } if set_values.include?(:project) || set_values.include?(:runner) + hash[:user] = -> { username } if include_user? + hash[:project] = -> { project_path } if include_project? hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:client_id] = -> { client } if include_client? hash[:caller_id] = caller_id if set_values.include?(:caller_id) hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) hash[:related_class] = related_class if set_values.include?(:related_class) hash[:feature_category] = feature_category if set_values.include?(:feature_category) + hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) + hash[:job_id] = -> { job&.id } if set_values.include?(:job) end end @@ -103,32 +108,41 @@ module Gitlab end def project_path - associated_routable = project || runner_project + associated_routable = project || runner_project || job_project associated_routable&.full_path end def username - user&.username + associated_user = user || job_user + associated_user&.username end def root_namespace_path - associated_routable = namespace || project || runner_project || runner_group + associated_routable = namespace || project || runner_project || runner_group || job_project associated_routable&.full_path_components&.first end def include_namespace? - set_values.include?(:namespace) || set_values.include?(:project) || set_values.include?(:runner) + set_values.include?(:namespace) || set_values.include?(:project) || set_values.include?(:runner) || set_values.include?(:job) end def include_client? set_values.include?(:user) || set_values.include?(:runner) || set_values.include?(:remote_ip) end + def include_user? + set_values.include?(:user) || set_values.include?(:job) + end + + def include_project? + set_values.include?(:project) || set_values.include?(:runner) || set_values.include?(:job) + end + def client - if user - "user/#{user.id}" - elsif runner + if runner "runner/#{runner.id}" + elsif user + "user/#{user.id}" else "ip/#{remote_ip}" end @@ -150,6 +164,18 @@ module Gitlab runner.groups.first end end + + def job_project + strong_memoize(:job_project) do + job&.project + end + end + + def job_user + strong_memoize(:job_user) do + job&.user + end + end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 0b0aaacbaff..09775297def 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -41,7 +41,8 @@ module Gitlab auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, - gitlab_shell_operation: { threshold: 600, interval: 1.minute } + gitlab_shell_operation: { threshold: 600, interval: 1.minute }, + pipelines_create: { threshold: 25, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb index 53d1135a2d7..6c4ecc04cdc 100644 --- a/lib/gitlab/asciidoc/include_processor.rb +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -33,7 +33,7 @@ module Gitlab max_include_depth = doc.attributes.fetch('max-include-depth').to_i return false if max_include_depth < 1 - return false if target_uri?(target) + return false if target_http?(target) return false if included.size >= max_includes true diff --git a/lib/gitlab/auth/ldap/dn.rb b/lib/gitlab/auth/ldap/dn.rb index ea88dedadf5..a188aa168c1 100644 --- a/lib/gitlab/auth/ldap/dn.rb +++ b/lib/gitlab/auth/ldap/dn.rb @@ -30,7 +30,7 @@ module Gitlab def self.normalize_value(given_value) dummy_dn = "placeholder=#{given_value}" normalized_dn = new(*dummy_dn).to_normalized_s - normalized_dn.sub(/\Aplaceholder=/, '') + normalized_dn.delete_prefix('placeholder=') end ## diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 41a8739b0b6..1a25ed10d81 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,6 +5,7 @@ module Gitlab module OAuth class Provider LABELS = { + "alicloud" => "AliCloud", "dingtalk" => "DingTalk", "github" => "GitHub", "gitlab" => "GitLab.com", diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb index b0a8c3a8cbb..52ff3aaa423 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb @@ -22,8 +22,6 @@ module Gitlab def perform(start_id, end_id) eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) - return if eligible_mrs.empty? - eligible_mrs.each_slice(10) do |slice| MergeRequest.where(id: slice).update_all(draft: true) end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb new file mode 100644 index 00000000000..084c788c8cb --- /dev/null +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill group_features for an array of groups + class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BaseJob + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, batch_size) + pause_ms = 0 if pause_ms < 0 + + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| + batch_metrics.time_operation(:upsert_group_features) do + upsert_group_features(sub_batch, batch_size) + end + + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: connection) + .where(source_key_column => start_id..stop_id) + .where(type: 'Group') + end + + def upsert_group_features(relation, batch_size) + connection.execute( + <<~SQL + INSERT INTO group_features (group_id, created_at, updated_at) + SELECT namespaces.id as group_id, now(), now() + FROM namespaces + WHERE namespaces.type = 'Group' AND namespaces.id IN(#{relation.select(:id).limit(batch_size).to_sql}) + ON CONFLICT (group_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb b/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb deleted file mode 100644 index 2d46ff6b933..00000000000 --- a/lib/gitlab/background_migration/backfill_incident_issue_escalation_statuses.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # BackfillIncidentIssueEscalationStatuses adds - # IncidentManagement::IssuableEscalationStatus records for existing Incident issues. - # They will be added with no policy, and escalations_started_at as nil. - class BackfillIncidentIssueEscalationStatuses - def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL - INSERT INTO incident_management_issuable_escalation_statuses (issue_id, created_at, updated_at) - SELECT issues.id, current_timestamp, current_timestamp - FROM issues - WHERE issues.issue_type = 1 - AND issues.id BETWEEN #{start_id} AND #{stop_id} - ON CONFLICT (issue_id) DO NOTHING; - SQL - - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb new file mode 100644 index 00000000000..1f0d606f001 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `routes.namespace_id` column, by setting it to project.project_namespace_id + class BackfillNamespaceIdForProjectRoute + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + cleanup_gin_index('routes') + + batch_metrics.time_operation(:update_all) do + ActiveRecord::Base.connection.execute <<~SQL + WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{sub_batch.to_sql} + ) + UPDATE routes + SET namespace_id = route_and_ns.project_namespace_id + FROM route_and_ns + WHERE id = route_and_ns.route_id + SQL + end + + pause_ms = [0, pause_ms].max + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def cleanup_gin_index(table_name) + sql = "select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'" + index_names = ActiveRecord::Base.connection.select_values(sql) + + index_names.each do |index_name| + ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')") + end + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + .joins('INNER JOIN projects ON routes.source_id = projects.id') + .where(source_key_column => start_id..stop_id) + .where(namespace_id: nil) + .where(source_type: 'Project') + .where.not(projects: { project_namespace_id: nil }) + .select("routes.id, projects.project_namespace_id") + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb new file mode 100644 index 00000000000..a16efa4222b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `issues.work_item_type_id` column, replacing any + # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` + class BackfillWorkItemTypeIdForIssues + # Basic AR model for issues table + class MigrationIssue < ApplicationRecord + include ::EachBatch + + self.table_name = 'issues' + + scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } + end + + MAX_UPDATE_RETRIES = 3 + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, base_type, base_type_id) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id, base_type) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + reconstructed_sub_batch = MigrationIssue.unscoped.base_query(base_type).where(id: first..last) + + batch_metrics.time_operation(:update_all) do + update_with_retry(reconstructed_sub_batch, base_type_id) + end + + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + # Retry mechanism required as update statements on the issues table will randomly take longer than + # expected due to gin indexes https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71869#note_775796352 + def update_with_retry(sub_batch, base_type_id) + update_attempt = 1 + + begin + update_batch(sub_batch, base_type_id) + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => e + update_attempt += 1 + + if update_attempt <= MAX_UPDATE_RETRIES + # sleeping 30 seconds as it might take a long time to clean the gin index pending list + sleep(30) + retry + end + + raise e + end + end + + def update_batch(sub_batch, base_type_id) + sub_batch.update_all(work_item_type_id: base_type_id) + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, end_id, base_type) + MigrationIssue.where(source_key_column => start_id..end_id).base_query(base_type) + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb new file mode 100644 index 00000000000..06036eebcb9 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for back-filling issue's work_item_type_id for a single issue type. + # Batches will be scoped to records where the foreign key is NULL and only of a given issue type + # + # If no more batches exist in the table, returns nil. + class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy + def apply_additional_filters(relation, job_arguments:) + issue_type = job_arguments.first + + relation.where(issue_type: issue_type) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 5569bac0e19..e7a68b183b8 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -23,6 +23,7 @@ module Gitlab quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + relation = apply_additional_filters(relation, job_arguments: job_arguments) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop @@ -33,6 +34,22 @@ module Gitlab next_batch_bounds end + + # Strategies based on PrimaryKeyBatchingStrategy can use + # this method to easily apply additional filters. + # + # Example: + # + # class MatchingType < PrimaryKeyBatchingStrategy + # def apply_additional_filters(relation, job_arguments:) + # type = job_arguments.first + # + # relation.where(type: type) + # end + # end + def apply_additional_filters(relation, job_arguments: []) + relation + end end end end diff --git a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb new file mode 100644 index 00000000000..b703faf6a6c --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Cleanup draft column data inserted by a faulty regex + # + class CleanupDraftDataFromFaultyRegex + # Migration only version of MergeRequest table + ## + class MergeRequest < ActiveRecord::Base + LEAKY_REGEXP_STR = "^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP" + CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" + + include EachBatch + + self.table_name = 'merge_requests' + + def self.eligible + where(state_id: 1) + .where(draft: true) + .where("title ~* ?", LEAKY_REGEXP_STR) + .where("title !~* ?", CORRECTED_REGEXP_STR) + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) + + return if eligible_mrs.empty? + + eligible_mrs.each_slice(10) do |slice| + MergeRequest.where(id: slice).update_all(draft: false) + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'CleanupDraftDataFromFaultyRegex', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb index 80931353e2f..a087d2529eb 100644 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ b/lib/gitlab/background_migration/encrypt_static_object_token.rb @@ -52,9 +52,9 @@ module Gitlab WHERE cte_id = id SQL end - - mark_job_as_succeeded(start_id, end_id) end + + mark_job_as_succeeded(start_id, end_id) end private diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb new file mode 100644 index 00000000000..defd9ea832b --- /dev/null +++ b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fix project name duplicates and backfill missing project namespace ids + class FixDuplicateProjectNameAndPath + SUB_BATCH_SIZE = 10 + # isolated project active record + class Project < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'projects' + + scope :without_project_namespace, -> { where(project_namespace_id: nil) } + scope :id_in, ->(ids) { where(id: ids) } + end + + def perform(start_id, end_id) + @project_ids = fetch_project_ids(start_id, end_id) + backfill_project_namespaces_service = init_backfill_service(project_ids) + backfill_project_namespaces_service.cleanup_gin_index('projects') + + project_ids.each_slice(SUB_BATCH_SIZE) do |ids| + ActiveRecord::Base.connection.execute(update_projects_name_and_path_sql(ids)) + end + + backfill_project_namespaces_service.backfill_project_namespaces + + mark_job_as_succeeded(start_id, end_id) + end + + private + + attr_accessor :project_ids + + def fetch_project_ids(start_id, end_id) + Project.without_project_namespace.where(id: start_id..end_id) + end + + def init_backfill_service(project_ids) + service = Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.new + service.project_ids = project_ids + service.sub_batch_size = SUB_BATCH_SIZE + + service + end + + def update_projects_name_and_path_sql(project_ids) + <<~SQL + WITH cte (project_id, path_from_route ) AS ( + #{path_from_route_sql(project_ids).to_sql} + ) + UPDATE + projects + SET + name = concat(projects.name, '-', id), + path = CASE + WHEN projects.path <> cte.path_from_route THEN path_from_route + ELSE projects.path + END + FROM + cte + WHERE + projects.id = cte.project_id; + SQL + end + + def path_from_route_sql(project_ids) + Project.without_project_namespace.id_in(project_ids) + .joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'") + .select("projects.id, SUBSTRING(routes.path FROM '[^/]+(?=/$|$)') AS path_from_route") + end + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'FixDuplicateProjectNameAndPath', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb new file mode 100644 index 00000000000..07231098a5f --- /dev/null +++ b/lib/gitlab/background_migration/merge_topics_with_same_name.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to merge project topics with the same case insensitive name + class MergeTopicsWithSameName + # Temporary AR model for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + # Temporary AR model for project topic assignment + class ProjectTopic < ActiveRecord::Base + self.table_name = 'project_topics' + end + + def perform(topic_names) + topic_names.each do |topic_name| + topics = Topic.where('LOWER(name) = ?', topic_name) + .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc) + .to_a + topic_to_keep = topics.shift + merge_topics(topic_to_keep, topics) if topics.any? + end + end + + private + + def merge_topics(topic_to_keep, topics_to_remove) + description = topic_to_keep.description + + topics_to_remove.each do |topic| + description ||= topic.description if topic.description.present? + process_avatar(topic_to_keep, topic) if topic.avatar.present? + + ProjectTopic.transaction do + ProjectTopic.where(topic_id: topic.id) + .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id)) + .update_all(topic_id: topic_to_keep.id) + ProjectTopic.where(topic_id: topic.id).delete_all + end + end + + Topic.where(id: topics_to_remove).delete_all + + topic_to_keep.update( + description: description, + total_projects_count: total_projects_count(topic_to_keep.id), + non_private_projects_count: non_private_projects_count(topic_to_keep.id) + ) + end + + # We intentionally use application code here because we need to copy/remove avatar files + def process_avatar(topic_to_keep, topic_to_remove) + topic_to_remove = ::Projects::Topic.find(topic_to_remove.id) + topic_to_keep = ::Projects::Topic.find(topic_to_keep.id) + unless topic_to_keep.avatar.present? + topic_to_keep.avatar = topic_to_remove.avatar + topic_to_keep.save! + end + + topic_to_remove.remove_avatar! + topic_to_remove.save! + end + + def total_projects_count(topic_id) + ProjectTopic.where(topic_id: topic_id).count + end + + def non_private_projects_count(topic_id) + ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id') + .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb new file mode 100644 index 00000000000..ec4631d1e34 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate category of integrations to third_party_wiki for confluence and shimo + class MigrateShimoConfluenceIntegrationCategory + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id) + define_batchable_model('integrations', connection: ::ActiveRecord::Base.connection) + .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) + .update_all(category: 'third_party_wiki') + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb new file mode 100644 index 00000000000..9e102ea1517 --- /dev/null +++ b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to populates the migration_plan column of container_repositories + # with the current plan of the namespaces that owns the container_repository + # + # The plan can be NULL, in which case no UPDATE + # will be executed. + class PopulateContainerRepositoryMigrationPlan + def perform(start_id, end_id) + (start_id..end_id).each do |id| + execute(<<~SQL) + WITH selected_plan AS ( + SELECT "plans"."name" + FROM "container_repositories" + INNER JOIN "projects" ON "projects"."id" = "container_repositories"."project_id" + INNER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id" + INNER JOIN "gitlab_subscriptions" ON "gitlab_subscriptions"."namespace_id" = "namespaces"."traversal_ids"[1] + INNER JOIN "plans" ON "plans"."id" = "gitlab_subscriptions"."hosted_plan_id" + WHERE "container_repositories"."id" = #{id} + ) + UPDATE container_repositories + SET migration_plan = selected_plan.name + FROM selected_plan + WHERE container_repositories.id = #{id}; + SQL + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def connection + @connection ||= ::ActiveRecord::Base.connection + end + + def execute(sql) + connection.execute(sql) + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb index e873ad412f2..97927ef48c2 100644 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -5,9 +5,40 @@ module Gitlab # This class creates/updates those namespace statistics # that haven't been created nor initialized. # It also updates the related namespace statistics - # This is only required in EE class PopulateNamespaceStatistics def perform(group_ids, statistics) + # Updating group statistics might involve calling Gitaly. + # For example, when calculating `wiki_size`, we will need + # to perform the request to check if the repo exists and + # also the repository size. + # + # The `allow_n_plus_1_calls` method is only intended for + # dev and test. It won't be raised in prod. + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + relation(group_ids).each do |group| + upsert_namespace_statistics(group, statistics) + end + end + end + + private + + def upsert_namespace_statistics(group, statistics) + response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute + + error_message("#{response.message} group: #{group.id}") if response.error? + end + + def logger + @logger ||= ::Gitlab::BackgroundMigration::Logger.build + end + + def error_message(message) + logger.error(message: "Namespace Statistics Migration: #{message}") + end + + def relation(group_ids) + Group.includes(:namespace_statistics).where(id: group_ids) end end end diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index c34cc57ce60..bd7d7d02162 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -7,6 +7,8 @@ module Gitlab # # rubocop: disable Metrics/ClassLength class BackfillProjectNamespaces + attr_accessor :project_ids, :sub_batch_size + SUB_BATCH_SIZE = 25 PROJECT_NAMESPACE_STI_NAME = 'Project' @@ -18,7 +20,7 @@ module Gitlab case migration_type when 'up' - backfill_project_namespaces(namespace_id) + backfill_project_namespaces mark_job_as_succeeded(start_id, end_id, namespace_id, 'up') when 'down' cleanup_backfilled_project_namespaces(namespace_id) @@ -28,11 +30,7 @@ module Gitlab end end - private - - attr_accessor :project_ids, :sub_batch_size - - def backfill_project_namespaces(namespace_id) + def backfill_project_namespaces project_ids.each_slice(sub_batch_size) do |project_ids| # cleanup gin indexes on namespaces table cleanup_gin_index('namespaces') @@ -64,6 +62,8 @@ module Gitlab end end + private + def cleanup_backfilled_project_namespaces(namespace_id) project_ids.each_slice(sub_batch_size) do |project_ids| # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 78a8f39e143..e210c18e3d1 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -2,11 +2,16 @@ module Gitlab class Blame - attr_accessor :blob, :commit + attr_accessor :blob, :commit, :range - def initialize(blob, commit) + def initialize(blob, commit, range: nil) @blob = blob @commit = commit + @range = range + end + + def first_line + range&.first || 1 end def groups(highlight: true) @@ -14,14 +19,14 @@ module Gitlab groups = [] current_group = nil - i = 0 - blame.each do |commit, line| + i = first_line - 1 + blame.each do |commit, line, previous_path| commit = Commit.new(commit, project) commit.lazy_author # preload author if prev_sha != commit.sha groups << current_group if current_group - current_group = { commit: commit, lines: [] } + current_group = { commit: commit, lines: [], previous_path: previous_path } end current_group[:lines] << (highlight ? highlighted_lines[i].html_safe : line) @@ -37,7 +42,7 @@ module Gitlab private def blame - @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path) + @blame ||= Gitlab::Git::Blame.new(repository, @commit.id, @blob.path, range: range) end def highlighted_lines diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index ef936581c10..10233cf4228 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -447,9 +447,8 @@ module Gitlab end def state - state = STATE_PARAMS.inject({}) do |h, param| + state = STATE_PARAMS.each_with_object({}) do |param, h| h[param] = send(param) # rubocop:disable GitlabSecurity/PublicSend - h end Base64.urlsafe_encode64(state.to_json) end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 2b190d89fa4..2c9524c89ff 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -6,6 +6,8 @@ module Gitlab # Base GitLab CI Configuration facade # class Config + include Gitlab::Utils::StrongMemoize + ConfigError = Class.new(StandardError) TIMEOUT_SECONDS = 30.seconds TIMEOUT_MESSAGE = 'Resolving config took longer than expected' @@ -22,6 +24,11 @@ module Gitlab def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path + @project = project + + if use_config_variables? + pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) + end @context = self.logger.instrument(:config_build_context) do build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) @@ -82,7 +89,13 @@ module Gitlab end def included_templates - @context.expandset.filter_map { |i| i[:template] } + @context.includes.filter_map { |i| i[:location] if i[:type] == :template } + end + + def metadata + { + includes: @context.includes + } end private @@ -149,6 +162,10 @@ module Gitlab end def build_variables_without_instrumentation(project:, pipeline:) + if use_config_variables? + return pipeline.variables_builder.config_variables + end + Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project @@ -178,6 +195,12 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end + def use_config_variables? + strong_memoize(:use_config_variables) do + ::Feature.enabled?(:ci_variables_builder_config_variables, @project, default_enabled: :yaml) + end + end + # Overridden in EE def rescue_errors RESCUE_ERRORS diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 43475742214..46afedbcc3a 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -23,7 +23,7 @@ module Gitlab validates :config, presence: true validates :name, presence: true validates :name, type: Symbol - validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) } + validates :name, length: { maximum: 255 } validates :config, disallowed_keys: { in: %i[only except start_in], diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 512cfdde474..2def565bc19 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -70,16 +70,20 @@ module Gitlab } end - def mask_variables_from(location) - variables.reduce(location.dup) do |loc, variable| + def mask_variables_from(string) + variables.reduce(string.dup) do |str, variable| if variable[:masked] - Gitlab::Ci::MaskSecret.mask!(loc, variable[:value]) + Gitlab::Ci::MaskSecret.mask!(str, variable[:value]) else - loc + str end end end + def includes + expandset.map(&:metadata) + end + protected attr_writer :expandset, :execution_deadline, :logger diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 4f79e64ca9a..1244c7f7475 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -28,6 +28,14 @@ module Gitlab end end + def metadata + super.merge( + type: :artifact, + location: masked_location, + extra: { job_name: masked_job_name } + ) + end + private def project @@ -52,7 +60,7 @@ module Gitlab end unless artifact_job.present? - errors.push("Job `#{job_name}` not found in parent pipeline or does not have artifacts!") + errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") return false end @@ -80,6 +88,12 @@ module Gitlab parent_pipeline: context.parent_pipeline } end + + def masked_job_name + strong_memoize(:masked_job_name) do + context.mask_variables_from(job_name) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index a660dd339d8..89da0796906 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -16,8 +16,6 @@ module Gitlab @params = params @context = context @errors = [] - - validate! end def matching? @@ -48,6 +46,30 @@ module Gitlab expanded_content_hash end + def validate! + context.logger.instrument(:config_file_validation) do + validate_execution_time! + validate_location! + validate_content! if errors.none? + validate_hash! if errors.none? + end + end + + def metadata + { + context_project: context.project&.full_path, + context_sha: context.sha + } + end + + def eql?(other) + other.hash == hash + end + + def hash + [params, context.project&.full_path, context.sha].hash + end + protected def expanded_content_hash @@ -66,13 +88,6 @@ module Gitlab nil end - def validate! - validate_execution_time! - validate_location! - validate_content! if errors.none? - validate_hash! if errors.none? - end - def validate_execution_time! context.check_execution_time! end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 3aa665c7d18..ee9cc1552fe 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -19,6 +19,14 @@ module Gitlab strong_memoize(:content) { fetch_local_content } end + def metadata + super.merge( + type: :local, + location: masked_location, + extra: {} + ) + end + private def validate_content! diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 27e097ba980..3d4436530a8 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -27,17 +27,25 @@ module Gitlab strong_memoize(:content) { fetch_local_content } end + def metadata + super.merge( + type: :file, + location: masked_location, + extra: { project: masked_project_name, ref: masked_ref_name } + ) + end + private def validate_content! if !can_access_local_content? - errors.push("Project `#{project_name}` not found or access denied!") + errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") elsif sha.nil? - errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!") + errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!") elsif content.nil? - errors.push("Project `#{project_name}` file `#{masked_location}` does not exist!") + errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!") elsif content.blank? - errors.push("Project `#{project_name}` file `#{masked_location}` is empty!") + errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!") end end @@ -76,6 +84,18 @@ module Gitlab variables: context.variables } end + + def masked_project_name + strong_memoize(:masked_project_name) do + context.mask_variables_from(project_name) + end + end + + def masked_ref_name + strong_memoize(:masked_ref_name) do + context.mask_variables_from(ref_name) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 8335a9ef625..e7b007b4d8d 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -18,6 +18,14 @@ module Gitlab strong_memoize(:content) { fetch_remote_content } end + def metadata + super.merge( + type: :remote, + location: masked_location, + extra: {} + ) + end + private def validate_location! diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index c3d120dfdce..9469f09ce13 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -20,6 +20,14 @@ module Gitlab strong_memoize(:content) { fetch_template_content } end + def metadata + super.merge( + type: :template, + location: masked_location, + extra: {} + ) + end + private def validate_location! diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 79a04ad409e..c1250c82750 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -48,8 +48,8 @@ module Gitlab .flat_map(&method(:expand_project_files)) .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) - .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) + .each(&method(:verify!)) end def normalize_location(location) @@ -111,26 +111,6 @@ module Gitlab end end - def verify_duplicates!(location) - logger.instrument(:config_mapper_verify) do - verify_max_includes_and_add_location!(location) - end - end - - def verify_max_includes_and_add_location!(location) - if expandset.count >= MAX_INCLUDES - raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" - end - - # Scope location to context to allow support of - # relative includes - scoped_location = location.merge( - context_project: context.project, - context_sha: context.sha) - - expandset.add(scoped_location) - end - def select_first_matching(location) logger.instrument(:config_mapper_select) do select_first_matching_without_instrumentation(location) @@ -147,6 +127,18 @@ module Gitlab matching.first end + def verify!(location_object) + verify_max_includes! + location_object.validate! + expandset.add(location_object) + end + + def verify_max_includes! + if expandset.count >= MAX_INCLUDES + raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + end + end + def expand_variables(data) logger.instrument(:config_mapper_variables) do expand_variables_without_instrumentation(data) diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 7baae2f53d7..13a159f3745 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -14,6 +14,7 @@ module Gitlab def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) @json_data = json_data @report = report + @project = report.project @validate = validate @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled end @@ -43,31 +44,41 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - if Feature.enabled?(:show_report_validation_warnings, default_enabled: :yaml) - # We want validation to happen regardless of VALIDATE_SCHEMA CI variable + # We want validation to happen regardless of VALIDATE_SCHEMA + # CI variable. + # + # Previously it controlled BOTH validation and enforcement of + # schema validation result. + # + # After 15.0 we will enforce schema validation by default + # See: https://gitlab.com/groups/gitlab-org/-/epics/6968 + schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) } + + if validate schema_validation_passed = schema_validator.valid? - if validate - schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed - - schema_validation_passed - else - # We treat all schema validation errors as warnings - schema_validator.errors.each { |error| report.add_warning('Schema', error) } + # Validation warnings are errors + schema_validator.errors.each { |error| report.add_error('Schema', error) } + schema_validator.warnings.each { |warning| report.add_error('Schema', warning) } - true - end + schema_validation_passed else - return true if !validate || schema_validator.valid? + # Validation warnings are warnings + schema_validator.errors.each { |error| report.add_warning('Schema', error) } + schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) } - schema_validator.errors.each { |error| report.add_error('Schema', error) } - - false + true end end def schema_validator - @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version) + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new( + report.type, + report_data, + report.version, + project: @project, + scanner: top_level_scanner + ) end def report_data @@ -137,7 +148,7 @@ module Gitlab metadata_version: report_version, details: data['details'] || {}, signatures: signatures, - project_id: report.project_id, + project_id: @project.id, vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled)) end @@ -280,7 +291,7 @@ module Gitlab report_type: report.type, primary_identifier_fingerprint: primary_identifier&.fingerprint, location_fingerprint: location_fingerprint, - project_id: report.project_id + project_id: @project.id } if uuid_v5_name_components.values.any?(&:nil?) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 0ab1a128052..cef029bd749 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -8,14 +8,14 @@ module Gitlab class SchemaValidator # https://docs.gitlab.com/ee/update/deprecations.html#147 SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1] }.freeze # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags @@ -26,19 +26,19 @@ module Gitlab 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 - KNOWN_VERSIONS_TO_DEPRECATE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze + KNOWN_VERSIONS_TO_REMOVE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze - VERSIONS_TO_DEPRECATE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_DEPRECATE).freeze + VERSIONS_TO_REMOVE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_REMOVE).freeze DEPRECATED_VERSIONS = { - cluster_image_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, - container_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, - coverage_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, - dast: VERSIONS_TO_DEPRECATE_IN_15_0, - api_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, - dependency_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, - sast: VERSIONS_TO_DEPRECATE_IN_15_0, - secret_detection: VERSIONS_TO_DEPRECATE_IN_15_0 + cluster_image_scanning: VERSIONS_TO_REMOVE_IN_15_0, + container_scanning: VERSIONS_TO_REMOVE_IN_15_0, + coverage_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, + dast: VERSIONS_TO_REMOVE_IN_15_0, + api_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, + dependency_scanning: VERSIONS_TO_REMOVE_IN_15_0, + sast: VERSIONS_TO_REMOVE_IN_15_0, + secret_detection: VERSIONS_TO_REMOVE_IN_15_0 }.freeze class Schema @@ -86,20 +86,110 @@ module Gitlab end end - def initialize(report_type, report_data, report_version = nil) - @report_type = report_type + def initialize(report_type, report_data, report_version = nil, project: nil, scanner: nil) + @report_type = report_type&.to_sym @report_data = report_data @report_version = report_version + @project = project + @scanner = scanner + @errors = [] + @warnings = [] + @deprecation_warnings = [] + + populate_errors + populate_warnings + populate_deprecation_warnings end def valid? errors.empty? end - def errors - @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + def populate_errors + schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + + log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty? + + if Feature.enabled?(:enforce_security_report_validation, @project) + @errors += schema_validation_errors + else + @warnings += schema_validation_errors + end + end + + def populate_warnings + add_unsupported_report_version_message if !report_uses_supported_schema_version? && !report_uses_deprecated_schema_version? + end + + def populate_deprecation_warnings + add_deprecated_report_version_message if report_uses_deprecated_schema_version? + end + + def add_deprecated_report_version_message + log_warnings(problem_type: 'using_deprecated_schema_version') + + message = "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_schema_versions}" + add_message_as(level: :deprecation_warning, message: message) + end + + def log_warnings(problem_type:) + Gitlab::AppLogger.info( + message: 'security report schema validation problem', + security_report_type: report_type, + security_report_version: report_version, + project_id: @project.id, + security_report_failure: problem_type, + security_report_scanner_id: @scanner&.dig('id'), + security_report_scanner_version: @scanner&.dig('version') + ) + end + + def add_unsupported_report_version_message + log_warnings(problem_type: 'using_unsupported_schema_version') + + if Feature.enabled?(:enforce_security_report_validation, @project) + handle_unsupported_report_version(treat_as: :error) + else + handle_unsupported_report_version(treat_as: :warning) + end + end + + def report_uses_deprecated_schema_version? + DEPRECATED_VERSIONS[report_type].include?(report_version) + end + + def report_uses_supported_schema_version? + SUPPORTED_VERSIONS[report_type].include?(report_version) end + def handle_unsupported_report_version(treat_as:) + if report_version.nil? + message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}" + add_message_as(level: treat_as, message: message) + else + message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}" + end + + add_message_as(level: treat_as, message: message) + end + + def supported_schema_versions + SUPPORTED_VERSIONS[report_type].join(", ") + end + + def add_message_as(level:, message:) + case level + when :deprecation_warning + @deprecation_warnings << message + when :error + @errors << message + when :warning + @warnings << message + end + end + + attr_reader :errors, :warnings, :deprecation_warnings + private attr_reader :report_type, :report_data, :report_version diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..7bcb2d5867f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json @@ -0,0 +1,977 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json new file mode 100644 index 00000000000..a13e0418499 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json @@ -0,0 +1,911 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..050c34669b3 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json new file mode 100644 index 00000000000..62ed293ad44 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json @@ -0,0 +1,1291 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "minLength": 1, + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + }, + "discovered_at": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", + "examples": [ + "2020-01-28T03:26:02.956" + ] + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json new file mode 100644 index 00000000000..1e3f4188845 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json @@ -0,0 +1,968 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "dependency_files", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json new file mode 100644 index 00000000000..4c57d20dbaa --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json @@ -0,0 +1,869 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json new file mode 100644 index 00000000000..b1337954e97 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json @@ -0,0 +1,892 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.1" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb new file mode 100644 index 00000000000..cb02f09f819 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Limit + class RateLimit < Chain::Base + include Chain::Helpers + + def perform! + return unless throttle_enabled? + + # We exclude child-pipelines from the rate limit because they represent + # sub-pipelines that would otherwise hit the rate limit due to having the + # same scope (project, user, sha). + # + return if pipeline.parent_pipeline? + + if rate_limit_throttled? + create_log_entry + error(throttle_message) unless dry_run? + end + end + + def break? + @pipeline.errors.any? + end + + private + + def rate_limit_throttled? + ::Gitlab::ApplicationRateLimiter.throttled?( + :pipelines_create, scope: [project, current_user, command.sha] + ) + end + + def create_log_entry + Gitlab::AppJsonLogger.info( + class: self.class.name, + namespace_id: project.namespace_id, + project_id: project.id, + commit_sha: command.sha, + current_user_id: current_user.id, + subscription_plan: project.actual_plan_name, + message: 'Activated pipeline creation rate limit' + ) + end + + def throttle_message + 'Too many pipelines created in the last minute. Try again later.' + end + + def throttle_enabled? + ::Feature.enabled?( + :ci_throttle_pipelines_creation, + project, + default_enabled: :yaml) + end + + def dry_run? + ::Feature.enabled?( + :ci_throttle_pipelines_creation_dry_run, + project, + default_enabled: :yaml) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb index 2fcf1740b5f..f9b3b6cd644 100644 --- a/lib/gitlab/ci/pipeline/chain/template_usage.rb +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -19,7 +19,7 @@ module Gitlab def track_event(template) Gitlab::UsageDataCounters::CiTemplateUniqueCounter - .track_unique_project_event(project_id: pipeline.project_id, template: template, config_source: pipeline.config_source) + .track_unique_project_event(project: pipeline.project, template: template, config_source: pipeline.config_source, user: current_user) end def included_templates diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 8c528056d0c..70f2919d38d 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -9,6 +9,7 @@ module Gitlab attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline + delegate :project, to: :pipeline def initialize(type, pipeline, created_at) @type = type @@ -38,6 +39,10 @@ module Gitlab errors.present? end + def warnings? + warnings.present? + end + def add_scanner(scanner) scanners[scanner.key] ||= scanner end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb index c1de03cea44..1ac66a0c671 100644 --- a/lib/gitlab/ci/reports/security/scanner.rb +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -12,6 +12,7 @@ module Gitlab "gemnasium-maven" => 3, "gemnasium-python" => 3, "bandit" => 1, + "spotbugs" => 1, "semgrep" => 2 }.freeze diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index 00920dfbd54..d0388c65f58 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -12,7 +12,6 @@ module Gitlab def initialize(name = nil) @name = name @test_cases = {} - @all_test_cases = [] @total_time = 0.0 end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb new file mode 100644 index 00000000000..944c24ca128 --- /dev/null +++ b/lib/gitlab/ci/runner_releases.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class RunnerReleases + include Singleton + + RELEASES_VALIDITY_PERIOD = 1.day + RELEASES_VALIDITY_AFTER_ERROR_PERIOD = 5.seconds + + INITIAL_BACKOFF = 5.seconds + MAX_BACKOFF = 1.hour + BACKOFF_GROWTH_FACTOR = 2.0 + + def initialize + reset! + end + + # Returns a sorted list of the publicly available GitLab Runner releases + # + def releases + return @releases unless Time.now.utc >= @expire_time + + @releases = fetch_new_releases + end + + def reset! + @expire_time = Time.now.utc + @releases = nil + @backoff_count = 0 + end + + public_class_method :instance + + private + + def fetch_new_releases + response = Gitlab::HTTP.try_get(::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url) + + releases = response.success? ? extract_releases(response) : nil + ensure + @expire_time = (releases ? RELEASES_VALIDITY_PERIOD : next_backoff).from_now + end + + def extract_releases(response) + response.parsed_response.map { |release| parse_runner_release(release) }.sort! + end + + def parse_runner_release(release) + ::Gitlab::VersionInfo.parse(release['name'].delete_prefix('v')) + end + + def next_backoff + return MAX_BACKOFF if @backoff_count >= 11 # optimization to prevent expensive exponentiation and possible overflows + + backoff = (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**@backoff_count)) + .clamp(INITIAL_BACKOFF, MAX_BACKOFF) + .seconds + @backoff_count += 1 + + backoff + end + end + end +end diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb new file mode 100644 index 00000000000..baf041fc358 --- /dev/null +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class RunnerUpgradeCheck + include Singleton + + def initialize + reset! + end + + def check_runner_upgrade_status(runner_version) + return :unknown unless runner_version + + releases = RunnerReleases.instance.releases + parsed_runner_version = runner_version.is_a?(::Gitlab::VersionInfo) ? runner_version : ::Gitlab::VersionInfo.parse(runner_version) + + raise ArgumentError, "'#{runner_version}' is not a valid version" unless parsed_runner_version.valid? + + available_releases = releases.reject { |release| release > @gitlab_version } + + return :recommended if available_releases.any? { |available_release| patch_update?(available_release, parsed_runner_version) } + return :recommended if outside_backport_window?(parsed_runner_version, releases) + return :available if available_releases.any? { |available_release| available_release > parsed_runner_version } + + :not_available + end + + def reset! + @gitlab_version = ::Gitlab::VersionInfo.parse(::Gitlab::VERSION) + end + + public_class_method :instance + + private + + def patch_update?(available_release, runner_version) + # https://docs.gitlab.com/ee/policy/maintenance.html#patch-releases + available_release.major == runner_version.major && + available_release.minor == runner_version.minor && + available_release.patch > runner_version.patch + end + + def outside_backport_window?(runner_version, releases) + return false if runner_version >= releases.last # return early if runner version is too new + + latest_minor_releases = releases.map { |r| version_without_patch(r) }.uniq { |v| v.to_s } + latest_version_position = latest_minor_releases.count - 1 + runner_version_position = latest_minor_releases.index(version_without_patch(runner_version)) + + return true if runner_version_position.nil? # consider outside if version is too old + + # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases + latest_version_position - runner_version_position > 2 + end + + def version_without_patch(version) + ::Gitlab::VersionInfo.new(version.major, version.minor, 0) + end + end + end +end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index df572188194..0074f3675e0 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -5,20 +5,36 @@ module Gitlab module Status module Build class Manual < Status::Extended + def self.matches?(build, user) + build.playable? + end + def illustration { image: 'illustrations/manual_action.svg', size: 'svg-394', title: _('This job requires a manual action'), - content: _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + content: illustration_content } end - def self.matches?(build, user) - build.playable? + private + + def illustration_content + if can?(user, :update_build, subject) + _('This job requires manual intervention to start. Before starting this job, you can add variables below for last-minute configuration changes.') + else + generic_permission_failure_message + end + end + + def generic_permission_failure_message + _("This job does not run automatically and must be started manually, but you do not have access to it.") end end end end end end + +Gitlab::Ci::Status::Build::Manual.prepend_mod_with('Gitlab::Ci::Status::Build::Manual') diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index bdcd3240380..c078c99f352 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -4,7 +4,7 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/C++.gitlab-ci.yml # use the official gcc image, based on debian -# can use verions as well, like gcc:5.2 +# can use versions as well, like gcc:5.2 # see https://hub.docker.com/_/gcc/ image: gcc diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 19e4ffdbe1e..bd8e1020c4e 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -5,21 +5,6 @@ image: golang:latest -variables: - # Please edit to your GitLab project - REPO_NAME: gitlab.com/namespace/project - -# The problem is that to be able to use go get, one needs to put -# the repository in the $GOPATH. So for example if your gitlab domain -# is gitlab.com, and that your repository is namespace/project, and -# the default GOPATH being /go, then you'd need to have your -# repository in /go/src/gitlab.com/namespace/project -# Thus, making a symbolic link corrects this. -before_script: - - mkdir -p "$GOPATH/src/$(dirname $REPO_NAME)" - - ln -svf "$CI_PROJECT_DIR" "$GOPATH/src/$REPO_NAME" - - cd "$GOPATH/src/$REPO_NAME" - stages: - test - build @@ -35,7 +20,8 @@ format: compile: stage: build script: - - go build -race -ldflags "-extldflags '-static'" -o $CI_PROJECT_DIR/mybinary + - mkdir -p mybinaries + - go build -o mybinaries ./... artifacts: paths: - - mybinary + - mybinaries diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index cc204207f84..0cc5090f85e 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 1a99db67441..d41182ec9be 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -32,6 +32,16 @@ dependency_scanning: .ds-analyzer: extends: dependency_scanning allow_failure: true + variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/$DS_ANALYZER_NAME:$DS_MAJOR_VERSION" + # DS_ANALYZER_NAME is an undocumented variable used in job definitions + # to inject the analyzer name in the image name. + DS_ANALYZER_NAME: "" + image: + name: "$DS_ANALYZER_IMAGE$DS_IMAGE_SUFFIX" # `rules` must be overridden explicitly by each child job # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: @@ -46,13 +56,8 @@ gemnasium-dependency_scanning: extends: - .ds-analyzer - .cyclone-dx-reports - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -77,13 +82,8 @@ gemnasium-maven-dependency_scanning: extends: - .ds-analyzer - .cyclone-dx-reports - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "gemnasium-maven" # Stop reporting Gradle as "maven". # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 DS_REPORT_PACKAGE_MANAGER_MAVEN_WHEN_JAVA: "false" @@ -105,13 +105,8 @@ gemnasium-python-dependency_scanning: extends: - .ds-analyzer - .cyclone-dx-reports - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "gemnasium-python" # Stop reporting Pipenv and Setuptools as "pip". # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false" @@ -138,13 +133,8 @@ gemnasium-python-dependency_scanning: bundler-audit-dependency_scanning: extends: .ds-analyzer - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "bundler-audit" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -158,13 +148,8 @@ bundler-audit-dependency_scanning: retire-js-dependency_scanning: extends: .ds-analyzer - image: - name: "$DS_ANALYZER_IMAGE" variables: - # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" + DS_ANALYZER_NAME: "retire.js" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index bc4f2099d94..89eb91c981f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index ce584091eab..78f28b59aa5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 5ddfb2a54be..488e7ec72fd 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -1,7 +1,14 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/iac_scanning/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/iac_scanning/index.html + variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" iac-sast: @@ -25,7 +32,7 @@ kics-iac-sast: name: "$SAST_ANALYZER_IMAGE" variables: SAST_ANALYZER_IMAGE_TAG: 1 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 8cc9ea0200c..7415fa3104c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -7,6 +7,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" @@ -101,7 +102,11 @@ flawfinder-sast: - if: $CI_COMMIT_BRANCH exists: - '**/*.c' + - '**/*.cc' - '**/*.cpp' + - '**/*.c++' + - '**/*.cp' + - '**/*.cxx' kubesec-sast: extends: .sast-analyzer @@ -246,8 +251,9 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + SEARCH_MAX_DEPTH: 20 SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED when: never @@ -262,6 +268,7 @@ semgrep-sast: - '**/*.tsx' - '**/*.c' - '**/*.go' + - '**/*.java' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 0ef6f63bb94..6aacd082fd7 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -6,12 +6,14 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECRET_DETECTION_IMAGE_SUFFIX: "" + SECRETS_ANALYZER_VERSION: "3" SECRET_DETECTION_EXCLUDED_PATHS: "" .secret-analyzer: stage: test - image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION$SECRET_DETECTION_IMAGE_SUFFIX" services: [] allow_failure: true variables: @@ -31,14 +33,7 @@ secret_detection: script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi # Historic scan - - | - if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ] - then - echo "historic scan" - git fetch --unshallow origin $CI_COMMIT_REF_NAME - /analyzer run - exit - fi + - if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ]; then echo "Running Secret Detection Historic Scan"; /analyzer run; exit; fi # Default branch scan - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi # Push event diff --git a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml new file mode 100644 index 00000000000..67c69115948 --- /dev/null +++ b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml @@ -0,0 +1,96 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml + +# Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template has three jobs: +# - `command`: Run MATLAB scripts, functions, and statements. +# - `test`: Run tests authored using the MATLAB unit testing framework or Simulink Test. +# - `test_artifacts_job`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. +# +# You can copy and paste one or more jobs in this template into your `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# - To run MATLAB and Simulink, MATLAB must be installed on the runner that will run the jobs. +# The runner will use the topmost MATLAB version on the system path. +# The build fails if the operating system cannot find MATLAB on the path. +# - The jobs in this template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported +# in MATLAB R2019a and later. + +# The `command` runs MATLAB scripts, functions, and statements. To use the job in your pipeline, +# substitute `command` with the code you want to run. +# +command: + script: matlab -batch command + +# If the value of `command` is the name of a MATLAB script or function, do not specify the file extension. +# For example, to run a script named `myscript.m` in the root of your repository, specify the `command` like this: +# +# "myscript" +# +# If you specify more than one script, function, or statement, use a comma or semicolon to separate them. +# For example, to run `myscript.m` in a folder named `myfolder` located in the root of the repository, +# you can specify the `command` like this: +# +# "addpath('myfolder'), myscript" +# +# MATLAB exits with exit code 0 if the specified script, function, or statement executes successfully without +# error. Otherwise, MATLAB terminates with a nonzero exit code, which causes the job to fail. To have the +# job fail in certain conditions, use the [`assert`][1] or [`error`][2] functions. +# +# [1] https://www.mathworks.com/help/matlab/ref/assert.html +# [2] https://www.mathworks.com/help/matlab/ref/error.html + +# The `test` runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function +# to run the tests and then the [`assertSuccess`][4] method to fail the job if any of the tests fail. +# +test: + script: matlab -batch "results = runtests('IncludeSubfolders',true), assertSuccess(results);" + +# By default, the job includes any files in your [MATLAB Project][5] that have a `Test` label. If your repository +# does not have a MATLAB project, then the job includes all tests in the root of your repository or in any of +# its subfolders. +# +# [3] https://www.mathworks.com/help/matlab/ref/runtests.html +# [4] https://www.mathworks.com/help/matlab/ref/matlab.unittest.testresult.assertsuccess.html +# [5] https://www.mathworks.com/help/matlab/projects.html + +# The `test_artifacts_job` runs your tests and additionally generates test and coverage artifacts. +# It uses the plugin classes in the [`matlab.unittest.plugins`][6] package to generate a JUnit test results +# report and a Cobertura code coverage report. Like the `run_tests` job, this job runs all the tests in your +# project and fails the build if any of the tests fail. +# +test_artifacts_job: + script: | + matlab -batch " + import matlab.unittest.TestRunner + import matlab.unittest.Verbosity + import matlab.unittest.plugins.CodeCoveragePlugin + import matlab.unittest.plugins.XMLPlugin + import matlab.unittest.plugins.codecoverage.CoberturaFormat + + suite = testsuite(pwd,'IncludeSubfolders',true); + + [~,~] = mkdir('artifacts'); + + runner = TestRunner.withTextOutput('OutputDetail',Verbosity.Detailed); + runner.addPlugin(XMLPlugin.producingJUnitFormat('artifacts/results.xml')) + runner.addPlugin(CodeCoveragePlugin.forFolder(pwd,'IncludingSubfolders',true, ... + 'Producing',CoberturaFormat('artifacts/cobertura.xml'))) + + results = runner.run(suite) + assertSuccess(results);" + + artifacts: + reports: + junit: "./artifacts/results.xml" + cobertura: "./artifacts/cobertura.xml" + paths: + - "./artifacts" + +# You can modify the contents of the `test_artifacts_job` depending on your goals. For more +# information on how to customize the test runner and generate various test and coverage artifacts, +# see [Generate Artifacts Using MATLAB Unit Test Plugins][7]. +# +# [6] https://www.mathworks.com/help/matlab/ref/matlab.unittest.plugins-package.html +# [7] https://www.mathworks.com/help/matlab/matlab_prog/generate-artifacts-using-matlab-unit-test-plugins.html diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 6ed5e05ed4c..191d5b6b11c 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Pip's cache doesn't store the python packages -# https://pip.pypa.io/en/stable/reference/pip_install/#caching +# https://pip.pypa.io/en/stable/topics/caching/ # # If you want to also cache the installed packages, you have to install # them in a virtualenv and cache it as well. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index bd8ba71effe..b6e811aa84f 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -3,19 +3,36 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: API-Fuzzing.latest.gitlab-ci.yml +# +# You also need to add a `fuzz` stage to your `stages:` configuration. A sample configuration for API Fuzzing: +# +# stages: +# - build +# - test +# - deploy +# - fuzz + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ # -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# Configure API Fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: - FUZZAPI_VERSION: "1" + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + # + FUZZAPI_VERSION: "1" + FUZZAPI_IMAGE_SUFFIX: "" FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: stage: fuzz - image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX allow_failure: true rules: - if: $API_FUZZING_DISABLED @@ -23,6 +40,10 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + FUZZAPI_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-fuzz-api diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 65a2b20d5c0..66db311f897 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 container_scanning: - image: "$CS_ANALYZER_IMAGE" + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" stage: test variables: # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your @@ -47,4 +47,10 @@ container_scanning: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 0e0afa489a3..b491b3e3c0c 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -27,11 +27,12 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" + DAST_API_IMAGE_SUFFIX: "" DAST_API_IMAGE: api-fuzzing dast_api: stage: dast - image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true rules: - if: $DAST_API_DISABLED @@ -39,6 +40,10 @@ dast_api: - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-dast-api @@ -50,3 +55,5 @@ dast_api: - gl-*.log reports: dast: gl-dast-api-report.json + +# end diff --git a/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml new file mode 100644 index 00000000000..8a0913e8f66 --- /dev/null +++ b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml @@ -0,0 +1,27 @@ +# Shopify Theme Kit is a CLI tool for Shopify Themes: https://shopify.github.io/themekit/ +# See the full usage of this template described in: https://medium.com/@gogl.alex/how-to-deploy-shopify-themes-automatically-1ac17ee1229c + +image: python:2 + +stages: + - deploy:staging + - deploy:production + +staging: + image: python:2 + stage: deploy:staging + script: + - curl -s https://shopify.github.io/themekit/scripts/install.py | python + - theme deploy --env=staging + only: + variables: + - $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH + +production: + image: python:2 + stage: deploy:production + script: + - curl -s https://shopify.github.io/themekit/scripts/install.py | python + - theme deploy --env=production --allow-live + only: + - tags diff --git a/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml b/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml new file mode 100644 index 00000000000..18d59035b78 --- /dev/null +++ b/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml @@ -0,0 +1,149 @@ +# This file is a template, and might need editing before it works on your project. +# Here is a live project example that is using this template: +# https://gitlab.com/szandany/h2 + +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/liquibase.gitlab-ci.yml + +# This template must be configured with CI/CD variables before it will work. +# See https://www.liquibase.com/blog/secure-database-developer-flow-using-gitlab-pipelines +# to learn how to configure the Liquibase template by using variables. +# Be sure to add the variables before running pipelines with this template. +# You may not want to run all the jobs in this template. You can comment out or delete the jobs you don't wish to use. + +# List of stages for jobs and their order of execution. +stages: + - build + - test + - deploy + - compare + + +# Helper functions to determine if the database is ready for deployments (function isUpToDate) or rollbacks (function isRollback) when tag is applied. +.functions: &functions | + function isUpToDate(){ + status=$(liquibase status --verbose) + if [[ $status == *'is up to date'* ]]; then + echo "database is already up to date" & exit 0 + fi; + } + + function isRollback(){ + if [ -z "$TAG" ]; then + echo "No TAG provided, running any pending changes" + elif [[ "$(liquibase rollbackSQL $TAG)" ]]; then + liquibase --logLevel=info --logFile=${CI_JOB_NAME}_${CI_PIPELINE_ID}.log rollback $TAG && exit 0 + else exit 0 + fi; + } + + +# This is a series of Liquibase commands that can be run while doing database migrations from Liquibase docs at https://docs.liquibase.com/commands/home.html +.liquibase_job: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image at - https://hub.docker.com/r/liquibase/liquibase + before_script: + - liquibase --version + - *functions + - isRollback + - isUpToDate + - liquibase checks run + - liquibase update + - liquibase rollbackOneUpdate --force # This is a Pro command. Try Pro free trial here - https://liquibase.org/try-liquibase-pro-free + - liquibase tag $CI_PIPELINE_ID + - liquibase --logFile=${CI_JOB_NAME}_${CI_PIPELINE_ID}.log --logLevel=info update + - liquibase history + artifacts: + paths: + - ${CI_JOB_NAME}_${CI_PIPELINE_ID}.log + expire_in: 1 week + + +# This job runs in the build stage, which runs first. +build-job: + extends: .liquibase_job + stage: build + environment: + name: DEV + script: + - echo "This job tested successfully with liquibase in DEV environment" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job runs in the test stage. It only starts when the job in the build stage completes successfully. +test-job: + extends: .liquibase_job + stage: test + environment: + name: TEST + script: + - echo "This job testsed successfully with liquibase in TEST environment" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job runs in the deploy stage. It only starts when the jobs in the test stage completes successfully. +deploy-prod: + extends: .liquibase_job + stage: deploy + environment: + name: PROD + script: + - echo "This job deployed successfully Liquibase in a production environment from the $CI_COMMIT_BRANCH branch." + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + + +# This job compares dev database with test database to detect any drifts in the pipeline. Learn more about comparing database with Liquibase here https://docs.liquibase.com/commands/diff.html +DEV->TEST: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: compare + environment: + name: TEST + script: + - echo "Comparing databases DEV --> TEST" + - liquibase diff + - liquibase --outputFile=diff_between_DEV_TEST.json diff --format=json + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - diff_between_DEV_TEST.json + expire_in: 1 week + + +# This job compares test database with prod database to detect any drifts in the pipeline. +TEST->PROD: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: compare + environment: + name: PROD + script: + - echo "Comparing databases TEST --> PROD" + - liquibase diff + - liquibase --outputFile=diff_between_TEST_PROD.json diff --format=json + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - diff_between_TEST_PROD.json + expire_in: 1 week + + +# This job creates a snapshot of prod database. You can use the snapshot file to run comparisons with the production database to investigate for any potential issues. https://www.liquibase.com/devsecops +snapshot PROD: + image: liquibase/liquibase:latest # Using the Liquibase Docker Image + stage: .post + environment: + name: PROD + script: + - echo "Snapshotting database PROD" + - liquibase --outputFile=snapshot_PROD_${CI_PIPELINE_ID}.json snapshot --snapshotFormat=json --log-level debug + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + paths: + - snapshot_PROD_${CI_PIPELINE_ID}.json + expire_in: 1 week diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index bfcf67693e7..bcb1fe83ea2 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -10,7 +10,7 @@ module Gitlab @pipeline = pipeline @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) - @group_variables_builder = Builder::Group.new(project.group) + @group_variables_builder = Builder::Group.new(project&.group) end def scoped_variables(job, environment:, dependencies:) @@ -24,11 +24,25 @@ module Gitlab variables.concat(user_variables(job.user)) variables.concat(job.dependency_variables) if dependencies variables.concat(secret_instance_variables) - variables.concat(secret_group_variables(environment: environment, ref: job.git_ref)) - variables.concat(secret_project_variables(environment: environment, ref: job.git_ref)) + variables.concat(secret_group_variables(environment: environment)) + variables.concat(secret_project_variables(environment: environment)) variables.concat(job.trigger_request.user_variables) if job.trigger_request variables.concat(pipeline.variables) - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule + variables.concat(pipeline_schedule_variables) + end + end + + def config_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless project + + variables.concat(project.predefined_variables) + variables.concat(pipeline.predefined_variables) + variables.concat(secret_instance_variables) + variables.concat(secret_group_variables(environment: nil)) + variables.concat(secret_project_variables(environment: nil)) + variables.concat(pipeline.variables) + variables.concat(pipeline_schedule_variables) end end @@ -75,21 +89,21 @@ module Gitlab end end - def secret_group_variables(environment:, ref:) - if memoize_secret_variables? - memoized_secret_group_variables(environment: environment) - else - return [] unless project.group - - project.group.ci_variables_for(ref, project, environment: environment) + def secret_group_variables(environment:) + strong_memoize_with(:secret_group_variables, environment) do + group_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) end end - def secret_project_variables(environment:, ref:) - if memoize_secret_variables? - memoized_secret_project_variables(environment: environment) - else - project.ci_variables_for(ref: ref, environment: environment) + def secret_project_variables(environment:) + strong_memoize_with(:secret_project_variables, environment) do + project_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) end end @@ -120,21 +134,15 @@ module Gitlab end end - def memoized_secret_project_variables(environment:) - strong_memoize_with(:secret_project_variables, environment) do - project_variables_builder - .secret_variables( - environment: environment, - protected_ref: protected_ref?) - end - end + def pipeline_schedule_variables + strong_memoize(:pipeline_schedule_variables) do + variables = if pipeline.pipeline_schedule + pipeline.pipeline_schedule.job_variables + else + [] + end - def memoized_secret_group_variables(environment:) - strong_memoize_with(:secret_group_variables, environment) do - group_variables_builder - .secret_variables( - environment: environment, - protected_ref: protected_ref?) + Gitlab::Ci::Variables::Collection.new(variables) end end @@ -150,14 +158,6 @@ module Gitlab end end - def memoize_secret_variables? - strong_memoize(:memoize_secret_variables) do - ::Feature.enabled?(:ci_variables_builder_memoize_secret_variables, - project, - default_enabled: :yaml) - end - end - def strong_memoize_with(name, *args) container = strong_memoize(name) { {} } diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 0d4b913b7a0..22a4ba8ac7a 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -22,7 +22,7 @@ module Gitlab 'frame_src' => ContentSecurityPolicy::Directives.frame_src, 'img_src' => "'self' data: blob: http: https:", 'manifest_src' => "'self'", - 'media_src' => "'self'", + 'media_src' => "'self' data:", 'script_src' => ContentSecurityPolicy::Directives.script_src, 'style_src' => "'self' 'unsafe-inline'", 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", @@ -37,13 +37,13 @@ module Gitlab allow_webpack_dev_server(directives) allow_letter_opener(directives) allow_snowplow_micro(directives) if Gitlab::Tracking.snowplow_micro_enabled? - allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? end allow_websocket_connections(directives) allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn allow_framed_gitlab_paths(directives) + allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? # The follow section contains workarounds to patch Safari's lack of support for CSP Level 3 # See https://gitlab.com/gitlab-org/gitlab/-/issues/343579 diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index a4508bc93c5..0e6841e10a7 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -12,6 +12,16 @@ module Gitlab Gitlab::UrlBuilder.build(deployment.deployable) end + commit_url = + if (commit = deployment.commit) + Gitlab::UrlBuilder.build(commit) + end + + user_url = + if deployment.deployed_by + Gitlab::UrlBuilder.build(deployment.deployed_by) + end + { object_kind: 'deployment', status: deployment.status, @@ -22,10 +32,10 @@ module Gitlab environment: deployment.environment.name, project: deployment.project.hook_attrs, short_sha: deployment.short_sha, - user: deployment.deployed_by.hook_attrs, - user_url: Gitlab::UrlBuilder.build(deployment.deployed_by), - commit_url: Gitlab::UrlBuilder.build(deployment.commit), - commit_title: deployment.commit.title, + user: deployment.deployed_by&.hook_attrs, + user_url: user_url, + commit_url: commit_url, + commit_title: deployment.commit_title, ref: deployment.ref } end diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 73518d36d43..dec583f5a42 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -43,10 +43,9 @@ module Gitlab if note.for_commit? data[:commit] = build_data_for_commit(project, user, note) elsif note.for_issue? - data[:issue] = note.noteable.hook_attrs - data[:issue][:labels] = note.noteable.labels_hook_attrs + data[:issue] = Gitlab::HookData::IssueBuilder.new(note.noteable).build elsif note.for_merge_request? - data[:merge_request] = note.noteable.hook_attrs + data[:merge_request] = Gitlab::HookData::MergeRequestBuilder.new(note.noteable).build elsif note.for_snippet? data[:snippet] = note.noteable.hook_attrs end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 1b16873f737..1895f0fab32 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -161,24 +161,6 @@ module Gitlab end end - def self.nulls_order(field, direction = :asc, nulls_order = :nulls_last) - raise ArgumentError unless [:nulls_last, :nulls_first].include?(nulls_order) - raise ArgumentError unless [:asc, :desc].include?(direction) - - case nulls_order - when :nulls_last then nulls_last_order(field, direction) - when :nulls_first then nulls_first_order(field, direction) - end - end - - def self.nulls_last_order(field, direction = 'ASC') - Arel.sql("#{field} #{direction} NULLS LAST") - end - - def self.nulls_first_order(field, direction = 'ASC') - Arel.sql("#{field} #{direction} NULLS FIRST") - end - def self.random "RANDOM()" end @@ -228,7 +210,7 @@ module Gitlab end def self.db_config_names - ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) + ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) - ['geo'] end # This returns all matching schemas that a given connection can use @@ -236,13 +218,16 @@ module Gitlab # This does not look at literal connection names, but rather compares # models that are holders for a given db_config_name def self.gitlab_schemas_for_connection(connection) - connection_name = self.db_config_name(connection) - primary_model = self.database_base_models.fetch(connection_name) - - self.schemas_to_base_models - .select { |_, models| models.include?(primary_model) } - .keys - .map!(&:to_sym) + db_name = self.db_config_name(connection) + primary_model = self.database_base_models.fetch(db_name.to_sym) + + self.schemas_to_base_models.select do |_, child_models| + child_models.any? do |child_model| + child_model == primary_model || \ + # The model might indicate a child connection, ensure that this is enclosed in a `db_config` + self.database_base_models[self.db_config_share_with(child_model.connection_db_config)] == primary_model + end + end.keys.map!(&:to_sym) end def self.db_config_for_connection(connection) @@ -271,6 +256,17 @@ module Gitlab db_config&.name || 'unknown' end + # Currently the database configuration can only be shared with `main:` + # If the `database_tasks: false` is being used + # This is to be refined: https://gitlab.com/gitlab-org/gitlab/-/issues/356580 + def self.db_config_share_with(db_config) + if db_config.database_tasks? + nil # no sharing + else + 'main' # share with `main:` + end + end + def self.read_only? false end diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb index 3e6d7ac3c9f..14fe0c14c24 100644 --- a/lib/gitlab/database/background_migration/batch_metrics.rb +++ b/lib/gitlab/database/background_migration/batch_metrics.rb @@ -5,17 +5,24 @@ module Gitlab module BackgroundMigration class BatchMetrics attr_reader :timings + attr_reader :affected_rows def initialize @timings = {} + @affected_rows = {} end - def time_operation(label) + def time_operation(label, &blk) + instrument_operation(label, instrument_affected_rows: false, &blk) + end + + def instrument_operation(label, instrument_affected_rows: true) start_time = monotonic_time - yield + count = yield timings_for_label(label) << monotonic_time - start_time + affected_rows_for_label(label) << count if instrument_affected_rows && count.is_a?(Integer) end private @@ -24,6 +31,10 @@ module Gitlab timings[label] ||= [] end + def affected_rows_for_label(label) + affected_rows[label] ||= [] + end + def monotonic_time Gitlab::Metrics::System.monotonic_time end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index f3160679d64..ebc3ee240bd 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -25,6 +25,7 @@ module Gitlab scope :except_succeeded, -> { without_status(:succeeded) } scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) } scope :with_preloads, -> { preload(:batched_migration) } + scope :created_since, ->(date_time) { where('created_at >= ?', date_time) } state_machine :status, initial: :pending do state :pending, value: 0 @@ -62,7 +63,13 @@ module Gitlab job.split_and_retry! if job.can_split?(exception) rescue SplitAndRetryError => error - Gitlab::AppLogger.error(message: error.message, batched_job_id: job.id) + Gitlab::AppLogger.error( + message: error.message, + batched_job_id: job.id, + batched_migration_id: job.batched_migration.id, + job_class_name: job.migration_job_class_name, + job_arguments: job.migration_job_arguments + ) end after_transition do |job, transition| @@ -72,13 +79,23 @@ module Gitlab job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message) - Gitlab::ErrorTracking.track_exception(exception, batched_job_id: job.id) if exception - - Gitlab::AppLogger.info(message: 'BatchedJob transition', batched_job_id: job.id, previous_state: transition.from_name, new_state: transition.to_name) + Gitlab::ErrorTracking.track_exception(exception, batched_job_id: job.id, job_class_name: job.migration_job_class_name, job_arguments: job.migration_job_arguments) if exception + + Gitlab::AppLogger.info( + message: 'BatchedJob transition', + batched_job_id: job.id, + previous_state: transition.from_name, + new_state: transition.to_name, + batched_migration_id: job.batched_migration.id, + job_class_name: job.migration_job_class_name, + job_arguments: job.migration_job_arguments, + exception_class: exception&.class, + exception_message: exception&.message + ) end end - delegate :job_class, :table_name, :column_name, :job_arguments, + delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name, to: :batched_migration, prefix: :migration attribute :pause_ms, :integer, default: 100 diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 65c15795de6..d94bf060d05 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -6,6 +6,8 @@ module Gitlab class BatchedMigration < SharedModel JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" + MAXIMUM_FAILED_RATIO = 0.5 + MINIMUM_JOBS = 50 self.table_name = :batched_background_migrations @@ -21,28 +23,60 @@ module Gitlab validate :validate_batched_jobs_status, if: -> { status_changed? && finished? } scope :queue_order, -> { order(id: :asc) } - scope :queued, -> { where(status: [:active, :paused]) } + scope :queued, -> { with_statuses(:active, :paused) } + + # on_hold_until is a temporary runtime status which puts execution "on hold" + scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') } + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals end - enum status: { - paused: 0, - active: 1, - finished: 3, - failed: 4, - finalizing: 5 - } + state_machine :status, initial: :paused do + state :paused, value: 0 + state :active, value: 1 + state :finished, value: 3 + state :failed, value: 4 + state :finalizing, value: 5 + + event :pause do + transition any => :paused + end + + event :execute do + transition any => :active + end + + event :finish do + transition any => :finished + end + + event :failure do + transition any => :failed + end + + event :finalize do + transition any => :finalizing + end + + before_transition any => :active do |migration| + migration.started_at = Time.current if migration.respond_to?(:started_at) + end + end attribute :pause_ms, :integer, default: 100 + def self.valid_status + state_machine.states.map(&:name) + end + def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) for_configuration(job_class_name, table_name, column_name, job_arguments).first end def self.active_migration - active.queue_order.first + executable.queue_order.first end def self.successful_rows_counts(migrations) @@ -74,11 +108,23 @@ module Gitlab batched_jobs.with_status(:failed).each_batch(of: 100) do |batch| self.class.transaction do batch.lock.each(&:split_and_retry!) - self.active! + self.execute! end end - self.active! + self.execute! + end + + def should_stop? + return unless started_at + + total_jobs = batched_jobs.created_since(started_at).count + + return if total_jobs < MINIMUM_JOBS + + failed_jobs = batched_jobs.with_status(:failed).created_since(started_at).count + + failed_jobs.fdiv(total_jobs) > MAXIMUM_FAILED_RATIO end def next_min_value @@ -136,6 +182,10 @@ module Gitlab BatchOptimizer.new(self).optimize! end + def hold!(until_time: 10.minutes.from_now) + update!(on_hold_until: until_time) + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 06cd40f1e06..59ff9a9744f 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -6,13 +6,13 @@ module Gitlab class BatchedMigrationRunner FailedToFinalize = Class.new(RuntimeError) - def self.finalize(job_class_name, table_name, column_name, job_arguments, connection: ApplicationRecord.connection) + def self.finalize(job_class_name, table_name, column_name, job_arguments, connection:) new(connection: connection).finalize(job_class_name, table_name, column_name, job_arguments) end - def initialize(migration_wrapper = BatchedMigrationWrapper.new, connection: ApplicationRecord.connection) - @migration_wrapper = migration_wrapper + def initialize(connection:, migration_wrapper: BatchedMigrationWrapper.new(connection: connection)) @connection = connection + @migration_wrapper = migration_wrapper end # Runs the next batched_job for a batched_background_migration. @@ -30,6 +30,7 @@ module Gitlab migration_wrapper.perform(next_batched_job) active_migration.optimize! + active_migration.failure! if next_batched_job.failed? && active_migration.should_stop? else finish_active_migration(active_migration) end @@ -67,7 +68,7 @@ module Gitlab elsif migration.finished? Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" else - migration.finalizing! + migration.finalize! migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) } run_migration_while(migration, :finalizing) @@ -78,7 +79,7 @@ module Gitlab private - attr_reader :migration_wrapper, :connection + attr_reader :connection, :migration_wrapper def find_or_create_next_batched_job(active_migration) if next_batch_range = find_next_batch_range(active_migration) @@ -118,14 +119,14 @@ module Gitlab return if active_migration.batched_jobs.active.exists? if active_migration.batched_jobs.with_status(:failed).exists? - active_migration.failed! + active_migration.failure! else - active_migration.finished! + active_migration.finish! end end def run_migration_while(migration, status) - while migration.status == status.to_s + while migration.status_name == status run_migration_job(migration) migration.reload_last_job diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index 057f856d859..ec68f401ca2 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -4,10 +4,9 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationWrapper - extend Gitlab::Utils::StrongMemoize - - def initialize(connection: ApplicationRecord.connection) + def initialize(connection:, metrics: PrometheusMetrics.new) @connection = connection + @metrics = metrics end # Wraps the execution of a batched_background_migration. @@ -28,12 +27,12 @@ module Gitlab raise ensure - track_prometheus_metrics(batch_tracking_record) + metrics.track(batch_tracking_record) end private - attr_reader :connection + attr_reader :connection, :metrics def start_tracking_execution(tracking_record) tracking_record.run! @@ -63,80 +62,6 @@ module Gitlab job_class.new end end - - def track_prometheus_metrics(tracking_record) - migration = tracking_record.batched_migration - base_labels = migration.prometheus_labels - - metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size) - metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size) - metric_for(:gauge_interval).set(base_labels, tracking_record.batched_migration.interval) - metric_for(:gauge_job_duration).set(base_labels, (tracking_record.finished_at - tracking_record.started_at).to_i) - metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size) - metric_for(:gauge_migrated_tuples).set(base_labels, tracking_record.batched_migration.migrated_tuple_count) - metric_for(:gauge_total_tuple_count).set(base_labels, tracking_record.batched_migration.total_tuple_count) - metric_for(:gauge_last_update_time).set(base_labels, Time.current.to_i) - - if metrics = tracking_record.metrics - metrics['timings']&.each do |key, timings| - summary = metric_for(:histogram_timings) - labels = base_labels.merge(operation: key) - - timings.each do |timing| - summary.observe(labels, timing) - end - end - end - end - - def metric_for(name) - self.class.metrics[name] - end - - def self.metrics - strong_memoize(:metrics) do - { - gauge_batch_size: Gitlab::Metrics.gauge( - :batched_migration_job_batch_size, - 'Batch size for a batched migration job' - ), - gauge_sub_batch_size: Gitlab::Metrics.gauge( - :batched_migration_job_sub_batch_size, - 'Sub-batch size for a batched migration job' - ), - gauge_interval: Gitlab::Metrics.gauge( - :batched_migration_job_interval_seconds, - 'Interval for a batched migration job' - ), - gauge_job_duration: Gitlab::Metrics.gauge( - :batched_migration_job_duration_seconds, - 'Duration for a batched migration job' - ), - counter_updated_tuples: Gitlab::Metrics.counter( - :batched_migration_job_updated_tuples_total, - 'Number of tuples updated by batched migration job' - ), - gauge_migrated_tuples: Gitlab::Metrics.gauge( - :batched_migration_migrated_tuples_total, - 'Total number of tuples migrated by a batched migration' - ), - histogram_timings: Gitlab::Metrics.histogram( - :batched_migration_job_query_duration_seconds, - 'Query timings for a batched migration job', - {}, - [0.1, 0.25, 0.5, 1, 5].freeze - ), - gauge_total_tuple_count: Gitlab::Metrics.gauge( - :batched_migration_total_tuple_count, - 'Total tuple count the migration needs to touch' - ), - gauge_last_update_time: Gitlab::Metrics.gauge( - :batched_migration_last_update_time_seconds, - 'Unix epoch time in seconds' - ) - } - end - end end end end diff --git a/lib/gitlab/database/background_migration/prometheus_metrics.rb b/lib/gitlab/database/background_migration/prometheus_metrics.rb new file mode 100644 index 00000000000..ce1da4c59eb --- /dev/null +++ b/lib/gitlab/database/background_migration/prometheus_metrics.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class PrometheusMetrics + extend Gitlab::Utils::StrongMemoize + + QUERY_TIMING_BUCKETS = [0.1, 0.25, 0.5, 1, 5].freeze + + def track(job_record) + migration_record = job_record.batched_migration + base_labels = migration_record.prometheus_labels + + metric_for(:gauge_batch_size).set(base_labels, job_record.batch_size) + metric_for(:gauge_sub_batch_size).set(base_labels, job_record.sub_batch_size) + metric_for(:gauge_interval).set(base_labels, job_record.batched_migration.interval) + metric_for(:gauge_job_duration).set(base_labels, (job_record.finished_at - job_record.started_at).to_i) + metric_for(:counter_updated_tuples).increment(base_labels, job_record.batch_size) + metric_for(:gauge_migrated_tuples).set(base_labels, migration_record.migrated_tuple_count) + metric_for(:gauge_total_tuple_count).set(base_labels, migration_record.total_tuple_count) + metric_for(:gauge_last_update_time).set(base_labels, Time.current.to_i) + + track_timing_metrics(base_labels, job_record.metrics) + end + + def self.metrics + strong_memoize(:metrics) do + { + gauge_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_batch_size, + 'Batch size for a batched migration job' + ), + gauge_sub_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_sub_batch_size, + 'Sub-batch size for a batched migration job' + ), + gauge_interval: Gitlab::Metrics.gauge( + :batched_migration_job_interval_seconds, + 'Interval for a batched migration job' + ), + gauge_job_duration: Gitlab::Metrics.gauge( + :batched_migration_job_duration_seconds, + 'Duration for a batched migration job' + ), + counter_updated_tuples: Gitlab::Metrics.counter( + :batched_migration_job_updated_tuples_total, + 'Number of tuples updated by batched migration job' + ), + gauge_migrated_tuples: Gitlab::Metrics.gauge( + :batched_migration_migrated_tuples_total, + 'Total number of tuples migrated by a batched migration' + ), + histogram_timings: Gitlab::Metrics.histogram( + :batched_migration_job_query_duration_seconds, + 'Query timings for a batched migration job', + {}, + QUERY_TIMING_BUCKETS + ), + gauge_total_tuple_count: Gitlab::Metrics.gauge( + :batched_migration_total_tuple_count, + 'Total tuple count the migration needs to touch' + ), + gauge_last_update_time: Gitlab::Metrics.gauge( + :batched_migration_last_update_time_seconds, + 'Unix epoch time in seconds' + ) + } + end + end + + private + + def track_timing_metrics(base_labels, metrics) + return unless metrics && metrics['timings'] + + metrics['timings'].each do |key, timings| + summary = metric_for(:histogram_timings) + labels = base_labels.merge(operation: key) + + timings.each do |timing| + summary.observe(labels, timing) + end + end + end + + def metric_for(name) + self.class.metrics[name] + end + end + end + end +end diff --git a/lib/gitlab/database/consistency_checker.rb b/lib/gitlab/database/consistency_checker.rb new file mode 100644 index 00000000000..e398fef744c --- /dev/null +++ b/lib/gitlab/database/consistency_checker.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConsistencyChecker + BATCH_SIZE = 1000 + MAX_BATCHES = 25 + MAX_RUNTIME = 30.seconds # must be less than the scheduling frequency of the ConsistencyCheck jobs + + delegate :monotonic_time, to: :'Gitlab::Metrics::System' + + def initialize(source_model:, target_model:, source_columns:, target_columns:) + @source_model = source_model + @target_model = target_model + @source_columns = source_columns + @target_columns = target_columns + @source_sort_column = source_columns.first + @target_sort_column = target_columns.first + @result = { matches: 0, mismatches: 0, batches: 0, mismatches_details: [] } + end + + # rubocop:disable Metrics/AbcSize + def execute(start_id:) + current_start_id = start_id + + return build_result(next_start_id: nil) if max_id.nil? + return build_result(next_start_id: min_id) if current_start_id > max_id + + @start_time = monotonic_time + + MAX_BATCHES.times do + if (current_start_id <= max_id) && !over_time_limit? + ids_range = current_start_id...(current_start_id + BATCH_SIZE) + # rubocop: disable CodeReuse/ActiveRecord + source_data = source_model.where(source_sort_column => ids_range) + .order(source_sort_column => :asc).pluck(*source_columns) + target_data = target_model.where(target_sort_column => ids_range) + .order(target_sort_column => :asc).pluck(*target_columns) + # rubocop: enable CodeReuse/ActiveRecord + + current_start_id += BATCH_SIZE + result[:matches] += append_mismatches_details(source_data, target_data) + result[:batches] += 1 + else + break + end + end + + result[:mismatches] = result[:mismatches_details].length + metrics_counter.increment({ source_table: source_model.table_name, result: "match" }, result[:matches]) + metrics_counter.increment({ source_table: source_model.table_name, result: "mismatch" }, result[:mismatches]) + + build_result(next_start_id: current_start_id > max_id ? min_id : current_start_id) + end + # rubocop:enable Metrics/AbcSize + + private + + attr_reader :source_model, :target_model, :source_columns, :target_columns, + :source_sort_column, :target_sort_column, :start_time, :result + + def build_result(next_start_id:) + { next_start_id: next_start_id }.merge(result) + end + + def over_time_limit? + (monotonic_time - start_time) >= MAX_RUNTIME + end + + # This where comparing the items happen, and building the diff log + # It returns the number of matching elements + def append_mismatches_details(source_data, target_data) + # Mapping difference the sort key to the item values + # source - target + source_diff_hash = (source_data - target_data).index_by { |item| item.shift } + # target - source + target_diff_hash = (target_data - source_data).index_by { |item| item.shift } + + matches = source_data.length - source_diff_hash.length + + # Items that exist in the first table + Different items + source_diff_hash.each do |id, values| + result[:mismatches_details] << { + id: id, + source_table: values, + target_table: target_diff_hash[id] + } + end + + # Only the items that exist in the target table + target_diff_hash.each do |id, values| + next if source_diff_hash[id] # It's already added + + result[:mismatches_details] << { + id: id, + source_table: source_diff_hash[id], + target_table: values + } + end + + matches + end + + # rubocop: disable CodeReuse/ActiveRecord + def min_id + @min_id ||= source_model.minimum(source_sort_column) + end + + def max_id + @max_id ||= source_model.maximum(source_sort_column) + end + # rubocop: enable CodeReuse/ActiveRecord + + def metrics_counter + @metrics_counter ||= Gitlab::Metrics.counter( + :consistency_checks, + "Consistency Check Results" + ) + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index cccd4b48723..0d876f5124f 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -4,11 +4,13 @@ module Gitlab module Database module EachDatabase class << self - def each_database_connection(only: nil) + def each_database_connection(only: nil, include_shared: true) selected_names = Array.wrap(only) base_models = select_base_models(selected_names) base_models.each_pair do |connection_name, model| + next if !include_shared && Gitlab::Database.db_config_share_with(model.connection_db_config) + connection = model.connection with_shared_connection(connection, connection_name) do diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index dcd78bfd84f..ae0ea919b62 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -42,11 +42,11 @@ audit_events: :gitlab_main authentication_events: :gitlab_main award_emoji: :gitlab_main aws_roles: :gitlab_main -background_migration_jobs: :gitlab_main +background_migration_jobs: :gitlab_shared badges: :gitlab_main banned_users: :gitlab_main -batched_background_migration_jobs: :gitlab_main -batched_background_migrations: :gitlab_main +batched_background_migration_jobs: :gitlab_shared +batched_background_migrations: :gitlab_shared board_assignees: :gitlab_main board_group_recent_visits: :gitlab_main board_labels: :gitlab_main @@ -240,6 +240,7 @@ group_deletion_schedules: :gitlab_main group_deploy_keys: :gitlab_main group_deploy_keys_groups: :gitlab_main group_deploy_tokens: :gitlab_main +group_features: :gitlab_main group_group_links: :gitlab_main group_import_states: :gitlab_main group_merge_request_approval_settings: :gitlab_main @@ -393,7 +394,7 @@ postgres_indexes: :gitlab_shared postgres_partitioned_tables: :gitlab_shared postgres_partitions: :gitlab_shared postgres_reindex_actions: :gitlab_shared -postgres_reindex_queued_actions: :gitlab_main +postgres_reindex_queued_actions: :gitlab_shared product_analytics_events_experimental: :gitlab_main programming_languages: :gitlab_main project_access_tokens: :gitlab_main @@ -435,6 +436,7 @@ protected_branches: :gitlab_main protected_branch_merge_access_levels: :gitlab_main protected_branch_push_access_levels: :gitlab_main protected_branch_unprotect_access_levels: :gitlab_main +protected_environment_approval_rules: :gitlab_main protected_environment_deploy_access_levels: :gitlab_main protected_environments: :gitlab_main protected_tag_create_access_levels: :gitlab_main @@ -558,4 +560,4 @@ x509_commit_signatures: :gitlab_main x509_issuers: :gitlab_main zentao_tracker_data: :gitlab_main zoom_meetings: :gitlab_main -batched_background_migration_job_transition_logs: :gitlab_main +batched_background_migration_job_transition_logs: :gitlab_shared diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 86b3afaa47b..3f03d9e2c12 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -78,15 +78,15 @@ module Gitlab end def primary_model_or_model_if_enabled - if force_no_sharing_primary_model? + if use_dedicated_connection? @model else @primary_model || @model end end - def force_no_sharing_primary_model? - return false unless @primary_model # Doesn't matter since we don't have an overriding primary model + def use_dedicated_connection? + return true unless @primary_model # We can only use dedicated connection, if re-use of connections is disabled return false unless ::Gitlab::SafeRequestStore.active? ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index a91df2eccdd..1be63da8896 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -13,13 +13,6 @@ module Gitlab WriteInsideReadOnlyTransactionError = Class.new(StandardError) READ_ONLY_TRANSACTION_KEY = :load_balacing_read_only_transaction - # The load balancer returned by connection might be different - # between `model.connection.load_balancer` vs `model.load_balancer` - # - # The used `model.connection` is dependent on `use_model_load_balancing`. - # See more in: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73949. - # - # Always use `model.load_balancer` or `model.sticking`. attr_reader :load_balancer # These methods perform writes after which we need to stick to the diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index 6d667e8ecf0..eceea1d8d9c 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -17,7 +17,12 @@ module Gitlab configure_connection setup_connection_proxy setup_service_discovery - setup_feature_flag_to_model_load_balancing + + ::Gitlab::Database::LoadBalancing::Logger.debug( + event: :setup, + model: model.name, + start_service_discovery: @start_service_discovery + ) end def configure_connection @@ -45,21 +50,6 @@ module Gitlab setup_class_attribute(:sticking, Sticking.new(load_balancer)) end - # TODO: This is temporary code to gradually redirect traffic to use - # a dedicated DB replicas, or DB primaries (depending on configuration) - # This implements a sticky behavior for the current request if enabled. - # - # This is needed for Phase 3 and Phase 4 of application rollout - # https://gitlab.com/groups/gitlab-org/-/epics/6160#progress - # - # If `GITLAB_USE_MODEL_LOAD_BALANCING` is set, its value is preferred - # Otherwise, a `use_model_load_balancing` FF value is used - def setup_feature_flag_to_model_load_balancing - return if active_record_base? - - @model.singleton_class.prepend(ModelLoadBalancingFeatureFlagMixin) - end - def setup_service_discovery return unless configuration.service_discovery_enabled? @@ -84,31 +74,6 @@ module Gitlab def active_record_base? @model == ActiveRecord::Base end - - module ModelLoadBalancingFeatureFlagMixin - extend ActiveSupport::Concern - - def use_model_load_balancing? - # Cache environment variable and return env variable first if defined - default_use_model_load_balancing_env = Gitlab.dev_or_test_env? || nil - use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_USE_MODEL_LOAD_BALANCING', default_use_model_load_balancing_env)) - - unless use_model_load_balancing_env.nil? - return use_model_load_balancing_env - end - - # Check a feature flag using RequestStore (if active) - return false unless Gitlab::SafeRequestStore.active? - - Gitlab::SafeRequestStore.fetch(:use_model_load_balancing) do - Feature.enabled?(:use_model_load_balancing, default_enabled: :yaml) - end - end - - def connection - use_model_load_balancing? ? super : ApplicationRecord.connection - end - end end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index b2248b0f4eb..dc695a74a4b 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -33,20 +33,33 @@ module Gitlab # We use major version bumps to indicate significant changes and minor version bumps # to indicate backwards-compatible or otherwise minor changes (e.g. a Rails version bump). # However, this hasn't been strictly formalized yet. - MIGRATION_CLASSES = { - 1.0 => Class.new(ActiveRecord::Migration[6.1]) do - include LockRetriesConcern - include Gitlab::Database::MigrationHelpers::V2 + + class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase + include LockRetriesConcern + include Gitlab::Database::MigrationHelpers::V2 + end + + class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema + + # When running migrations, the `db:migrate` switches connection of + # ActiveRecord::Base depending where the migration runs. + # This helper class is provided to avoid confusion using `ActiveRecord::Base` + class MigrationRecord < ActiveRecord::Base end - }.freeze + end def self.[](version) - MIGRATION_CLASSES[version] || raise(ArgumentError, "Unknown migration version: #{version}") + version = version.to_s + name = "V#{version.tr('.', '_')}" + raise ArgumentError, "Unknown migration version: #{version}" unless const_defined?(name, false) + + const_get(name, false) end # The current version to be used in new migrations def self.current_version - 1.0 + 2.0 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7602e09981a..d016dea224b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -692,6 +692,8 @@ module Gitlab # batch_column_name - option for tables without a primary key, in this case # another unique integer column can be used. Example: :user_id def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id, limit: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + temp_column = "#{column}_for_type_change" # Using a descriptive name that includes orinal column's name risks @@ -956,7 +958,7 @@ module Gitlab Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" elsif !migration.finished? raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ - "but it is '#{migration.status}':" \ + "but it is '#{migration.status_name}':" \ "\t#{configuration}" \ "\n\n" \ "Finalize it manualy by running" \ @@ -1639,7 +1641,9 @@ into similar problems in the future (e.g. when new tables are created). old_value = Arel::Nodes::NamedFunction.new(type_cast_function, [old_value]) end - update_column_in_batches(table, new, old_value, batch_column_name: batch_column_name) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + update_column_in_batches(table, new, old_value, batch_column_name: batch_column_name) + end add_not_null_constraint(table, new) unless old_col.null diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb index b4e31565c60..5a25128f3a9 100644 --- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -6,8 +6,6 @@ module Gitlab module RestrictGitlabSchema extend ActiveSupport::Concern - MigrationSkippedError = Class.new(StandardError) - included do class_attribute :allowed_gitlab_schemas end @@ -25,11 +23,8 @@ module Gitlab def migrate(direction) if unmatched_schemas.any? - # TODO: Today skipping migration would raise an exception. - # Ideally, skipped migration should be ignored (not loaded), or softly ignored. - # Read more in: https://gitlab.com/gitlab-org/gitlab/-/issues/355014 - raise MigrationSkippedError, "Current migration is skipped since it modifies "\ - "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + migration_skipped + return end Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do @@ -41,6 +36,11 @@ module Gitlab private + def migration_skipped + say "Current migration is skipped since it modifies "\ + "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + end + def validator_class Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index 0e7f6075196..dd426962033 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -134,6 +134,8 @@ module Gitlab # batch_column_name - option is for tables without primary key, in this # case another unique integer column can be used. Example: :user_id def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name) with_lock_retries do @@ -181,6 +183,8 @@ module Gitlab # case another unique integer column can be used. Example: :user_id # def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name) with_lock_retries do diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index a2a4a37ab87..0261ade0fe7 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -84,7 +84,7 @@ module Gitlab FROM #{connection.quote_table_name(batch_table_name)} SQL - migration_status = batch_max_value.nil? ? :finished : :active + status_event = batch_max_value.nil? ? :finish : :execute batch_max_value ||= batch_min_value migration = Gitlab::Database::BackgroundMigration::BatchedMigration.new( @@ -98,7 +98,7 @@ module Gitlab batch_class_name: batch_class_name, batch_size: batch_size, sub_batch_size: sub_batch_size, - status: migration_status + status_event: status_event ) # Below `BatchedMigration` attributes were introduced after the diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 9d28db6b886..7c21346007a 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -6,11 +6,8 @@ module Gitlab class Instrumentation STATS_FILENAME = 'migration-stats.json' - attr_reader :observations - def initialize(result_dir:, observer_classes: ::Gitlab::Database::Migrations::Observers.all_observers) @observer_classes = observer_classes - @observations = [] @result_dir = result_dir end @@ -38,15 +35,16 @@ module Gitlab on_each_observer(observers) { |observer| observer.after } on_each_observer(observers) { |observer| observer.record } - record_observation(observation) + record_observation(observation, destination_dir: per_migration_result_dir) end private attr_reader :observer_classes - def record_observation(observation) - @observations << observation + def record_observation(observation, destination_dir:) + stats_file_location = File.join(destination_dir, STATS_FILENAME) + File.write(stats_file_location, observation.to_json) end def on_each_observer(observers, &block) diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 02645a0d452..3b6f52b43a8 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -6,7 +6,7 @@ module Gitlab class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' - SCHEMA_VERSION = 2 # Version of the output format produced by the runner + SCHEMA_VERSION = 3 # Version of the output format produced by the runner class << self def up @@ -17,6 +17,10 @@ module Gitlab Runner.new(direction: :down, migrations: migrations_for_down, result_dir: BASE_RESULT_DIR.join('down')) end + def background_migrations + TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) + end + def migration_context @migration_context ||= ApplicationRecord.connection.migration_context end @@ -76,13 +80,8 @@ module Gitlab end end ensure - if instrumentation - stats_filename = File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME) - File.write(stats_filename, instrumentation.observations.to_json) - - metadata_filename = File.join(result_dir, METADATA_FILENAME) - File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) - end + metadata_filename = File.join(result_dir, METADATA_FILENAME) + File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks # This clearing makes subsequent rake tasks in the same execution pick up database schema changes caused by diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 821d68c06c9..74e54d62e05 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -4,12 +4,10 @@ module Gitlab module Database module Migrations class TestBackgroundRunner - # TODO - build a rake task to call this method, and support it in the gitlab-com-database-testing project. - # Until then, we will inject a migration with a very high timestamp during database testing - # that calls this class to run jobs - # See https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing/-/issues/41 for details + attr_reader :result_dir - def initialize + def initialize(result_dir:) + @result_dir = result_dir @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end @@ -24,18 +22,30 @@ module Gitlab # without .to_f, we do integer division # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) duration_per_migration_type = for_duration / jobs_to_run.count.to_f - jobs_to_run.each do |_migration_name, jobs| + jobs_to_run.each do |migration_name, jobs| run_until = duration_per_migration_type.from_now - jobs.shuffle.each do |j| - break if run_until <= Time.current - run_job(j) - end + run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) end end private + def run_jobs_for_migration(migration_name:, jobs:, run_until:) + per_background_migration_result_dir = File.join(@result_dir, migration_name) + + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} + + jobs.shuffle.each do |j| + break if run_until <= Time.current + + instrumentation.observe(version: nil, name: batch_names.next, connection: ActiveRecord::Migration.connection) do + run_job(j) + end + end + end + def run_job(job) Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index e56ffddac4f..034e18ec9f4 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -40,16 +40,20 @@ module Gitlab # 1. The minimum value for the partitioning column in the table # 2. If no data is present yet, the current month def partition_table_by_date(table_name, column_name, min_date: nil, max_date: nil) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + assert_table_is_allowed(table_name) assert_not_in_transaction_block(scope: ERROR_SCOPE) max_date ||= Date.today + 1.month - min_date ||= connection.select_one(<<~SQL)['minimum'] || max_date - 1.month - SELECT date_trunc('MONTH', MIN(#{column_name})) AS minimum - FROM #{table_name} - SQL + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + min_date ||= connection.select_one(<<~SQL)['minimum'] || max_date - 1.month + SELECT date_trunc('MONTH', MIN(#{column_name})) AS minimum + FROM #{table_name} + SQL + end raise "max_date #{max_date} must be greater than min_date #{min_date}" if min_date >= max_date @@ -154,6 +158,8 @@ module Gitlab # finalize_backfilling_partitioned_table :audit_events # def finalize_backfilling_partitioned_table(table_name) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + assert_table_is_allowed(table_name) assert_not_in_transaction_block(scope: ERROR_SCOPE) @@ -170,8 +176,10 @@ module Gitlab primary_key = connection.primary_key(table_name) copy_missed_records(table_name, partitioned_table_name, primary_key) - disable_statement_timeout do - execute("VACUUM FREEZE ANALYZE #{partitioned_table_name}") + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + disable_statement_timeout do + execute("VACUUM FREEZE ANALYZE #{partitioned_table_name}") + end end end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index 06e2b114c91..391375d472f 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -27,9 +27,15 @@ module Gitlab # to reduce amount of labels sort schemas used gitlab_schemas = gitlab_schemas.to_a.sort.join(",") + # Temporary feature to observe relation of `gitlab_schemas` to `db_config_name` + # depending on primary model + ci_dedicated_primary_connection = ::Ci::ApplicationRecord.connection_class? && + ::Ci::ApplicationRecord.load_balancer.configuration.use_dedicated_connection? + schemas_metrics.increment({ gitlab_schemas: gitlab_schemas, - db_config_name: db_config_name + db_config_name: db_config_name, + ci_dedicated_primary_connection: ci_dedicated_primary_connection }) end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index ab40ba5d59b..3f0176cb654 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -69,8 +69,10 @@ module Gitlab schemas = self.dml_schemas(tables) if (schemas - self.allowed_gitlab_schemas).any? - raise DMLAccessDeniedError, "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ - "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'." + raise DMLAccessDeniedError, \ + "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ + "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'. " \ + "#{documentation_url}" end end @@ -93,11 +95,19 @@ module Gitlab end def raise_dml_not_allowed_error(message) - raise DMLNotAllowedError, "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. #{message}" + raise DMLNotAllowedError, \ + "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. " \ + "#{message}. #{documentation_url}" \ end def raise_ddl_not_allowed_error(message) - raise DDLNotAllowedError, "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. #{message}" + raise DDLNotAllowedError, \ + "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. " \ + "#{message}. #{documentation_url}" + end + + def documentation_url + "For more information visit: https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html" end end end diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb index f4ea59deb50..ece9327b658 100644 --- a/lib/gitlab/database/reindexing/grafana_notifier.rb +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -5,10 +5,10 @@ module Gitlab module Reindexing # This can be used to send annotations for reindexing to a Grafana API class GrafanaNotifier - def initialize(api_key = ENV['GITLAB_GRAFANA_API_KEY'], api_url = ENV['GITLAB_GRAFANA_API_URL'], additional_tag = ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env) - @api_key = api_key - @api_url = api_url - @additional_tag = additional_tag + def initialize(api_key: nil, api_url: nil, additional_tag: nil) + @api_key = api_key || default_api_key + @api_url = api_url || default_api_url + @additional_tag = additional_tag || default_additional_tag end def notify_start(action) @@ -35,10 +35,22 @@ module Gitlab private + def default_api_key + Gitlab::CurrentSettings.database_grafana_api_key || ENV['GITLAB_GRAFANA_API_KEY'] + end + + def default_api_url + Gitlab::CurrentSettings.database_grafana_api_url || ENV['GITLAB_GRAFANA_API_URL'] + end + + def default_additional_tag + Gitlab::CurrentSettings.database_grafana_tag || ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env + end + def base_payload(action) { time: (action.action_start.utc.to_f * 1000).to_i, - tags: ['reindex', @additional_tag, action.index.tablename, action.index.name].compact + tags: ['reindex', @additional_tag.presence, action.index.tablename, action.index.name].compact } end diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb index af1fd8fb03e..860f87a28a3 100644 --- a/lib/gitlab/diff/custom_diff.rb +++ b/lib/gitlab/diff/custom_diff.rb @@ -2,17 +2,29 @@ module Gitlab module Diff module CustomDiff + RENDERED_TIMEOUT_BACKGROUND = 20.seconds + RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds + BACKGROUND_EXECUTION = 'background' + FOREGROUND_EXECUTION = 'foreground' + LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' + LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' + LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + class << self def preprocess_before_diff(path, old_blob, new_blob) return unless path.ends_with? '.ipynb' - transformed_diff(old_blob&.data, new_blob&.data)&.tap do - transformed_for_diff(new_blob, old_blob) - Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + Timeout.timeout(timeout_time) do + transformed_diff(old_blob&.data, new_blob&.data)&.tap do + transformed_for_diff(new_blob, old_blob) + log_event(LOG_IPYNBDIFF_GENERATED) + end end + rescue Timeout::Error => e + rendered_timeout.increment(source: execution_source) + log_event(LOG_IPYNBDIFF_TIMEOUT, e) rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e - Gitlab::ErrorTracking.log_exception(e) - nil + log_event(LOG_IPYNBDIFF_INVALID, e) end def transformed_diff(before, after) @@ -50,6 +62,27 @@ module Gitlab blobs_with_transformed_diffs[b] = true if b end end + + def rendered_timeout + @rendered_timeout ||= Gitlab::Metrics.counter( + :ipynb_semantic_diff_timeouts_total, + 'Counts the times notebook rendering timed out' + ) + end + + def timeout_time + Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND + end + + def execution_source + Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION + end + + def log_event(message, error = nil) + Gitlab::AppLogger.info({ message: message }) + Gitlab::ErrorTracking.track_exception(error) if error + nil + end end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 89822af2455..61bb0c797b4 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,7 +44,13 @@ module Gitlab new_blob_lazy old_blob_lazy - diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff unless use_renderable_diff? + if use_semantic_ipynb_diff? && !use_renderable_diff? + diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff + end + end + + def use_semantic_ipynb_diff? + strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project, default_enabled: :yaml) } end def use_renderable_diff? @@ -375,7 +381,7 @@ module Gitlab end def rendered - return unless use_renderable_diff? && ipynb? + return unless use_semantic_ipynb_diff? && use_renderable_diff? && ipynb? && modified_file? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end @@ -410,7 +416,7 @@ module Gitlab end def ipynb? - modified_file? && file_path.ends_with?('.ipynb') + file_path.ends_with?('.ipynb') end # We can't use Object#try because Blob doesn't inherit from Object, but diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index c2b834c71b5..316a0d2815a 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -9,8 +9,8 @@ module Gitlab SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze attr_reader :marker_ranges - attr_writer :text, :rich_text, :discussable - attr_accessor :index, :type, :old_pos, :new_pos, :line_code + attr_writer :text, :rich_text + attr_accessor :index, :old_pos, :new_pos, :line_code, :type def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text = text @@ -24,9 +24,7 @@ module Gitlab # When line code is not provided from cache store we build it # using the parent_file(Diff::File or Conflict::File). @line_code = line_code || calculate_line_code - @marker_ranges = [] - @discussable = true end def self.init_from_hash(hash) @@ -81,23 +79,28 @@ module Gitlab end def added? - %w[new new-nonewline].include?(type) + %w[new new-nonewline new-nomappinginraw].include?(type) end def removed? - %w[old old-nonewline].include?(type) + %w[old old-nonewline old-nomappinginraw].include?(type) end def meta? %w[match new-nonewline old-nonewline].include?(type) end + def has_mapping_in_raw? + # Used for rendered diff, when the displayed line doesn't have a matching line in the raw diff + !type&.ends_with?('nomappinginraw') + end + def match? type == :match end def discussable? - @discussable && !meta? + has_mapping_in_raw? && !meta? end def suggestible? diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 77b65fea726..cbfc20d3d62 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -44,7 +44,7 @@ module Gitlab free_right_index = nil i += 1 end - elsif line.meta? || line.unchanged? + elsif line.meta? || line.unchanged? || !line.has_mapping_in_raw? # line in the right panel is the same as in the left one lines << { left: line, diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index e700e730f20..cf97569ca31 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -6,6 +6,14 @@ module Gitlab include Gitlab::Utils::StrongMemoize class DiffFile < Gitlab::Diff::File + RENDERED_TIMEOUT_BACKGROUND = 10.seconds + RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds + BACKGROUND_EXECUTION = 'background' + FOREGROUND_EXECUTION = 'foreground' + LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' + LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' + LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + attr_reader :source_diff delegate :repository, :diff_refs, :fallback_diff_refs, :unfolded, :unique_identifier, @@ -52,14 +60,17 @@ module Gitlab def notebook_diff strong_memoize(:notebook_diff) do - Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) - - IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, - raise_if_invalid_nb: true, - diffy_opts: { include_diff_info: true }) + Timeout.timeout(timeout_time) do + IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, + raise_if_invalid_nb: true, diffy_opts: { include_diff_info: true })&.tap do + log_event(LOG_IPYNBDIFF_GENERATED) + end + end + rescue Timeout::Error => e + rendered_timeout.increment(source: Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION) + log_event(LOG_IPYNBDIFF_TIMEOUT, e) rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e - Gitlab::ErrorTracking.log_exception(e) - nil + log_event(LOG_IPYNBDIFF_INVALID, e) end end @@ -87,10 +98,7 @@ module Gitlab line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0 # Lines that do not appear on the original diff should not be commentable - - unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] - line.discussable = false - end + line.type = "#{line.type || 'unchanged'}-nomappinginraw" unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] line.line_code = line_code(line) line @@ -113,12 +121,29 @@ module Gitlab additions = {} source_diff.highlighted_diff_lines.each do |line| - removals[line.old_pos] = line.new_pos - additions[line.new_pos] = line.old_pos + removals[line.old_pos] = line.new_pos unless source_diff.new_file? + additions[line.new_pos] = line.old_pos unless source_diff.deleted_file? end [removals, additions] end + + def rendered_timeout + @rendered_timeout ||= Gitlab::Metrics.counter( + :ipynb_semantic_diff_timeouts_total, + 'Counts the times notebook diff rendering timed out' + ) + end + + def timeout_time + Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND + end + + def log_event(message, error = nil) + Gitlab::AppLogger.info({ message: message }) + Gitlab::ErrorTracking.track_exception(error) if error + nil + end end end end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index bb57494c729..71b1d4ed8f9 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -34,7 +34,7 @@ module Gitlab create_issue_or_note - if issue_creator_address + if from_address add_email_participant send_thank_you_email unless reply_email? end @@ -98,7 +98,7 @@ module Gitlab title: mail.subject, description: message_including_template, confidential: true, - external_author: external_author + external_author: from_address }, spam_params: nil ).execute @@ -176,22 +176,8 @@ module Gitlab ).execute end - def issue_creator_address - reply_to_address || from_address - end - def from_address - mail.from.first || mail.sender - end - - def reply_to_address - (mail.reply_to || []).first - end - - def external_author - return issue_creator_address unless reply_to_address && from_address - - _("%{from_address} (reply to: %{reply_to_address})") % { from_address: from_address, reply_to_address: reply_to_address } + (mail.reply_to || []).first || mail.from.first || mail.sender end def can_handle_legacy_format? @@ -205,7 +191,7 @@ module Gitlab def add_email_participant return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) - @issue.issue_email_participants.create(email: issue_creator_address) + @issue.issue_email_participants.create(email: from_address) end end end diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb index ac9585bcd1a..bd2c91755c8 100644 --- a/lib/gitlab/email/message/in_product_marketing.rb +++ b/lib/gitlab/email/message/in_product_marketing.rb @@ -7,7 +7,7 @@ module Gitlab UnknownTrackError = Class.new(StandardError) def self.for(track) - valid_tracks = [Namespaces::InviteTeamEmailService::TRACK, Namespaces::InProductMarketingEmailsService::TRACKS.keys].flatten + valid_tracks = Namespaces::InProductMarketingEmailsService::TRACKS.keys raise UnknownTrackError unless valid_tracks.include?(track) "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize diff --git a/lib/gitlab/email/message/in_product_marketing/invite_team.rb b/lib/gitlab/email/message/in_product_marketing/invite_team.rb deleted file mode 100644 index e9334b687f4..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/invite_team.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class InviteTeam < Base - def subject_line - s_('InProductMarketing|Invite your teammates to GitLab') - end - - def tagline - '' - end - - def title - s_('InProductMarketing|GitLab is better with teammates to help out!') - end - - def subtitle - '' - end - - def body_line1 - s_('InProductMarketing|Invite your teammates today and build better code together. You can even assign tasks to new teammates such as setting up CI/CD, to help get projects up and running.') - end - - def body_line2 - '' - end - - def cta_text - s_('InProductMarketing|Invite your teammates to help') - end - - def logo_path - 'mailers/in_product_marketing/team-0.png' - end - - def series? - false - end - - private - - def validate_series! - raise ArgumentError, "Only one email is sent for this track. Value of `series` should be 0." unless @series == 0 - end - end - end - end - end -end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 3c5d223b106..f539d627dcb 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -46,12 +46,13 @@ module Gitlab def custom_emoji_tag(name, image_source) data = { - name: name + name: name, + fallback_src: image_source, + unicode_version: 'custom' # Prevents frontend to check for Unicode support } + options = { title: name, data: data } - ActionController::Base.helpers.content_tag('gl-emoji', title: name, data: data) do - emoji_image_tag(name, image_source).html_safe - end + ActionController::Base.helpers.content_tag('gl-emoji', "", options) end end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 2e0060c7c18..f26ab6e3ed1 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -15,6 +15,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 ENCODING_CONFIDENCE_THRESHOLD = 50 + UNICODE_REPLACEMENT_CHARACTER = "�" + def encode!(message) message = force_encode_utf8(message) return message if message.valid_encoding? @@ -65,6 +67,10 @@ module Gitlab message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) end + def encode_utf8_with_replacement_character(data) + encode_utf8(data, replace: UNICODE_REPLACEMENT_CHARACTER) + end + def encode_utf8(message, replace: "") message = force_encode_utf8(message) return message if message.valid_encoding? @@ -99,6 +105,35 @@ module Gitlab io.tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end + ESCAPED_CHARS = { + "a" => "\a", "b" => "\b", "e" => "\e", "f" => "\f", + "n" => "\n", "r" => "\r", "t" => "\t", "v" => "\v", + "\"" => "\"" + }.freeze + + # rubocop:disable Style/AsciiComments + # `unquote_path` decode filepaths that are returned by some git commands. + # The path may be returned in double-quotes if it contains special characters, + # that are encoded in octal. Also, some characters (see `ESCAPED_CHARS`) are escaped. + # eg. "\311\240\304\253\305\247\305\200\310\247\306\200" (quotes included) is decoded as É Ä«Å§Å€È§Æ€ + # + # Based on `unquote_c_style` from git source + # https://github.com/git/git/blob/v2.35.1/quote.c#L399 + # rubocop:enable Style/AsciiComments + def unquote_path(filename) + return filename unless filename[0] == '"' + + filename = filename[1..-2].gsub(/\\(?:([#{ESCAPED_CHARS.keys.join}\\])|(\d{3}))/) do + if c = Regexp.last_match(1) + c == "\\" ? "\\" : ESCAPED_CHARS[c] + elsif c = Regexp.last_match(2) + c.to_i(8).chr + end + end + + filename.force_encoding("UTF-8") + end + private def force_encode_utf8(message) diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 70c363877b1..4bef92f5c23 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -28,13 +28,10 @@ module Gitlab # If the `Feature.enabled?` check is false, we return nil implicitly, # which will assign the control. Otherwise we call super, which will # assign a variant evenly, or based on our provided distribution rules. - def execute_assigment + def execute_assignment super if ::Feature.enabled?(feature_flag_name, self, type: :experiment, default_enabled: :yaml) end - # NOTE: There's a typo in the name of this method that we'll fix up. - alias_method :execute_assignment, :execute_assigment - # This is what's provided to the `Feature.enabled?` call that will be # used to determine experiment inclusion. An experiment may provide an # override for this method to make the experiment work on user, group, diff --git a/lib/gitlab/fips.rb b/lib/gitlab/fips.rb index 1dd363ceb17..97813f13a91 100644 --- a/lib/gitlab/fips.rb +++ b/lib/gitlab/fips.rb @@ -5,6 +5,17 @@ module Gitlab class FIPS # A simple utility class for FIPS-related helpers + Technology = Gitlab::SSHPublicKey::Technology + + SSH_KEY_TECHNOLOGIES = [ + Technology.new(:rsa, SSHData::PublicKey::RSA, [3072, 4096], %w(ssh-rsa)), + Technology.new(:dsa, SSHData::PublicKey::DSA, [], %w(ssh-dss)), + Technology.new(:ecdsa, SSHData::PublicKey::ECDSA, [256, 384, 521], %w(ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521)), + Technology.new(:ed25519, SSHData::PublicKey::ED25519, [256], %w(ssh-ed25519)), + Technology.new(:ecdsa_sk, SSHData::PublicKey::SKECDSA, [256], %w(sk-ecdsa-sha2-nistp256@openssh.com)), + Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) + ].freeze + class << self # Returns whether we should be running in FIPS mode or not # diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 08321d5fda6..82ef7eed56a 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -37,7 +37,7 @@ module Gitlab if was_embedded?(markdown) moved_markdown else - moved_markdown.sub(/\A!/, "") + moved_markdown.delete_prefix('!') end end end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 5669a65cbd9..30977adaea1 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -5,35 +5,45 @@ module Gitlab class Blame include Gitlab::EncodingHelper - attr_reader :lines, :blames + attr_reader :lines, :blames, :range - def initialize(repository, sha, path) + def initialize(repository, sha, path, range: nil) @repo = repository @sha = sha @path = path + @range = range @lines = [] @blames = load_blame end def each @blames.each do |blame| - yield(blame.commit, blame.line) + yield(blame.commit, blame.line, blame.previous_path) end end private + def range_spec + "#{range.first},#{range.last}" if range + end + def load_blame - output = encode_utf8(@repo.gitaly_commit_client.raw_blame(@sha, @path)) + output = encode_utf8( + @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) + ) process_raw_blame(output) end def process_raw_blame(output) + start_line = nil lines = [] final = [] info = {} commits = {} + commit_id = nil + previous_paths = {} # process the output output.split("\n").each do |line| @@ -45,6 +55,15 @@ module Gitlab commit_id = m[1] commits[commit_id] = nil unless commits.key?(commit_id) info[m[3].to_i] = [commit_id, m[2].to_i] + + # Assumption: the first line returned by git blame is lowest-numbered + # This is true unless we start passing it `--incremental`. + start_line = m[3].to_i if start_line.nil? + elsif line.start_with?("previous ") + # previous 1485b69e7b839a21436e81be6d3aa70def5ed341 initial-commit + # previous 9521e52704ee6100e7d2a76896a4ef0eb53ff1b8 "\303\2511\\\303\251\\303\\251\n" + # ^ char index 50 + previous_paths[commit_id] = unquote_path(line[50..]) end end @@ -54,7 +73,13 @@ module Gitlab # get it together info.sort.each do |lineno, (commit_id, old_lineno)| - final << BlameLine.new(lineno, old_lineno, commits[commit_id], lines[lineno - 1]) + final << BlameLine.new( + lineno, + old_lineno, + commits[commit_id], + lines[lineno - start_line], + previous_paths[commit_id] + ) end @lines = final @@ -62,13 +87,14 @@ module Gitlab end class BlameLine - attr_accessor :lineno, :oldlineno, :commit, :line + attr_accessor :lineno, :oldlineno, :commit, :line, :previous_path - def initialize(lineno, oldlineno, commit, line) + def initialize(lineno, oldlineno, commit, line, previous_path) @lineno = lineno @oldlineno = oldlineno @commit = commit @line = line + @previous_path = previous_path end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 8325eadce2f..a66517b4ca0 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -140,7 +140,7 @@ module Gitlab text.start_with?(BINARY_NOTICE_PATTERN) end end - def initialize(raw_diff, expanded: true) + def initialize(raw_diff, expanded: true, replace_invalid_utf8_chars: true) @expanded = expanded case raw_diff @@ -157,6 +157,8 @@ module Gitlab else raise "Invalid raw diff type: #{raw_diff.class}" end + + encode_diff_to_utf8(replace_invalid_utf8_chars) end def to_hash @@ -227,6 +229,13 @@ module Gitlab private + def encode_diff_to_utf8(replace_invalid_utf8_chars) + return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol, default_enabled: :yaml) + return unless replace_invalid_utf8_chars && !detect_binary?(@diff) + + @diff = Gitlab::EncodingHelper.encode_utf8_with_replacement_character(@diff) + end + def init_from_hash(hash) raw_diff = hash.symbolize_keys diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 24b67424f28..0ffe8bee953 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -9,8 +9,6 @@ module Gitlab attr_reader :limits - delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.default_limits { max_files: ::Commit.diff_safe_max_files, max_lines: ::Commit.diff_safe_max_lines } end @@ -26,8 +24,7 @@ module Gitlab limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes - - OpenStruct.new(limits) + limits end def initialize(iterator, options = {}) @@ -140,11 +137,11 @@ module Gitlab end def over_safe_limits?(files) - if files >= safe_max_files + if files >= limits[:safe_max_files] @collapsed_safe_files = true - elsif @line_count > safe_max_lines + elsif @line_count > limits[:safe_max_lines] @collapsed_safe_lines = true - elsif @byte_count >= safe_max_bytes + elsif @byte_count >= limits[:safe_max_bytes] @collapsed_safe_bytes = true end @@ -179,7 +176,7 @@ module Gitlab @iterator.each_with_index do |raw, iterator_index| @empty = false - if @enforce_limits && i >= max_files + if @enforce_limits && i >= limits[:max_files] @overflow = true @overflow_max_files = true break @@ -194,7 +191,7 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && @line_count >= max_lines + if @enforce_limits && @line_count >= limits[:max_lines] # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true @@ -202,7 +199,7 @@ module Gitlab break end - if @enforce_limits && @byte_count >= max_bytes + if @enforce_limits && @byte_count >= limits[:max_bytes] # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 47cfb483509..1d7966a11ed 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -24,7 +24,7 @@ module Gitlab # Ex. # Ref.extract_branch_name('refs/heads/master') #=> 'master' def self.extract_branch_name(str) - str.gsub(%r{\Arefs/heads/}, '') + str.delete_prefix('refs/heads/') end def initialize(repository, name, target, dereferenced_target) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1492ea1ce76..ab365069adf 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -841,11 +841,11 @@ module Gitlab end end - def import_repository(url) + def import_repository(url, http_authorization_header: '', mirror: false) raise ArgumentError, "don't use disk paths with import_repository: #{url.inspect}" if url.start_with?('.', '/') wrapped_gitaly_errors do - gitaly_repository_client.import_repository(url) + gitaly_repository_client.import_repository(url, http_authorization_header: http_authorization_header, mirror: mirror) end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0e3f9c2598d..4fe5c8df36f 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -315,11 +315,12 @@ module Gitlab response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end - def raw_blame(revision, path) + def raw_blame(revision, path, range:) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), - path: encode_binary(path) + path: encode_binary(path), + range: (encode_binary(range) if range) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) @@ -466,7 +467,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h) + request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 5c447dfd417..1e199a55b5a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -145,10 +145,12 @@ module Gitlab ) end - def import_repository(source) + def import_repository(source, http_authorization_header: '', mirror: false) request = Gitaly::CreateRepositoryFromURLRequest.new( repository: @gitaly_repo, - url: source + url: source, + http_authorization_header: http_authorization_header, + mirror: mirror ) GitalyClient.call( diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 7ce88280209..8873db24118 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -24,6 +24,8 @@ module Gitlab increment_project_counter(project, object_type, operation, integer) increment_global_counter(object_type, operation, integer) + + project.import_state&.expire_etag_cache end def summary(project) diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4dec9543a13..97de2a49e72 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,7 +72,7 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import - if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + if parallel_import_batch.present? spread_parallel_import else parallel_import_deprecated @@ -209,7 +209,11 @@ module Gitlab # Default batch settings for parallel import (can be redefined in Importer classes) # Example: { size: 100, delay: 1.minute } def parallel_import_batch - {} + if Feature.enabled?(:distribute_github_parallel_import, default_enabled: :yaml) + { size: 1000, delay: 1.minute } + else + {} + end end def abort_on_failure diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9f18513f066..3c85d56874f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -53,13 +53,13 @@ module Gitlab # made globally available to the frontend push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) push_frontend_feature_flag(:source_editor_toolbar, default_enabled: :yaml) push_frontend_feature_flag(:gl_avatar_for_all_user_avatars, default_enabled: :yaml) push_frontend_feature_flag(:mr_attention_requests, default_enabled: :yaml) + push_frontend_feature_flag(:markdown_continue_lists, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. @@ -73,6 +73,15 @@ module Gitlab push_to_gon_attributes(:features, name, enabled) end + # Exposes the state of a feature flag to the frontend code. + # Can be used for more complex feature flag checks. + # + # name - The name of the feature flag, e.g. `my_feature`. + # enabled - Boolean to be pushed directly to the frontend. Should be fetched by checking a feature flag. + def push_force_frontend_feature_flag(name, enabled) + push_to_gon_attributes(:features, name, !!enabled) + end + def push_to_gon_attributes(key, name, enabled) var_name = name.to_s.camelize(:lower) # Here the `true` argument signals gon that the value should be merged diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index 20068758502..3335e511714 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -5,7 +5,7 @@ module Gitlab class Deprecation REASONS = { renamed: 'This was renamed.', - discouraged: 'Use of this is not recommended.' + alpha: 'This feature is in Alpha, and can be removed or changed at any point.' }.freeze include ActiveModel::Validations diff --git a/lib/gitlab/graphql/known_operations.rb b/lib/gitlab/graphql/known_operations.rb index ead52935945..a551c9bb6da 100644 --- a/lib/gitlab/graphql/known_operations.rb +++ b/lib/gitlab/graphql/known_operations.rb @@ -14,7 +14,6 @@ module Gitlab end end - ANONYMOUS = Operation.new("anonymous").freeze UNKNOWN = Operation.new("unknown").freeze def self.default @@ -24,7 +23,7 @@ module Gitlab def initialize(operation_names) @operation_hash = operation_names .map { |name| Operation.new(name).freeze } - .concat([ANONYMOUS, UNKNOWN]) + .concat([UNKNOWN]) .index_by(&:name) end @@ -32,7 +31,7 @@ module Gitlab def from_query(query) operation_name = query.selected_operation_name - return ANONYMOUS unless operation_name + return UNKNOWN unless operation_name @operation_hash[operation_name] || UNKNOWN end diff --git a/lib/gitlab/graphql/pagination/active_record_array_connection.rb b/lib/gitlab/graphql/pagination/active_record_array_connection.rb new file mode 100644 index 00000000000..9e40f79b2fd --- /dev/null +++ b/lib/gitlab/graphql/pagination/active_record_array_connection.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +# Connection for an array of Active Record instances. +# Resolvers needs to handle cursors (before and after). +# This connection will handle (first and last). +# Supports batch loaded items. +# Expects the array to use a fixed DESC order. This is similar to +# ExternallyPaginatedArrayConnection. +module Gitlab + module Graphql + module Pagination + class ActiveRecordArrayConnection < GraphQL::Pagination::ArrayConnection + include ::Gitlab::Graphql::ConnectionCollectionMethods + prepend ::Gitlab::Graphql::ConnectionRedaction + + delegate :<<, to: :items + + def nodes + load_nodes + + @nodes + end + + def next_page? + load_nodes + + if before + true + elsif first + limit_value < items.size + else + false + end + end + + def previous_page? + load_nodes + + if after + true + elsif last + limit_value < items.size + else + false + end + end + + # see https://graphql-ruby.org/pagination/custom_connections#connection-wrapper + alias_method :has_next_page, :next_page? + alias_method :has_previous_page, :previous_page? + + def cursor_for(item) + # item could be a batch loaded item. Sync it to have the id. + cursor = { 'id' => Gitlab::Graphql::Lazy.force(item).id.to_s } + encode(cursor.to_json) + end + + # Part of the implied interface for default objects for BatchLoader: objects must be clonable + def dup + self.class.new( + items.dup, + first: first, + after: after, + max_page_size: max_page_size, + last: last, + before: before + ) + end + + private + + def limit_value + # note: only first _or_ last can be specified, not both + @limit_value ||= [first, last, max_page_size].compact.min + end + + def load_nodes + @nodes ||= begin + limited_nodes = items + + limited_nodes = limited_nodes.first(first) if first + limited_nodes = limited_nodes.last(last) if last + + limited_nodes + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 61903c566f0..c284160e539 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -14,10 +14,6 @@ # Issue.order(created_at: :asc).order(:id) # Issue.order(due_date: :asc) # -# You can also use `Gitlab::Database.nulls_last_order`: -# -# Issue.reorder(::Gitlab::Database.nulls_last_order('due_date', 'DESC')) -# # It will tolerate non-attribute ordering, but only attributes determine the cursor. # For example, this is legitimate: # diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index e8335a3c79c..bf9b73d918a 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -73,9 +73,24 @@ module Gitlab strong_memoize(:generic_keyset_pagination_items) do rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items) - success ? rebuilt_items_with_keyset_order : original_items + if success + rebuilt_items_with_keyset_order + else + if original_items.is_a?(ActiveRecord::Relation) + old_keyset_pagination_usage.increment({ model: original_items.model.to_s }) + end + + original_items + end end end + + def old_keyset_pagination_usage + @old_keyset_pagination_usage ||= Gitlab::Metrics.counter( + :old_keyset_pagination_usage, + 'The number of times the old keyset pagination code was used' + ) + end end end end diff --git a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb index a3c3f2f2b7e..45f90de2f17 100644 --- a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb +++ b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb @@ -2,7 +2,7 @@ module Gitlab module Graphql module Project - class DastProfileConnectionExtension < GraphQL::Schema::Field::ConnectionExtension + class DastProfileConnectionExtension < GraphQL::Schema::FieldExtension def after_resolve(value:, object:, context:, **rest) preload_authorizations(context[:project_dast_profiles]) context[:project_dast_profiles] = nil diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 5c8aa5050ed..add9e880475 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -13,7 +13,7 @@ module Gitlab event_type: event_type, user: user.hook_attrs, project: issuable.project.hook_attrs, - object_attributes: issuable.hook_attrs, + object_attributes: issuable_builder.new(issuable).build, labels: issuable.labels.map(&:hook_attrs), changes: final_changes(changes.slice(*safe_keys)), # DEPRECATED @@ -53,10 +53,7 @@ module Gitlab end def final_changes(changes_hash) - changes_hash.reduce({}) do |hash, (key, changes_array)| - hash[key] = Hash[CHANGES_KEYS.zip(changes_array)] - hash - end + changes_hash.transform_values { |changes_array| Hash[CHANGES_KEYS.zip(changes_array)] } end end end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index aaca16d8d7c..06ddd65d075 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -60,6 +60,7 @@ module Gitlab human_time_estimate: merge_request.human_time_estimate, assignee_ids: merge_request.assignee_ids, assignee_id: merge_request.assignee_ids.first, # This key is deprecated + labels: merge_request.labels_hook_attrs, state: merge_request.state, # This key is deprecated blocking_discussions_resolved: merge_request.mergeable_discussions_state? } diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 002708beb3c..7b1657d3854 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -29,17 +29,13 @@ module Gitlab http = super http.hostname_override = hostname if hostname - if Feature.enabled?(:header_read_timeout_buffered_io, default_enabled: :yaml) - gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) + gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) - http.instance_variables.each do |variable| - gitlab_http.instance_variable_set(variable, http.instance_variable_get(variable)) - end - - return gitlab_http + http.instance_variables.each do |variable| + gitlab_http.instance_variable_set(variable, http.instance_variable_get(variable)) end - http + gitlab_http end private diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index d01f7d0074f..8b775d567c8 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 46, - 'de' => 15, + 'da_DK' => 44, + 'de' => 14, 'en' => 100, 'eo' => 0, - 'es' => 40, + 'es' => 39, 'fil_PH' => 0, - 'fr' => 11, + 'fr' => 10, 'gl_ES' => 0, 'id_ID' => 0, - 'it' => 2, + 'it' => 1, 'ja' => 34, 'ko' => 12, - 'nb_NO' => 30, + 'nb_NO' => 29, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 49, - 'ro_RO' => 22, - 'ru' => 32, - 'tr_TR' => 14, - 'uk' => 48, - 'zh_CN' => 95, + 'pt_BR' => 50, + 'ro_RO' => 36, + 'ru' => 31, + 'tr_TR' => 13, + 'uk' => 46, + 'zh_CN' => 97, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 3bb34ab2811..74be56df221 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -248,10 +248,9 @@ module Gitlab variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string end else - variables.inject({}) do |hash, variable| + variables.each_with_object({}) do |variable, hash| variable_name = variable[/\w+/] hash[variable_name] = Gitlab::Utils.random_string - hash end end end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b43d0a0c3eb..e38496ecf67 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -17,11 +17,11 @@ module Gitlab public def initialize(attributes = {}) - @options = OpenStruct.new(attributes) + @options = attributes + end - self.class.instance_eval do - def_delegators :@options, *attributes.keys - end + def method_missing(method, *args) + @options[method] end def execute(current_user, project) diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 7534ab5a9ce..db90886ad11 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -3,19 +3,23 @@ module Gitlab module ImportExport class AvatarSaver + include DurationMeasuring + def initialize(project:, shared:) @project = project @shared = shared end def save - return true unless @project.avatar.exists? + with_duration_measuring do + break true unless @project.avatar.exists? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared, - relative_export_path: 'avatar' - ).save + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar' + ).save + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2b0467d8779..64ef3dd4830 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -66,7 +66,7 @@ module Gitlab current_size = 0 Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| - if [301, 302, 307].include?(fragment.code) + if [301, 302, 303, 307].include?(fragment.code) Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) elsif fragment.code == 200 current_size += fragment.bytesize diff --git a/lib/gitlab/import_export/duration_measuring.rb b/lib/gitlab/import_export/duration_measuring.rb new file mode 100644 index 00000000000..c192be6ae29 --- /dev/null +++ b/lib/gitlab/import_export/duration_measuring.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module DurationMeasuring + extend ActiveSupport::Concern + + included do + attr_reader :duration_s + + def with_duration_measuring + result = nil + + @duration_s = Benchmark.realtime do + result = yield + end + + result + end + end + end + end +end diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb index e5d52f945b5..d049609187b 100644 --- a/lib/gitlab/import_export/fast_hash_serializer.rb +++ b/lib/gitlab/import_export/fast_hash_serializer.rb @@ -92,7 +92,7 @@ module Gitlab def simple_serialize subject.as_json( - tree.merge(include: nil, preloads: nil)) + tree.merge(include: nil, preloads: nil, unsafe: true)) end def serialize_includes diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 55b8c1d4531..ebabf537ce5 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -37,7 +37,7 @@ module Gitlab def serialize_root(exportable_path = @exportable_path) attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) + relations_schema.merge(include: nil, preloads: nil, unsafe: true)) json_writer.write_attributes(exportable_path, attributes) end @@ -145,8 +145,8 @@ module Gitlab arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction] reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] - order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position) - reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position) + order_expression = arel_table[column].public_send(direction).public_send(nulls_position) # rubocop:disable GitlabSecurity/PublicSend + reverse_order_expression = arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position) # rubocop:disable GitlabSecurity/PublicSend ::Gitlab::Pagination::Keyset::Order.build([ ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 47acd49d529..22a7a8dd7cd 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class LfsSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring attr_accessor :lfs_json, :project, :shared @@ -16,17 +17,19 @@ module Gitlab end def save - project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| - batch.each do |lfs_object| - save_lfs_object(lfs_object) - end + with_duration_measuring do + project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end - append_lfs_json_for_batch(batch) - end + append_lfs_json_for_batch(batch) + end - write_lfs_json + write_lfs_json - true + true + end rescue StandardError => e shared.error(e) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d3b1bb6a57d..b1f2a17d4b7 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -16,7 +16,7 @@ module Gitlab def map @map ||= begin - @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| if member['user'] old_user_id = member['user']['id'] existing_user_id = existing_users_email_map[get_email(member)] @@ -24,8 +24,6 @@ module Gitlab else add_team_member(member) end - - hash end end end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index aafed850afa..63c5afa9595 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -4,6 +4,8 @@ module Gitlab module ImportExport module Project class TreeSaver + include DurationMeasuring + attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}, logger: Gitlab::Import::Logger) @@ -15,9 +17,11 @@ module Gitlab end def save - stream_export + with_duration_measuring do + stream_export - true + true + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index fae07039139..454e84bbc04 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class RepoSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring attr_reader :exportable, :shared @@ -13,9 +14,12 @@ module Gitlab end def save - return true unless repository_exists? # it's ok to have no repo + with_duration_measuring do + # it's ok to have no repo + break true unless repository_exists? - bundle_to_disk + bundle_to_disk + end end def repository diff --git a/lib/gitlab/import_export/snippets_repo_saver.rb b/lib/gitlab/import_export/snippets_repo_saver.rb index d3b0fe1c18c..ca0d38272e5 100644 --- a/lib/gitlab/import_export/snippets_repo_saver.rb +++ b/lib/gitlab/import_export/snippets_repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class SnippetsRepoSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring def initialize(current_user:, project:, shared:) @project = project @@ -12,13 +13,15 @@ module Gitlab end def save - create_snippets_repo_directory + with_duration_measuring do + create_snippets_repo_directory - @project.snippets.find_each.all? do |snippet| - Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, - shared: @shared, - repository: snippet.repository) - .save + @project.snippets.find_each.all? do |snippet| + Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, + shared: @shared, + repository: snippet.repository) + .save + end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 9f58609fa17..05132fd3edd 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -3,16 +3,20 @@ module Gitlab module ImportExport class UploadsSaver + include DurationMeasuring + def initialize(project:, shared:) @project = project @shared = shared end def save - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).save + with_duration_measuring do + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index e8f68f93af0..db5040ec0f6 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -4,17 +4,20 @@ module Gitlab module ImportExport class VersionSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring def initialize(shared:) @shared = shared end def save - mkdir_p(@shared.export_path) + with_duration_measuring do + mkdir_p(@shared.export_path) - File.write(version_file, Gitlab::ImportExport.version, mode: 'w') - File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') - File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') + File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index ef342f3819f..43ad64603a6 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -11,19 +11,12 @@ module Gitlab class InsecureKeyFingerprint attr_accessor :key - alias_attribute :fingerprint_md5, :fingerprint - - # # Gets the base64 encoded string representing a rsa or dsa key # def initialize(key_base64) @key = key_base64 end - def fingerprint - OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') - end - def fingerprint_sha256 Digest::SHA256.base64digest(Base64.decode64(@key)).scan(/../).join('').delete("=") end diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 82c2b3297c1..f347db7bc8c 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -3,12 +3,12 @@ module Gitlab module Integrations class StiType < ActiveRecord::Type::String - NAMESPACED_INTEGRATIONS = Set.new(%w( + NAMESPACED_INTEGRATIONS = %w[ Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao - )).freeze + ].to_set.freeze def self.namespaced_integrations NAMESPACED_INTEGRATIONS diff --git a/lib/gitlab/lazy.rb b/lib/gitlab/lazy.rb index d7a22aa339e..c589d613efc 100644 --- a/lib/gitlab/lazy.rb +++ b/lib/gitlab/lazy.rb @@ -15,10 +15,10 @@ module Gitlab @block = block end - def method_missing(name, *args, &block) + def method_missing(...) __evaluate__ - @result.__send__(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + @result.__send__(...) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(name, include_private = false) diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index 03655eb7237..89b0f0c802f 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -99,7 +99,7 @@ module Gitlab case actor when DeployKey, Key # Since fingerprint is based on the public key, let's take more bytes from attr_encrypted_db_key_base - actor.fingerprint.delete(':').first(16) + Settings.attr_encrypted_db_key_base_32 + actor.fingerprint_sha256.first(16) + Settings.attr_encrypted_db_key_base_32 when User # Take the last 16 characters as they're more unique than the first 16 actor.id.to_s + actor.encrypted_password.last(16) + Settings.attr_encrypted_db_key_base.first(16) diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index f4984e11c14..51277497c99 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -38,6 +38,10 @@ module Gitlab end end + def full_host + proc { |_env| Settings.gitlab['base_url'] } + end + private def cas3_signout_handler diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 065a3a0cf20..8c0f082f61c 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -120,7 +120,7 @@ module Gitlab .from(array_cte) .join(Arel.sql("LEFT JOIN LATERAL (#{initial_keyset_query.to_sql}) #{table_name} ON TRUE")) - order_by_columns.each { |column| q.where(column.column_expression.not_eq(nil)) } + order_by_columns.each { |c| q.where(c.column_expression.not_eq(nil)) unless c.column.nullable? } q.as('array_scope_lateral_query') end @@ -200,7 +200,7 @@ module Gitlab .project([*order_by_columns.original_column_names_as_arel_string, Arel.sql('position')]) .from("UNNEST(#{list(order_by_columns.array_aggregated_column_names)}) WITH ORDINALITY AS u(#{list(order_by_columns.original_column_names)}, position)") - order_by_columns.each { |column| q.where(Arel.sql(column.original_column_name).not_eq(nil)) } # ignore rows where all columns are NULL + order_by_columns.each { |c| q.where(Arel.sql(c.original_column_name).not_eq(nil)) unless c.column.nullable? } # ignore rows where all columns are NULL q.order(Arel.sql(order_by_without_table_references)).take(1) end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 1a00692bdbe..290e94401b8 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -99,6 +99,8 @@ module Gitlab field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') elsif field_value.nil? nil + elsif lower_named_function?(column_definition) + field_value.downcase else field_value.to_s end @@ -184,6 +186,10 @@ module Gitlab private + def lower_named_function?(column_definition) + column_definition.column_expression.is_a?(Arel::Nodes::NamedFunction) && column_definition.column_expression.name&.downcase == 'lower' + end + def composite_row_comparison_possible? !column_definitions.one? && column_definitions.all?(&:not_nullable?) && diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 5e79910a3e9..c36bd497aa3 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -11,13 +11,17 @@ module Gitlab # [transformed_scope, true] # true indicates that the new scope was successfully built # [orginal_scope, false] # false indicates that the order values are not supported in this class class SimpleOrderBuilder + NULLS_ORDER_REGEX = /(?<column_name>.*) (?<direction>\bASC\b|\bDESC\b) (?<nullable>\bNULLS LAST\b|\bNULLS FIRST\b)/.freeze + def self.build(scope) new(scope: scope).build end def initialize(scope:) @scope = scope - @order_values = scope.order_values + # We need to run 'compact' because 'nil' is not removed from order_values + # in some cases due to the use of 'default_scope'. + @order_values = scope.order_values.compact @model_class = scope.model @arel_table = @model_class.arel_table @primary_key = @model_class.primary_key @@ -28,10 +32,13 @@ module Gitlab primary_key_descending_order elsif Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + # Ordered by a primary key. Ex. 'ORDER BY id'. elsif ordered_by_primary_key? primary_key_order + # Ordered by one non-primary table column. Ex. 'ORDER BY created_at'. elsif ordered_by_other_column? column_with_tie_breaker_order + # Ordered by two table columns with the last column as a tie breaker. Ex. 'ORDER BY created, id ASC'. elsif ordered_by_other_column_with_tie_breaker? tie_breaker_attribute = order_values.second @@ -50,6 +57,77 @@ module Gitlab attr_reader :scope, :order_values, :model_class, :arel_table, :primary_key + def table_column?(name) + model_class.column_names.include?(name.to_s) + end + + def primary_key?(attribute) + arel_table[primary_key].to_s == attribute.to_s + end + + def lower_named_function?(attribute) + attribute.is_a?(Arel::Nodes::NamedFunction) && attribute.name&.downcase == 'lower' + end + + def arel_nulls?(order_value) + return unless order_value.is_a?(Arel::Nodes::NullsLast) || order_value.is_a?(Arel::Nodes::NullsFirst) + + column_name = order_value.try(:expr).try(:expr).try(:name) + + table_column?(column_name) + end + + def supported_column?(order_value) + return true if arel_nulls?(order_value) + + attribute = order_value.try(:expr) + return unless attribute + + if lower_named_function?(attribute) + attribute.expressions.one? && attribute.expressions.first.respond_to?(:name) && table_column?(attribute.expressions.first.name) + else + attribute.respond_to?(:name) && table_column?(attribute.name) + end + end + + # This method converts the first order value to a corresponding arel expression + # if the order value uses either NULLS LAST or NULLS FIRST ordering in raw SQL. + # + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/356644 + # We should stop matching raw literals once we switch to using the Arel methods. + def convert_raw_nulls_order! + order_value = order_values.first + + return unless order_value.is_a?(Arel::Nodes::SqlLiteral) + + # Detect NULLS LAST or NULLS FIRST ordering by looking at the raw SQL string. + if matches = order_value.match(NULLS_ORDER_REGEX) + return unless table_column?(matches[:column_name]) + + column_attribute = arel_table[matches[:column_name]] + direction = matches[:direction].downcase.to_sym + nullable = matches[:nullable].downcase.parameterize(separator: '_').to_sym + + # Build an arel order expression for NULLS ordering. + order = direction == :desc ? column_attribute.desc : column_attribute.asc + arel_order_expression = nullable == :nulls_first ? order.nulls_first : order.nulls_last + + order_values[0] = arel_order_expression + end + end + + def nullability(order_value, attribute_name) + nullable = model_class.columns.find { |column| column.name == attribute_name }.null + + if nullable && order_value.is_a?(Arel::Nodes::Ascending) + :nulls_last + elsif nullable && order_value.is_a?(Arel::Nodes::Descending) + :nulls_first + else + :not_nullable + end + end + def primary_key_descending_order Gitlab::Pagination::Keyset::Order.build([ Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( @@ -69,63 +147,76 @@ module Gitlab end def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) - order_expression = order_values.first - attribute_name = order_expression.expr.name - - column_nullable = model_class.columns.find { |column| column.name == attribute_name }.null - - nullable = if column_nullable && order_expression.is_a?(Arel::Nodes::Ascending) - :nulls_last - elsif column_nullable && order_expression.is_a?(Arel::Nodes::Descending) - :nulls_first - else - :not_nullable - end - Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: attribute_name, - order_expression: order_expression, - nullable: nullable, - distinct: false - ), + column(order_values.first), tie_breaker_column_order ]) end - def ordered_by_primary_key? - return unless order_values.one? + def column(order_value) + return nulls_order_column(order_value) if arel_nulls?(order_value) + return lower_named_function_column(order_value) if lower_named_function?(order_value.expr) - attribute = order_values.first.try(:expr) + attribute_name = order_value.expr.name - return unless attribute + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute_name, + order_expression: order_value, + nullable: nullability(order_value, attribute_name), + distinct: false + ) + end - arel_table[primary_key].to_s == attribute.to_s + def nulls_order_column(order_value) + attribute = order_value.expr.expr + + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute.name, + column_expression: attribute, + order_expression: order_value, + reversed_order_expression: order_value.reverse, + order_direction: order_value.expr.direction, + nullable: order_value.is_a?(Arel::Nodes::NullsLast) ? :nulls_last : :nulls_first, + distinct: false + ) end - def ordered_by_other_column? + def lower_named_function_column(order_value) + attribute_name = order_value.expr.expressions.first.name + + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute_name, + column_expression: Arel::Nodes::NamedFunction.new("LOWER", [model_class.arel_table[attribute_name]]), + order_expression: order_value, + nullable: nullability(order_value, attribute_name), + distinct: false + ) + end + + def ordered_by_primary_key? return unless order_values.one? attribute = order_values.first.try(:expr) + attribute && primary_key?(attribute) + end - return unless attribute - return unless attribute.try(:name) + def ordered_by_other_column? + return unless order_values.one? - model_class.column_names.include?(attribute.name.to_s) + convert_raw_nulls_order! + + supported_column?(order_values.first) end def ordered_by_other_column_with_tie_breaker? return unless order_values.size == 2 - attribute = order_values.first.try(:expr) - tie_breaker_attribute = order_values.second.try(:expr) + convert_raw_nulls_order! - return unless attribute - return unless tie_breaker_attribute - return unless attribute.respond_to?(:name) + return unless supported_column?(order_values.first) - model_class.column_names.include?(attribute.name.to_s) && - arel_table[primary_key].to_s == tie_breaker_attribute.to_s + tie_breaker_attribute = order_values.second.try(:expr) + tie_breaker_attribute && primary_key?(tie_breaker_attribute) end def default_tie_breaker_column_order diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index fca75d1fe01..00304f48dc5 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -11,8 +11,8 @@ module Gitlab @request_context = request_context end - def paginate(relation, exclude_total_headers: false) - paginate_with_limit_optimization(add_default_order(relation)).tap do |data| + def paginate(relation, exclude_total_headers: false, skip_default_order: false) + paginate_with_limit_optimization(add_default_order(relation, skip_default_order: skip_default_order)).tap do |data| add_pagination_headers(data, exclude_total_headers) end end @@ -27,7 +27,6 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops, default_enabled: :yaml) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT @@ -47,7 +46,9 @@ module Gitlab false end - def add_default_order(relation) + def add_default_order(relation, skip_default_order: false) + return relation if skip_default_order + if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/patch/legacy_database_config.rb b/lib/gitlab/patch/database_config.rb index 6040f737c75..702e8d404b1 100644 --- a/lib/gitlab/patch/legacy_database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -28,7 +28,7 @@ module Gitlab module Patch - module LegacyDatabaseConfig + module DatabaseConfig extend ActiveSupport::Concern prepended do @@ -73,23 +73,34 @@ module Gitlab @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables super.to_h do |env, configs| - # This check is taken from Rails where the transformation - # of a flat database.yml is done into `primary:` - # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 - if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } - configs = { "main" => configs } - - @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + # TODO: To be removed in 15.0. See https://gitlab.com/gitlab-org/gitlab/-/issues/338182 + # This preload is needed to convert legacy `database.yml` + # from `production: adapter: postgresql` + # into a `production: main: adapter: postgresql` + unless Gitlab::Utils.to_boolean(ENV['SKIP_DATABASE_CONFIG_VALIDATION'], default: false) + # This check is taken from Rails where the transformation + # of a flat database.yml is done into `primary:` + # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 + if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } + configs = { "main" => configs } + + @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + end end - if Gitlab.ee? && File.exist?(Rails.root.join("config/database_geo.yml")) - migrations_paths = ["ee/db/geo/migrate"] - migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + if Gitlab.ee? + if !configs.key?("geo") && File.exist?(Rails.root.join("config/database_geo.yml")) + configs["geo"] = Rails.application.config_for(:database_geo).stringify_keys + end + + if configs.key?("geo") + migrations_paths = Array(configs["geo"]["migrations_paths"]) + migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty? + migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] - configs["geo"] = - Rails.application.config_for(:database_geo) - .merge(migrations_paths: migrations_paths, schema_migrations_path: "ee/db/geo/schema_migrations") - .stringify_keys + configs["geo"]["migrations_paths"] = migrations_paths.uniq + configs["geo"]["schema_migrations_path"] = "ee/db/geo/schema_migrations" if configs["geo"]["schema_migrations_path"].blank? + end end [env, configs] diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 847f70693f3..e7a12edf763 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -29,7 +29,7 @@ module Gitlab end def project_path - URI.parse(preview).path.sub(%r{\A/}, '') + URI.parse(preview).path.delete_prefix('/') end def uri_encoded_project_path @@ -57,7 +57,7 @@ module Gitlab ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'), ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), - ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman', 'illustrations/logos/middleman.svg'), + ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'), ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index e6a73c71e85..4efa29337d1 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -24,7 +24,7 @@ module Gitlab end execution_message do if params[:merge_request_diff_head_sha].blank? - _("Merge request diff sha parameter is required for the merge quick action.") + _("The `/merge` quick action requires the SHA of the head of the branch.") elsif params[:merge_request_diff_head_sha] != quick_action_target.diff_head_sha _("Branch has been updated since the merge was requested.") elsif preferred_strategy = preferred_auto_merge_strategy(quick_action_target) @@ -291,7 +291,7 @@ module Gitlab parse_params do |attention_param| extract_users(attention_param) end - command :attention do |users| + command :attention, :attn do |users| next if users.empty? users.each do |user| diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index 98e52e8e767..ac0598d8d34 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -84,7 +84,7 @@ module Gitlab # MAX(relative_position) without the GROUP BY, due to index usage: # https://gitlab.com/gitlab-org/gitlab-foss/issues/54276#note_119340977 relation = scoped_items - .order(Gitlab::Database.nulls_last_order('position', 'DESC')) + .order(Arel.sql('position').desc.nulls_last) .group(grouping_column) .limit(1) @@ -101,7 +101,7 @@ module Gitlab def max_sibling sib = relative_siblings - .order(Gitlab::Database.nulls_last_order('relative_position', 'DESC')) + .order(model_class.arel_table[:relative_position].desc.nulls_last) .first neighbour(sib) @@ -109,7 +109,7 @@ module Gitlab def min_sibling sib = relative_siblings - .order(Gitlab::Database.nulls_last_order('relative_position', 'ASC')) + .order(model_class.arel_table[:relative_position].asc.nulls_last) .first neighbour(sib) diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb index 381adda7991..14883a34950 100644 --- a/lib/gitlab/security/scan_configuration.rb +++ b/lib/gitlab/security/scan_configuration.rb @@ -31,6 +31,8 @@ module Gitlab def configuration_path; end + def meta_info_path; end + private attr_reader :project, :configured diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index e2df60c46f1..ec514adafc8 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -4,12 +4,24 @@ module Gitlab class Seeder extend ActionView::Helpers::NumberHelper - MASS_INSERT_PROJECT_START = 'mass_insert_project_' - MASS_INSERT_USER_START = 'mass_insert_user_' + MASS_INSERT_PREFIX = 'mass_insert' + MASS_INSERT_PROJECT_START = "#{MASS_INSERT_PREFIX}_project_" + MASS_INSERT_GROUP_START = "#{MASS_INSERT_PREFIX}_group_" + MASS_INSERT_USER_START = "#{MASS_INSERT_PREFIX}_user_" REPORTED_USER_START = 'reported_user_' - ESTIMATED_INSERT_PER_MINUTE = 2_000_000 + ESTIMATED_INSERT_PER_MINUTE = 250_000 MASS_INSERT_ENV = 'MASS_INSERT' + module NamespaceSeed + extend ActiveSupport::Concern + + included do + scope :not_mass_generated, -> do + where.not("path LIKE '#{MASS_INSERT_GROUP_START}%'") + end + end + end + module ProjectSeed extend ActiveSupport::Concern @@ -30,6 +42,10 @@ module Gitlab end end + def self.log_message(message) + puts "#{Time.current}: #{message}" + end + def self.with_mass_insert(size, model) humanized_model_name = model.is_a?(String) ? model : model.model_name.human.pluralize(size) @@ -63,6 +79,7 @@ module Gitlab def self.quiet # Additional seed logic for models. + Namespace.include(NamespaceSeed) Project.include(ProjectSeed) User.include(UserSeed) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index bc0071f6333..a498e329c3f 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -98,7 +98,7 @@ module Gitlab storages << { name: key, path: storage_paths[key] } end - config = { socket_path: address.sub(/\Aunix:/, '') } + config = { socket_path: address.delete_prefix('unix:') } if Rails.env.test? socket_filename = options[:gitaly_socket] || "gitaly.socket" @@ -124,9 +124,9 @@ module Gitlab config[:storage] = storages - internal_socket_dir = options[:internal_socket_dir] || File.join(gitaly_dir, 'internal_sockets') - FileUtils.mkdir(internal_socket_dir) unless File.exist?(internal_socket_dir) - config[:internal_socket_dir] = internal_socket_dir + runtime_dir = options[:runtime_dir] || File.join(gitaly_dir, 'run') + FileUtils.mkdir(runtime_dir) unless File.exist?(runtime_dir) + config[:runtime_dir] = runtime_dir config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 8a2f3bbe0ee..78682a89655 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -15,16 +15,24 @@ module Gitlab Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) ].freeze + def self.technologies + if Gitlab::FIPS.enabled? + Gitlab::FIPS::SSH_KEY_TECHNOLOGIES + else + TECHNOLOGIES + end + end + def self.technology(name) - TECHNOLOGIES.find { |tech| tech.name.to_s == name.to_s } + technologies.find { |tech| tech.name.to_s == name.to_s } end def self.technology_for_key(key) - TECHNOLOGIES.find { |tech| key.instance_of?(tech.key_class) } + technologies.find { |tech| key.instance_of?(tech.key_class) } end def self.supported_types - TECHNOLOGIES.map(&:name) + technologies.map(&:name) end def self.supported_sizes(name) @@ -32,7 +40,7 @@ module Gitlab end def self.supported_algorithms - TECHNOLOGIES.flat_map { |tech| tech.supported_algorithms } + technologies.flat_map { |tech| tech.supported_algorithms } end def self.supported_algorithms_for_name(name) diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb index 5bca3efe6e1..fcf30cd6df9 100644 --- a/lib/gitlab/suggestions/commit_message.rb +++ b/lib/gitlab/suggestions/commit_message.rb @@ -13,7 +13,7 @@ module Gitlab end def message - project = suggestion_set.project + project = suggestion_set.target_project user_defined_message = @custom_message.presence || project.suggestion_commit_message.presence message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE @@ -37,8 +37,8 @@ module Gitlab 'branch_name' => ->(user, suggestion_set) { suggestion_set.branch }, 'files_count' => ->(user, suggestion_set) { suggestion_set.file_paths.length }, 'file_paths' => ->(user, suggestion_set) { format_paths(suggestion_set.file_paths) }, - 'project_name' => ->(user, suggestion_set) { suggestion_set.project.name }, - 'project_path' => ->(user, suggestion_set) { suggestion_set.project.path }, + 'project_name' => ->(user, suggestion_set) { suggestion_set.target_project.name }, + 'project_path' => ->(user, suggestion_set) { suggestion_set.target_project.path }, 'user_full_name' => ->(user, suggestion_set) { user.name }, 'username' => ->(user, suggestion_set) { user.username }, 'suggestions_count' => ->(user, suggestion_set) { suggestion_set.suggestions.size } diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb index 53885cdbf19..21a5acf8afe 100644 --- a/lib/gitlab/suggestions/suggestion_set.rb +++ b/lib/gitlab/suggestions/suggestion_set.rb @@ -9,8 +9,12 @@ module Gitlab @suggestions = suggestions end - def project - first_suggestion.project + def source_project + first_suggestion.source_project + end + + def target_project + first_suggestion.target_project end def branch diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 6a98fa12903..54db31ffd6c 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -198,3 +198,4 @@ module Gitlab end end end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index 67ecf498cf7..87861b61119 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -8,7 +8,8 @@ module Gitlab CUSTOM_DAY_AND_MONTH_LENGTH = { hours_per_day: 8, days_per_month: 20 }.freeze def parse(string) - string = string.sub(/\A-/, '') + negative_time = string.start_with?('-') + string = string.delete_prefix('-') seconds = begin @@ -19,7 +20,7 @@ module Gitlab nil end - seconds *= -1 if seconds && Regexp.last_match + seconds *= -1 if seconds && negative_time seconds end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index a58b4beb0df..0e7812d08b8 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -15,6 +15,21 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end + def definition(basename, category: nil, action: nil, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists + definition = YAML.load_file(Rails.root.join("config/events/#{basename}.yml")) + + dispatch_from_definition(definition, label: label, property: property, value: value, context: context, project: project, user: user, namespace: namespace, **extra) + end + + def dispatch_from_definition(definition, **event_data) + definition = definition.with_indifferent_access + + category ||= definition[:category] + action ||= definition[:action] + + event(category, action, **event_data) + end + def options(group) snowplow.options(group) end @@ -39,3 +54,5 @@ module Gitlab end end end + +Gitlab::Tracking.prepend_mod_with('Gitlab::Tracking') diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index fa40a8b678b..e3bf11b00b4 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -71,7 +71,10 @@ module Gitlab url.sub!("#{raw_credentials}@", '') user, _, password = raw_credentials.partition(':') - @credentials ||= { user: user.presence, password: password.presence } + + @credentials ||= {} + @credentials[:user] = user.presence if @credentials[:user].blank? + @credentials[:password] = password.presence if @credentials[:password].blank? end url = Addressable::URI.parse(url) diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb index e04e2e589b2..6cc67321ba1 100644 --- a/lib/gitlab/usage/service_ping/instrumented_payload.rb +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -22,7 +22,7 @@ module Gitlab private - # Not all metrics defintions have instrumentation classes + # Not all metrics definitions have instrumentation classes # The value can be computed only for those that have it def instrumented_metrics_defintions Gitlab::Usage::MetricDefinition.with_instrumentation_class diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index 794f3373043..3e653b186a0 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -18,16 +18,11 @@ module Gitlab private def with_instrumentation_classes(old_payload, output_method) - if Feature.enabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths - instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths + instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build - instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build - - old_payload.deep_merge(instrumented_payload) - else - old_payload - end + old_payload.deep_merge(instrumented_payload) end def all_metrics_values(cached) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 951ec5ea5c3..b465d4bcc9b 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -70,7 +70,7 @@ module Gitlab def system_usage_data issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - counts = { + { counts: { assignee_lists: count(List.assignee), ci_builds: count(::Ci::Build), @@ -166,12 +166,6 @@ module Gitlab data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } - - if Feature.disabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) - counts[:counts][:boards] = add_metric('CountBoardsMetric', time_frame: 'all') - end - - counts end # rubocop: enable Metrics/AbcSize @@ -513,7 +507,6 @@ module Gitlab { deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id), keys: distinct_count(::Key.regular_keys.where(time_period), :user_id), - merge_requests: distinct_count(::MergeRequest.where(time_period), :author_id), projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), @@ -801,14 +794,9 @@ module Gitlab sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) - Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result| + Users::InProductMarketingEmail::ACTIVE_TRACKS.keys.each_with_object({}) do |track, result| + series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) # rubocop: enable UsageData/LargeTable: - series_amount = - if track.to_sym == Namespaces::InviteTeamEmailService::TRACK - 0 - else - Namespaces::InProductMarketingEmailsService::TRACKS[track.to_sym][:interval_days].count - end 0.upto(series_amount - 1).map do |series| # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index b8de7de848d..cf3caf3f0c7 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -6,13 +6,18 @@ module Gitlab::UsageDataCounters KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__) class << self - def track_unique_project_event(project_id:, template:, config_source:) + def track_unique_project_event(project:, template:, config_source:, user:) expanded_template_name = expand_template_name(template) return unless expanded_template_name Gitlab::UsageDataCounters::HLLRedisCounter.track_event( - ci_template_event_name(expanded_template_name, config_source), values: project_id + ci_template_event_name(expanded_template_name, config_source), values: project.id ) + + namespace = project.namespace + if Feature.enabled?(:route_hll_to_snowplow, namespace, default_enabled: :yaml) + Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, user: user, project: project) + end end def ci_templates(relative_base = 'lib/gitlab/ci/templates') diff --git a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb new file mode 100644 index 00000000000..8a57a0331b8 --- /dev/null +++ b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module GitLabCliActivityUniqueCounter + GITLAB_CLI_API_REQUEST_ACTION = 'i_code_review_user_gitlab_cli_api_request' + GITLAB_CLI_USER_AGENT_REGEX = /GitLab\sCLI$/.freeze + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(GITLAB_CLI_USER_AGENT_REGEX) && track_unique_action_by_user(GITLAB_CLI_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 474ab9a4dd9..3b34cd77cf5 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -81,6 +81,12 @@ module Gitlab track(values, event_name, context: context, time: time) end + # Count unique events for a given time range. + # + # event_names - The list of the events to count. + # start_date - The start date of the time range. + # end_date - The end date of the time range. + # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| raise SlotMismatch, events unless events_in_same_slot?(events) @@ -100,6 +106,13 @@ module Gitlab known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } end + # Recent 7 or 28 days unique events data for events defined in /lib/gitlab/usage_data_counters/known_events/ + # + # - For metrics for which we store a key per day, we have the last 7 days or last 28 days of data. + # - For metrics for which we store a key per week, we have the last complete week or last 4 complete weeks + # daily or weekly information is in the file we have for events definition /lib/gitlab/usage_data_counters/known_events/ + # - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store. + # - The aggregation used doesn't affect data granulation. def unique_events_data categories.each_with_object({}) do |category, category_results| events_names = events_for_category(category) diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index a39fa7aca4f..f179f6d679d 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -219,6 +219,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_themekit + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_terraform category: ci_templates redis_slot: ci_templates @@ -615,3 +619,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_liquibase + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_matlab + category: ci_templates + redis_slot: ci_templates + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 42c51ec3921..df2864bba89 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -132,6 +132,11 @@ category: code_review aggregation: weekly feature_flag: usage_data_i_code_review_user_jetbrains_api_request +- name: i_code_review_user_gitlab_cli_api_request + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_gitlab_cli_api_request - name: i_code_review_user_create_mr_from_issue redis_slot: code_review category: code_review @@ -173,62 +178,50 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_click_single_file_mode_setting redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_click_file_browser_setting redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_click_whitespace_setting redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_view_inline redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_view_parallel redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_file_browser_tree_view redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_file_browser_list_view redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_show_whitespace redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_hide_whitespace redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_single_file redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_diff_multiple_files redis_slot: code_review category: code_review aggregation: weekly - feature_flag: diff_settings_usage_data - name: i_code_review_user_load_conflict_ui redis_slot: code_review category: code_review @@ -241,7 +234,6 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_diff_searches - name: i_code_review_total_suggestions_applied redis_slot: code_review category: code_review diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index fdf4bc58525..0d89a5181ec 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -25,25 +25,21 @@ redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: g_edit_by_sfe category: ide_edit redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: g_edit_by_sse category: ide_edit redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily - feature_flag: track_editor_edit_actions - name: i_search_total category: search redis_slot: search @@ -343,22 +339,18 @@ redis_slot: secure category: secure aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_expanding_testing_code_quality_report redis_slot: testing category: testing aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_expanding_testing_accessibility_report redis_slot: testing category: testing aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_expanding_testing_license_compliance_report redis_slot: testing category: testing aggregation: weekly - feature_flag: users_expanding_widgets_usage_data - name: users_visiting_testing_license_compliance_full_report redis_slot: testing category: testing diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 62b0d6dea86..82787b7bf29 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -188,3 +188,33 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_epic_related_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_related_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_blocking_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_blocking_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_blocked_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml index a56e0a6d370..d80b711f8eb 100644 --- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -3,9 +3,7 @@ category: error_tracking redis_slot: error_tracking aggregation: weekly - feature_flag: track_error_tracking_activity - name: error_tracking_view_list category: error_tracking redis_slot: error_tracking aggregation: weekly - feature_flag: track_error_tracking_activity diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index d40ac71afc6..977cc3549d8 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -50,7 +50,7 @@ module Gitlab def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? - { alt_usage_data_block: block.to_s } + { alt_usage_data_block: "non-SQL usage data block" } else { alt_usage_data_value: value } end @@ -58,9 +58,9 @@ module Gitlab def redis_usage_data(counter = nil, &block) if block_given? - { redis_usage_data_block: block.to_s } + { redis_usage_data_block: "non-SQL usage data block" } elsif counter.present? - { redis_usage_data_counter: counter } + { redis_usage_data_counter: counter.to_s } end end @@ -74,6 +74,13 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end + + def topology_usage_data + { + duration_s: 0, + failures: [] + } + end end end end diff --git a/lib/gitlab/utils/delegator_override/validator.rb b/lib/gitlab/utils/delegator_override/validator.rb index 402154b41c2..4449fa75877 100644 --- a/lib/gitlab/utils/delegator_override/validator.rb +++ b/lib/gitlab/utils/delegator_override/validator.rb @@ -28,7 +28,13 @@ module Gitlab end def add_target(target_class) - @target_classes << target_class if target_class + return unless target_class + + @target_classes << target_class + + # Also include all descendants inheriting from the target, + # to make sure we catch methods that are only defined in some of them. + @target_classes += target_class.descendants end # This will make sure allowlist we put into ancestors are all included diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 3bacad72050..a2d217fb42f 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -11,15 +11,19 @@ module Gitlab include Gitlab::Routing include Gitlab::Allowable - attr_reader :subject + # Presenters should always access the subject through an explicit getter defined with + # `presents ..., as:`, the `__subject__` method is only intended for internal use. + def __subject__ + @subject + end def can?(user, action, overridden_subject = nil) - super(user, action, overridden_subject || subject) + super(user, action, overridden_subject || __subject__) end # delegate all #can? queries to the subject def declarative_policy_delegate - subject + __subject__ end def present(**attributes) @@ -31,15 +35,15 @@ module Gitlab end def is_a?(type) - super || subject.is_a?(type) + super || __subject__.is_a?(type) end def web_url - url_builder.build(subject) + url_builder.build(__subject__) end def web_path - url_builder.build(subject, only_path: true) + url_builder.build(__subject__, only_path: true) end class_methods do @@ -58,7 +62,7 @@ module Gitlab # no-op end - define_method(as) { subject } if as + define_method(as) { __subject__ } if as end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 19d30daa577..d74efd458f6 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -226,6 +226,13 @@ module Gitlab end end + def detect_content_type + [ + Gitlab::Workhorse::DETECT_HEADER, + 'true' + ] + end + protected # This is the outermost encoding of a senddata: header. It is safe for diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 9374c5c8f8f..5d5d10b42f0 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -27,6 +27,11 @@ module Mattermost LEASE_TIMEOUT = 60 + Request = Struct.new(:parameters, keyword_init: true) do + def method_missing(method_name, *args, &block) + end + end + attr_accessor :current_resource_owner, :token, :base_uri def initialize(current_user) @@ -64,7 +69,7 @@ module Mattermost end def request - @request ||= OpenStruct.new(parameters: params) + @request ||= Request.new(parameters: params) end def params diff --git a/lib/prometheus/cleanup_multiproc_dir_service.rb b/lib/prometheus/cleanup_multiproc_dir_service.rb index 6418b4de166..b309247fa73 100644 --- a/lib/prometheus/cleanup_multiproc_dir_service.rb +++ b/lib/prometheus/cleanup_multiproc_dir_service.rb @@ -2,22 +2,17 @@ module Prometheus class CleanupMultiprocDirService - include Gitlab::Utils::StrongMemoize - - def execute - FileUtils.rm_rf(old_metrics) if old_metrics + def initialize(metrics_dir) + @metrics_dir = metrics_dir end - private + def execute + return if @metrics_dir.blank? - def old_metrics - strong_memoize(:old_metrics) do - Dir[File.join(multiprocess_files_dir, '*.db')] if multiprocess_files_dir - end - end + files_to_delete = Dir[File.join(@metrics_dir, '*.db')] + return if files_to_delete.blank? - def multiprocess_files_dir - ::Prometheus::Client.configuration.multiprocess_files_dir + FileUtils.rm_rf(files_to_delete) end end end diff --git a/lib/sidebars/groups/menus/group_information_menu.rb b/lib/sidebars/groups/menus/group_information_menu.rb index 9656811455e..3ce99e14a04 100644 --- a/lib/sidebars/groups/menus/group_information_menu.rb +++ b/lib/sidebars/groups/menus/group_information_menu.rb @@ -20,7 +20,7 @@ module Sidebars override :sprite_icon def sprite_icon - 'group' + context.group.subgroup? ? 'subgroup' : 'group' end override :active_routes diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index c012b3bb627..7bd9ac91efa 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -90,7 +90,10 @@ module Sidebars end def google_cloud_menu_item - feature_is_enabled = Feature.enabled?(:incubation_5mp_google_cloud, context.project) + enabled_for_user = Feature.enabled?(:incubation_5mp_google_cloud, context.current_user) + enabled_for_group = Feature.enabled?(:incubation_5mp_google_cloud, context.project.group) + enabled_for_project = Feature.enabled?(:incubation_5mp_google_cloud, context.project) + feature_is_enabled = enabled_for_user || enabled_for_group || enabled_for_project user_has_permissions = can?(context.current_user, :admin_project_google_cloud, context.project) unless feature_is_enabled && user_has_permissions diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb index 16335f5b076..5de70ea7d7f 100644 --- a/lib/sidebars/projects/menus/learn_gitlab_menu.rb +++ b/lib/sidebars/projects/menus/learn_gitlab_menu.rb @@ -45,9 +45,9 @@ module Sidebars } end - override :image_path - def image_path - 'learn_gitlab/graduation_hat.svg' + override :sprite_icon + def sprite_icon + 'bulb' end override :render? diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 77f09986b19..d82a02a342f 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -47,7 +47,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Container Registry'), link: project_container_registry_index_path(context.project), - active_routes: { controller: :repositories }, + active_routes: { controller: 'projects/registry/repositories' }, item_id: :container_registry ) end @@ -71,7 +71,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Harbor Registry'), link: project_harbor_registry_index_path(context.project), - active_routes: { controller: :harbor_registry }, + active_routes: { controller: 'projects/harbor/repositories' }, item_id: :harbor_registry ) end diff --git a/lib/sidebars/projects/menus/zentao_menu.rb b/lib/sidebars/projects/menus/zentao_menu.rb index db9e60326a4..1b5ba900a86 100644 --- a/lib/sidebars/projects/menus/zentao_menu.rb +++ b/lib/sidebars/projects/menus/zentao_menu.rb @@ -4,11 +4,6 @@ module Sidebars module Projects module Menus class ZentaoMenu < ::Sidebars::Menu - override :configure_menu_items - def configure_menu_items - render?.tap { |render| add_items if render } - end - override :link def link zentao_integration.url @@ -16,7 +11,7 @@ module Sidebars override :title def title - s_('ZentaoIntegration|ZenTao issues') + s_('ZentaoIntegration|ZenTao') end override :title_html_options @@ -26,9 +21,9 @@ module Sidebars } end - override :image_path - def image_path - 'logos/zentao.svg' + override :sprite_icon + def sprite_icon + 'external-link' end # Hardcode sizes so image doesn't flash before CSS loads https://gitlab.com/gitlab-org/gitlab/-/issues/321022 @@ -46,29 +41,11 @@ module Sidebars zentao_integration.active? end - def add_items - add_item(open_zentao_menu_item) - end - private def zentao_integration @zentao_integration ||= context.project.zentao_integration end - - def open_zentao_menu_item - ::Sidebars::MenuItem.new( - title: s_('ZentaoIntegration|Open ZenTao'), - link: zentao_integration.url, - active_routes: {}, - item_id: :open_zentao, - sprite_icon: 'external-link', - container_html_options: { - target: '_blank', - rel: 'noopener noreferrer' - } - ) - end end end end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index 2876f1eb688..3ae36087f6b 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -31,7 +31,7 @@ module SystemCheck end try_fixing_it("mkdir #{backup_dir}", *instructions) - for_more_information('doc/ssh/index.md in section "Overriding SSH settings on the GitLab server"') + for_more_information('doc/user/ssh.md#overriding-ssh-settings-on-the-gitlab-server') fix_and_rerun end diff --git a/lib/system_check/base_check.rb b/lib/system_check/base_check.rb index c36cacbaf4f..ae3a9412e5c 100644 --- a/lib/system_check/base_check.rb +++ b/lib/system_check/base_check.rb @@ -64,20 +64,14 @@ module SystemCheck call_or_return(@skip_reason) || 'skipped' end - # Define a reason why we skipped the SystemCheck (during runtime) + # Define or get a reason why we skipped the SystemCheck (during runtime) # # This is used when you need dynamic evaluation like when you have # multiple reasons why a check can fail # # @param [String] reason to be displayed - attr_writer :skip_reason - - # Skip reason defined during runtime - # - # This value have precedence over the one defined in the subclass - # - # @return [String] the reason - attr_reader :skip_reason + # @return [String] reason to be displayed + attr_accessor :skip_reason # Does the check support automatically repair routine? # diff --git a/lib/tasks/ci/build_artifacts.rake b/lib/tasks/ci/build_artifacts.rake deleted file mode 100644 index 4f4faef5a62..00000000000 --- a/lib/tasks/ci/build_artifacts.rake +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -require 'httparty' -require 'csv' - -namespace :ci do - namespace :build_artifacts do - desc "GitLab | CI | Fetch projects with incorrect artifact size on GitLab.com" - task :project_with_incorrect_artifact_size do - csv_url = ENV['SISENSE_PROJECT_IDS_WITH_INCORRECT_ARTIFACTS_URL'] - - # rubocop: disable Gitlab/HTTParty - body = HTTParty.get(csv_url) - # rubocop: enable Gitlab/HTTParty - - table = CSV.parse(body.parsed_response, headers: true) - puts table['PROJECT_ID'].join(' ') - end - end -end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 99ffeb4ec0b..42b12cd0ae3 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -10,7 +10,12 @@ namespace :dev do Gitlab::Database::EachDatabase.each_database_connection do |connection| # Make sure DB statistics are up to date. + # gitlab:setup task can insert quite a bit of data, especially with MASS_INSERT=1 + # so ANALYZE can take more than default 15s statement timeout. This being a dev task, + # we disable the statement timeout for ANALYZE to run and enable it back afterwards. + connection.execute('SET statement_timeout TO 0') connection.execute('ANALYZE') + connection.execute('RESET statement_timeout') end Rake::Task["gitlab:shell:setup"].invoke @@ -21,4 +26,51 @@ namespace :dev do Rails.configuration.eager_load = true Rails.application.eager_load! end + + # If there are any clients connected to the DB, PostgreSQL won't let + # you drop the database. It's possible that Sidekiq, Puma, or + # some other client will be hanging onto a connection, preventing + # the DROP DATABASE from working. To workaround this problem, this + # method terminates all the connections so that a subsequent DROP + # will work. + desc "Used to drop all connections in development" + task :terminate_all_connections do + # In production, we might want to prevent ourselves from shooting + # ourselves in the foot, so let's only do this in a test or + # development environment. + unless Rails.env.production? + cmd = <<~SQL + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid(); + SQL + + Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection| + connection.execute(cmd) + rescue ActiveRecord::NoDatabaseError + end + end + end + + databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml + + namespace :copy_db do + ALLOWED_DATABASES = %w[ci].freeze + + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + next unless ALLOWED_DATABASES.include?(name) + + desc "Copies the #{name} database from the main database" + task name => :environment do + Rake::Task["dev:terminate_all_connections"].invoke + + db_config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: name) + + ApplicationRecord.connection.create_database(db_config.database, template: ApplicationRecord.connection_db_config.database) + rescue ActiveRecord::DatabaseAlreadyExists + warn "Database '#{db_config.database}' already exists" + end + end + end end diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake index b1084495f3d..e0699d5eb41 100644 --- a/lib/tasks/gitlab/background_migrations.rake +++ b/lib/tasks/gitlab/background_migrations.rake @@ -80,8 +80,8 @@ namespace :gitlab do def display_migration_status(database_name, connection) Gitlab::Database::SharedModel.using_connection(connection) do - statuses = Gitlab::Database::BackgroundMigration::BatchedMigration.statuses - max_status_length = statuses.keys.map(&:length).max + valid_status = Gitlab::Database::BackgroundMigration::BatchedMigration.valid_status + max_status_length = valid_status.map(&:length).max format_string = "%-#{max_status_length}s | %s\n" puts "Database: #{database_name}\n" @@ -94,7 +94,7 @@ namespace :gitlab do migration.job_arguments.to_json ].join(',') - printf(format_string, migration.status, identification_fields) + printf(format_string, migration.status_name, identification_fields) end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 50ceb11581e..3a7e53a27e4 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -2,6 +2,14 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml +def each_database(databases, include_geo: false) + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| + next if !include_geo && database == 'geo' + + yield database + end +end + namespace :gitlab do namespace :db do desc 'GitLab | DB | Manually insert schema migration version on all configured databases' @@ -10,10 +18,10 @@ namespace :gitlab do end namespace :mark_migration_complete do - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| - desc "Gitlab | DB | Manually insert schema migration version on #{database} database" - task database, [:version] => :environment do |_, args| - mark_migration_complete(args[:version], only_on: database) + each_database(databases) do |database_name| + desc "Gitlab | DB | Manually insert schema migration version on #{database_name} database" + task database_name, [:version] => :environment do |_, args| + mark_migration_complete(args[:version], only_on: database_name) end end end @@ -39,10 +47,10 @@ namespace :gitlab do end namespace :drop_tables do - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| - desc "GitLab | DB | Drop all tables on the #{database} database" - task database => :environment do - drop_tables(only_on: database) + each_database(databases) do |database_name| + desc "GitLab | DB | Drop all tables on the #{database_name} database" + task database_name => :environment do + drop_tables(only_on: database_name) end end end @@ -76,16 +84,38 @@ namespace :gitlab do desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed' task configure: :environment do - # Check if we have existing db tables - # The schema_migrations table will still exist if drop_tables was called - if ActiveRecord::Base.connection.tables.count > 1 - Rake::Task['db:migrate'].invoke + databases_with_tasks = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env) + + databases_loaded = [] + + if databases_with_tasks.size == 1 + next unless databases_with_tasks.first.name == 'main' + + connection = Gitlab::Database.database_base_models['main'].connection + databases_loaded << configure_database(connection) else - # Add post-migrate paths to ensure we mark all migrations as up + Gitlab::Database.database_base_models.each do |name, model| + next unless databases_with_tasks.any? { |db_with_tasks| db_with_tasks.name == name } + + databases_loaded << configure_database(model.connection, database_name: name) + end + end + + Rake::Task['db:seed_fu'].invoke if databases_loaded.present? && databases_loaded.all? + end + + def configure_database(connection, database_name: nil) + database_name = ":#{database_name}" if database_name + load_database = connection.tables.count <= 1 + + if load_database Gitlab::Database.add_post_migrate_path_to_rails(force: true) - Rake::Task['db:structure:load'].invoke - Rake::Task['db:seed_fu'].invoke + Rake::Task["db:schema:load#{database_name}"].invoke + else + Rake::Task["db:migrate#{database_name}"].invoke end + + load_database end desc 'GitLab | DB | Run database migrations and print `unattended_migrations_completed` if action taken' @@ -155,6 +185,15 @@ namespace :gitlab do Gitlab::Database::Partitioning.sync_partitions end + namespace :create_dynamic_partitions do + each_database(databases) do |database_name| + desc "Create missing dynamic database partitions on the #{database_name} database" + task database_name => :environment do + Gitlab::Database::Partitioning.sync_partitions(only_on: database_name) + end + end + end + # This is targeted towards deploys and upgrades of GitLab. # Since we're running migrations already at this time, # we also check and create partitions as needed here. @@ -162,14 +201,12 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| - # We'll temporarily skip this enhancement for geo, since in some situations we - # wish to setup the geo database before the other databases have been setup, - # and partition management attempts to connect to the main database. - next if name == 'geo' - - Rake::Task["db:migrate:#{name}"].enhance do - Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + # We'll temporarily skip this enhancement for geo, since in some situations we + # wish to setup the geo database before the other databases have been setup, + # and partition management attempts to connect to the main database. + each_database(databases) do |database_name| + Rake::Task["db:migrate:#{database_name}"].enhance do + Rake::Task["gitlab:db:create_dynamic_partitions:#{database_name}"].invoke end end @@ -185,25 +222,17 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| - # We'll temporarily skip this enhancement for geo, since in some situations we - # wish to setup the geo database before the other databases have been setup, - # and partition management attempts to connect to the main database. - next if name == 'geo' - - Rake::Task["db:schema:load:#{name}"].enhance do - Rake::Task['gitlab:db:create_dynamic_partitions'].invoke + # We'll temporarily skip this enhancement for geo, since in some situations we + # wish to setup the geo database before the other databases have been setup, + # and partition management attempts to connect to the main database. + each_database(databases) do |database_name| + # :nocov: + Rake::Task["db:schema:load:#{database_name}"].enhance do + Rake::Task["gitlab:db:create_dynamic_partitions:#{database_name}"].invoke end + # :nocov: end - desc "Clear all connections" - task :clear_all_connections do - ActiveRecord::Base.clear_all_connections! - end - - Rake::Task['db:test:purge'].enhance(['gitlab:db:clear_all_connections']) - Rake::Task['db:drop'].enhance(['gitlab:db:clear_all_connections']) - # During testing, db:test:load restores the database schema from scratch # which does not include dynamic partitions. We cannot rely on application # initializers here as the application can continue to run while @@ -229,7 +258,7 @@ namespace :gitlab do end namespace :reindex do - ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database_name| + each_database(databases) do |database_name| desc "Reindex #{database_name} database without downtime to eliminate bloat" task database_name => :environment do unless Gitlab::Database::Reindexing.enabled? @@ -292,13 +321,22 @@ namespace :gitlab do task down: :environment do Gitlab::Database::Migrations::Runner.down.run end + + desc 'Sample traditional background migrations with instrumentation' + task :sample_background_migrations, [:duration_s] => [:environment] do |_t, args| + duration = args[:duration_s]&.to_i&.seconds || 30.minutes # Default of 30 minutes + + Gitlab::Database::Migrations::Runner.background_migrations.run_jobs(for_duration: duration) + end end desc 'Run all pending batched migrations' task execute_batched_migrations: :environment do - Gitlab::Database::BackgroundMigration::BatchedMigration.active.queue_order.each do |migration| - Gitlab::AppLogger.info("Executing batched migration #{migration.id} inline") - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(migration) + Gitlab::Database::EachDatabase.each_database_connection do |connection, name| + Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:active).queue_order.each do |migration| + Gitlab::AppLogger.info("Executing batched migration #{migration.id} on database #{name} inline") + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new(connection: connection).run_entire_migration(migration) + end end end diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake new file mode 100644 index 00000000000..cc5f6bb6e09 --- /dev/null +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml + +namespace :gitlab do + namespace :db do + desc 'Validates `config/database.yml` to ensure a correct behavior is configured' + task validate_config: :environment do + original_db_config = ActiveRecord::Base.connection_db_config + + # The include_replicas: is a legacy name to fetch all hidden entries (replica: true or database_tasks: false) + # Once we upgrade to Rails 7.x this should be changed to `include_hidden: true` + # Ref.: https://github.com/rails/rails/blob/f2d9316ba965e150ad04596085ee10eea4f58d3e/activerecord/lib/active_record/database_configurations.rb#L48 + db_configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, include_replicas: true) + db_configs = db_configs.reject(&:replica?) + + # Map each database connection into unique identifier of system+database + all_connections = db_configs.map do |db_config| + identifier = + begin + ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection + ActiveRecord::Base.connection.select_one("SELECT system_identifier, current_database() FROM pg_control_system()") + rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err + warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" + rescue ActiveRecord::NoDatabaseError + end + + { + name: db_config.name, + config: db_config, + database_tasks?: db_config.database_tasks?, + identifier: identifier + } + end.compact + + unique_connections = all_connections.group_by { |connection| connection[:identifier] } + primary_connection = all_connections.find { |connection| ActiveRecord::Base.configurations.primary?(connection[:name]) } + named_connections = all_connections.index_by { |connection| connection[:name] } + + warnings = [] + + # The `main:` should always have `database_tasks: true` + unless primary_connection[:database_tasks?] + warnings << "- The '#{primary_connection[:name]}' is required to use 'database_tasks: true'" + end + + # Each unique database should have exactly one configuration with `database_tasks: true` + unique_connections.each do |identifier, connections| + next unless identifier + + connections_with_tasks = connections.select { |connection| connection[:database_tasks?] } + if connections_with_tasks.many? + names = connections_with_tasks.pluck(:name) + + warnings << "- Many configurations (#{names.join(', ')}) " \ + "share the same database (#{identifier}). " \ + "This will result in failures provisioning or migrating this database. " \ + "Ensure that additional databases are configured " \ + "with 'database_tasks: false' or are pointing to a dedicated database host." + end + end + + # Each configuration with `database_tasks: false` should share the database with `main:` + all_connections.each do |connection| + share_with = Gitlab::Database.db_config_share_with(connection[:config]) + next unless share_with + + shared_connection = named_connections[share_with] + unless shared_connection + warnings << "- The '#{connection[:name]}' is expecting to share configuration with '#{share_with}', " \ + "but no such is to be found." + next + end + + # Skip if databases are yet to be provisioned + next unless connection[:identifier] && shared_connection[:identifier] + + unless connection[:identifier] == shared_connection[:identifier] + warnings << "- The '#{connection[:name]}' since it is using 'database_tasks: false' " \ + "should share database with '#{share_with}:'." + end + end + + if warnings.any? + warnings.unshift("Database config validation failure:") + + # Warn (for now) by default in production environment + if Gitlab::Utils.to_boolean(ENV['GITLAB_VALIDATE_DATABASE_CONFIG'], default: true) + warnings << "Use `export GITLAB_VALIDATE_DATABASE_CONFIG=0` to ignore this validation." + + raise warnings.join("\n") + else + warnings << "Use `export GITLAB_VALIDATE_DATABASE_CONFIG=1` to enforce this validation." + + warn warnings.join("\n") + end + end + + ensure + ActiveRecord::Base.establish_connection(original_db_config) # rubocop: disable Database/EstablishConnection + end + + Rake::Task['db:migrate'].enhance(['gitlab:db:validate_config']) + Rake::Task['db:schema:load'].enhance(['gitlab:db:validate_config']) + Rake::Task['db:schema:dump'].enhance(['gitlab:db:validate_config']) + + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| + Rake::Task["db:migrate:#{name}"].enhance(['gitlab:db:validate_config']) + Rake::Task["db:schema:load:#{name}"].enhance(['gitlab:db:validate_config']) + Rake::Task["db:schema:dump:#{name}"].enhance(['gitlab:db:validate_config']) + end + end +end diff --git a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake index 1cc18d14d78..203d500b616 100644 --- a/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake +++ b/lib/tasks/gitlab/refresh_project_statistics_build_artifacts_size.rake @@ -1,23 +1,43 @@ # frozen_string_literal: true +require 'httparty' +require 'csv' + namespace :gitlab do - desc "GitLab | Refresh build artifacts size project statistics for given project IDs" + desc "GitLab | Refresh build artifacts size project statistics for given list of Project IDs from remote CSV" BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500 - task :refresh_project_statistics_build_artifacts_size, [:project_ids] => :environment do |_t, args| - project_ids = [] - project_ids = $stdin.read.split unless $stdin.tty? - project_ids = args.project_ids.to_s.split unless project_ids.any? + task :refresh_project_statistics_build_artifacts_size, [:csv_url] => :environment do |_t, args| + csv_url = args.csv_url + + # rubocop: disable Gitlab/HTTParty + body = HTTParty.get(csv_url) + # rubocop: enable Gitlab/HTTParty + + table = CSV.parse(body.to_s, headers: true) + project_ids = table['PROJECT_ID'] + + puts "Loaded #{project_ids.size} project ids to import" + + imported = 0 + missing = 0 if project_ids.any? - project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE) do |ids| + project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE, false) do |ids| projects = Project.where(id: ids) Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects) + + # Take a short break to allow replication to catch up + Kernel.sleep(1) + + imported += projects.size + missing += ids.size - projects.size + puts "#{imported}/#{project_ids.size} (missing projects: #{missing})" end - puts 'Done.'.green + puts 'Done.' else - puts 'Please provide a string of space-separated project IDs as the argument or through the STDIN'.red + puts 'Project IDs must be listed in the CSV under the header PROJECT_ID'.red end end end diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index a5289476378..006dfad3a95 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -30,7 +30,7 @@ namespace :gitlab do # In production, we might want to prevent ourselves from shooting # ourselves in the foot, so let's only do this in a test or # development environment. - terminate_all_connections unless Rails.env.production? + Rake::Task["dev:terminate_all_connections"].invoke unless Rails.env.production? Rake::Task["db:reset"].invoke Rake::Task["db:seed_fu"].invoke @@ -38,24 +38,4 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end - - # If there are any clients connected to the DB, PostgreSQL won't let - # you drop the database. It's possible that Sidekiq, Puma, or - # some other client will be hanging onto a connection, preventing - # the DROP DATABASE from working. To workaround this problem, this - # method terminates all the connections so that a subsequent DROP - # will work. - def self.terminate_all_connections - cmd = <<~SQL - SELECT pg_terminate_backend(pg_stat_activity.pid) - FROM pg_stat_activity - WHERE datname = current_database() - AND pid <> pg_backend_pid(); - SQL - - Gitlab::Database::EachDatabase.each_database_connection do |connection| - connection.execute(cmd) - rescue ActiveRecord::NoDatabaseError - end - end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 358bc6c31eb..0aed017c84a 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -12,17 +12,16 @@ namespace :tw do CodeOwnerRule.new("Adoption", '@kpaizee'), CodeOwnerRule.new('Activation', '@kpaizee'), CodeOwnerRule.new('Adoption', '@kpaizee'), - CodeOwnerRule.new('APM', '@ngaskill'), - CodeOwnerRule.new('Authentication & Authorization', '@eread'), + CodeOwnerRule.new('Authentication and Authorization', '@eread'), CodeOwnerRule.new('Certify', '@msedlakjakubowski'), CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), CodeOwnerRule.new('Configure', '@marcia'), - CodeOwnerRule.new('Container Security', '@ngaskill'), + CodeOwnerRule.new('Container Security', '@claytoncornell'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), - CodeOwnerRule.new('Database', '@aqualls'), + CodeOwnerRule.new('Database', '@marcia'), CodeOwnerRule.new('Development', '@marcia'), CodeOwnerRule.new('Distribution', '@axil'), CodeOwnerRule.new('Distribution (Charts)', '@axil'), @@ -37,26 +36,28 @@ namespace :tw do CodeOwnerRule.new('Geo', '@axil'), CodeOwnerRule.new('Gitaly', '@eread'), CodeOwnerRule.new('Global Search', '@marcia'), - CodeOwnerRule.new('Health', '@ngaskill'), - CodeOwnerRule.new('Import', '@ngaskill'), + CodeOwnerRule.new('Import', '@eread'), CodeOwnerRule.new('Infrastructure', '@marcia'), CodeOwnerRule.new('Integrations', '@kpaizee'), CodeOwnerRule.new('Knowledge', '@aqualls'), CodeOwnerRule.new('License', '@sselhorn'), CodeOwnerRule.new('Memory', '@marcia'), - CodeOwnerRule.new('Monitor', '@ngaskill'), + CodeOwnerRule.new('Monitor', '@msedlakjakubowski'), + CodeOwnerRule.new('Observability', 'msedlakjakubowski'), CodeOwnerRule.new('Optimize', '@fneill'), - CodeOwnerRule.new('Package', '@ngaskill'), + CodeOwnerRule.new('Package', '@claytoncornell'), CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'), CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), + CodeOwnerRule.new('Pipeline Insights', '@marcel.amirault'), CodeOwnerRule.new('Portfolio Management', '@msedlakjakubowski'), - CodeOwnerRule.new('Product Intelligence', '@fneill'), + CodeOwnerRule.new('Product Intelligence', '@claytoncornell'), CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), CodeOwnerRule.new('Project Management', '@msedlakjakubowski'), CodeOwnerRule.new('Provision', '@sselhorn'), CodeOwnerRule.new('Purchase', '@sselhorn'), CodeOwnerRule.new('Redirect', 'Redirect'), CodeOwnerRule.new('Release', '@rdickenson'), + CodeOwnerRule.new('Respond', '@msedlakjakubowski'), CodeOwnerRule.new('Runner', '@sselhorn'), CodeOwnerRule.new('Sharding', '@marcia'), CodeOwnerRule.new('Source Code', '@aqualls'), @@ -64,9 +65,9 @@ namespace :tw do CodeOwnerRule.new('Static Site Editor', '@aqualls'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Testing', '@eread'), - CodeOwnerRule.new('Threat Insights', '@fneill'), + CodeOwnerRule.new('Threat Insights', '@claytoncornell'), CodeOwnerRule.new('Utilization', '@sselhorn'), - CodeOwnerRule.new('Vulnerability Research', '@fneill'), + CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'), CodeOwnerRule.new('Workspace', '@fneill') ].freeze diff --git a/lib/tasks/gitlab_danger.rake b/lib/tasks/gitlab_danger.rake deleted file mode 100644 index ff9464a588a..00000000000 --- a/lib/tasks/gitlab_danger.rake +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -desc 'Run local Danger rules' -task :danger_local do - require_relative '../../tooling/danger/project_helper' - require 'gitlab/popen' - - puts("#{Tooling::Danger::ProjectHelper.local_warning_message}\n") - - # _status will _always_ be 0, regardless of failure or success :( - output, _status = Gitlab::Popen.popen(%w{danger dry_run}) - - if output.empty? - puts(Tooling::Danger::ProjectHelper.success_message) - else - puts(output) - exit(1) - end -end |