diff options
Diffstat (limited to 'lib')
465 files changed, 10183 insertions, 2712 deletions
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index d6c212a9886..7163225777a 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -6,6 +6,7 @@ module API include PaginationParams feature_category :kubernetes_management + urgency :low before do authenticated_as_admin! @@ -136,7 +137,7 @@ module API end def ensure_feature_enabled! - not_found! unless Feature.enabled?(:certificate_based_clusters, clusterable_instance, default_enabled: :yaml, type: :ops) + not_found! unless clusterable_instance.certificate_based_clusters_enabled? end end end diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index 99be30809d2..7ce70d85d46 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -35,6 +35,14 @@ module API params do requires :plan_name, type: String, values: Plan.all_plans, desc: 'Name of the plan' + optional :ci_pipeline_size, type: Integer, desc: 'Maximum number of jobs in a single pipeline' + optional :ci_active_jobs, type: Integer, desc: 'Total number of jobs in currently active pipelines' + optional :ci_active_pipelines, type: Integer, desc: 'Maximum number of active pipelines per project' + optional :ci_project_subscriptions, type: Integer, desc: 'Maximum number of pipeline subscriptions to and from a project' + optional :ci_pipeline_schedules, type: Integer, desc: 'Maximum number of pipeline schedules' + optional :ci_needs_size_limit, type: Integer, desc: 'Maximum number of DAG dependencies that a job can have' + optional :ci_registered_group_runners, type: Integer, desc: 'Maximum number of runners registered per group' + optional :ci_registered_project_runners, type: Integer, desc: 'Maximum number of runners registered per project' optional :conan_max_file_size, type: Integer, desc: 'Maximum Conan package file size in bytes' optional :generic_packages_max_file_size, type: Integer, desc: 'Maximum generic package file size in bytes' optional :helm_max_file_size, type: Integer, desc: 'Maximum Helm chart file size in bytes' @@ -43,6 +51,7 @@ module API optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes' optional :pypi_max_file_size, type: Integer, desc: 'Maximum PyPI package file size in bytes' optional :terraform_module_max_file_size, type: Integer, desc: 'Maximum Terraform Module package file size in bytes' + optional :storage_size_limit, type: Integer, desc: 'Maximum storage size for the root namespace in megabytes' end put "application/plan_limits" do params = declared_params(include_missing: false) diff --git a/lib/api/alert_management_alerts.rb b/lib/api/alert_management_alerts.rb index 88230c86247..bbb7e7280c9 100644 --- a/lib/api/alert_management_alerts.rb +++ b/lib/api/alert_management_alerts.rb @@ -3,6 +3,7 @@ module API class AlertManagementAlerts < ::API::Base feature_category :incident_management + urgency :low params do requires :id, type: String, desc: 'The ID of a project' @@ -83,8 +84,6 @@ module API authorize!(:update_alert_management_metric_image, alert) - render_api_error!('Feature not available', 403) unless alert.metric_images_available? - metric_image = alert.metric_images.find_by_id(params[:metric_image_id]) render_api_error!('Metric image not found', 404) unless metric_image @@ -107,8 +106,6 @@ module API authorize!(:destroy_alert_management_metric_image, alert) - render_api_error!('Feature not available', 403) unless alert.metric_images_available? - metric_image = alert.metric_images.find_by_id(params[:metric_image_id]) render_api_error!('Metric image not found', 404) unless metric_image diff --git a/lib/api/api.rb b/lib/api/api.rb index 4dca47efdf2..0d74bc841b1 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -175,8 +175,8 @@ module API mount ::API::BulkImports mount ::API::Ci::JobArtifacts mount ::API::Ci::Jobs - mount ::API::Ci::Pipelines mount ::API::Ci::PipelineSchedules + mount ::API::Ci::Pipelines mount ::API::Ci::ResourceGroups mount ::API::Ci::Runner mount ::API::Ci::Runners @@ -184,14 +184,21 @@ module API mount ::API::Ci::Triggers mount ::API::Ci::Variables mount ::API::Clusters::Agents - mount ::API::Commits + mount ::API::Clusters::AgentTokens mount ::API::CommitStatuses + mount ::API::Commits + mount ::API::ComposerPackages + mount ::API::ConanInstancePackages + mount ::API::ConanProjectPackages mount ::API::ContainerRegistryEvent mount ::API::ContainerRepositories + mount ::API::DebianGroupPackages + mount ::API::DebianProjectPackages mount ::API::DependencyProxy mount ::API::DeployKeys mount ::API::DeployTokens mount ::API::Deployments + mount ::API::Discussions mount ::API::Environments mount ::API::ErrorTracking::ClientKeys mount ::API::ErrorTracking::Collector @@ -202,87 +209,79 @@ module API mount ::API::Features mount ::API::Files mount ::API::FreezePeriods + mount ::API::GenericPackages mount ::API::Geo + mount ::API::GoProxy mount ::API::GroupAvatar mount ::API::GroupBoards mount ::API::GroupClusters + mount ::API::GroupContainerRepositories + mount ::API::GroupDebianDistributions mount ::API::GroupExport mount ::API::GroupImport mount ::API::GroupLabels mount ::API::GroupMilestones - mount ::API::Groups - mount ::API::GroupContainerRepositories - mount ::API::GroupDebianDistributions + mount ::API::GroupPackages mount ::API::GroupVariables + mount ::API::Groups + mount ::API::HelmPackages mount ::API::ImportBitbucketServer mount ::API::ImportGithub - mount ::API::IssueLinks + mount ::API::Integrations + mount ::API::Integrations::JiraConnect::Subscriptions mount ::API::Invitations + mount ::API::IssueLinks mount ::API::Issues mount ::API::Keys mount ::API::Labels mount ::API::Lint mount ::API::Markdown + mount ::API::MavenPackages mount ::API::Members + mount ::API::MergeRequestApprovals mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::MergeRequestApprovals mount ::API::Metrics::Dashboard::Annotations mount ::API::Metrics::UserStarredDashboards mount ::API::Namespaces mount ::API::Notes - mount ::API::Discussions - mount ::API::ResourceLabelEvents - mount ::API::ResourceMilestoneEvents - mount ::API::ResourceStateEvents mount ::API::NotificationSettings - mount ::API::ProjectPackages - mount ::API::GroupPackages - mount ::API::PackageFiles - mount ::API::NugetProjectPackages - mount ::API::NugetGroupPackages - mount ::API::PypiPackages - mount ::API::ComposerPackages - mount ::API::ConanProjectPackages - mount ::API::ConanInstancePackages - mount ::API::DebianGroupPackages - mount ::API::DebianProjectPackages - mount ::API::MavenPackages - mount ::API::NpmProjectPackages mount ::API::NpmInstancePackages - mount ::API::GenericPackages - mount ::API::GoProxy - mount ::API::HelmPackages + mount ::API::NpmProjectPackages + mount ::API::NugetGroupPackages + mount ::API::NugetProjectPackages + mount ::API::PackageFiles mount ::API::Pages mount ::API::PagesDomains + mount ::API::PersonalAccessTokens mount ::API::ProjectClusters mount ::API::ProjectContainerRepositories mount ::API::ProjectDebianDistributions mount ::API::ProjectEvents mount ::API::ProjectExport - mount ::API::ProjectImport mount ::API::ProjectHooks + mount ::API::ProjectImport mount ::API::ProjectMilestones + mount ::API::ProjectPackages mount ::API::ProjectRepositoryStorageMoves - mount ::API::Projects mount ::API::ProjectSnapshots mount ::API::ProjectSnippets mount ::API::ProjectStatistics mount ::API::ProjectTemplates - mount ::API::Terraform::State - mount ::API::Terraform::StateVersion - mount ::API::Terraform::Modules::V1::Packages - mount ::API::PersonalAccessTokens + mount ::API::Projects mount ::API::ProtectedBranches mount ::API::ProtectedTags - mount ::API::Releases + mount ::API::PypiPackages mount ::API::Release::Links + mount ::API::Releases mount ::API::RemoteMirrors mount ::API::Repositories mount ::API::ResourceAccessTokens + mount ::API::ResourceLabelEvents + mount ::API::ResourceMilestoneEvents + mount ::API::ResourceStateEvents mount ::API::RubygemPackages mount ::API::Search - mount ::API::Integrations mount ::API::Settings mount ::API::SidekiqMetrics mount ::API::SnippetRepositoryStorageMoves @@ -294,12 +293,15 @@ module API mount ::API::SystemHooks mount ::API::Tags mount ::API::Templates + mount ::API::Terraform::Modules::V1::Packages + mount ::API::Terraform::State + mount ::API::Terraform::StateVersion mount ::API::Todos mount ::API::Topics mount ::API::Unleash mount ::API::UsageData - mount ::API::UsageDataQueries mount ::API::UsageDataNonSqlMetrics + mount ::API::UsageDataQueries mount ::API::UserCounts mount ::API::Users mount ::API::Version diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index a42d89ddf83..bd9fb37e18b 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -3,6 +3,7 @@ module API class Avatar < ::API::Base feature_category :users + urgency :high resource :avatar do desc 'Return avatar url for a user' do diff --git a/lib/api/badges.rb b/lib/api/badges.rb index d7c850c2f40..68095fb2975 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -32,7 +32,7 @@ module API params do use :pagination end - get ":id/badges" do + get ":id/badges", urgency: :default do source = find_source(source_type, params[:id]) badges = source.badges @@ -72,7 +72,10 @@ module API params do requires :badge_id, type: Integer, desc: 'The badge ID' end - get ":id/badges/:badge_id" do + # TODO: Set PUT /projects/:id/badges/:badge_id to low urgency and GET to default urgency + # after different urgencies are supported for different HTTP verbs. + # See https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1670 + get ":id/badges/:badge_id", urgency: :low do source = find_source(source_type, params[:id]) badge = find_badge(source) @@ -88,7 +91,7 @@ module API requires :image_url, type: String, desc: 'URL of the badge image' optional :name, type: String, desc: 'Name for the badge' end - post ":id/badges" do + post ":id/badges", urgency: :default do source = find_source_if_admin(source_type) badge = ::Badges::CreateService.new(declared_params(include_missing: false)).execute(source) diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 56633c07774..6e3005ce676 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -8,6 +8,7 @@ module API prepend_mod_with('API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule feature_category :team_planning + urgency :low before { authenticate! } diff --git a/lib/api/branches.rb b/lib/api/branches.rb index a2c9020ac84..b8444351029 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -52,7 +52,7 @@ module API merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - if Feature.enabled?(:api_caching_branches, user_project, type: :development, default_enabled: :yaml) + if Feature.enabled?(:api_caching_branches, user_project, type: :development) present_cached( branches, with: Entities::Branch, diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index 53967e0af5d..766e05eca23 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -5,6 +5,7 @@ module API include PaginationParams feature_category :importers + urgency :low helpers do def bulk_imports diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 173cfc9a59a..72e36d95dc5 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -53,7 +53,7 @@ module API # https://gitlab.com/gitlab-org/gitlab/-/issues/327703 forbidden! unless job - forbidden! unless job_token_valid?(job) + forbidden! unless job.valid_token?(job_token) forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete? forbidden!('Job has been erased!') if job.erased? @@ -77,6 +77,12 @@ module API job end + def authenticate_job_via_dependent_job! + forbidden! unless current_authenticated_job + forbidden! unless current_job + forbidden! unless can?(current_authenticated_job.user, :read_build, current_job) + end + def current_job id = params[:id] @@ -91,9 +97,28 @@ module API end end - def job_token_valid?(job) - token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s - token && job.valid_token?(token) + # TODO: Replace this with `#current_authenticated_job from API::Helpers` + # after the feature flag `ci_authenticate_running_job_token_for_artifacts` + # is removed. + # + # For the time being, this needs to be overridden because the API + # GET api/v4/jobs/:id/artifacts + # needs to allow requests using token whose job is not running. + # + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83713#note_942368526 + def current_authenticated_job + strong_memoize(:current_authenticated_job) do + ::Ci::AuthJobFinder.new(token: job_token).execute + end + end + + # The token used by runner to authenticate a request. + # In most cases, the runner uses the token belonging to the requested job. + # However, when requesting for job artifacts, the runner would use + # the token that belongs to downstream jobs that depend on the job that owns + # the artifacts. + def job_token + @job_token ||= (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s end def job_forbidden!(job, reason) @@ -111,11 +136,19 @@ module API # noop: overridden in EE end + def log_artifact_size(artifact) + Gitlab::ApplicationContext.push(artifact: artifact) + end + private def get_runner_config_from_request { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) } end + + def request_using_running_job_token? + current_job.present? && current_authenticated_job.present? && current_job != current_authenticated_job + end end end end diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 86897eb61ae..04999b5fb44 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -190,7 +190,7 @@ module API detail 'Retrieves a list of agents for the given job token' end route_setting :authentication, job_token_allowed: true - get '/allowed_agents', feature_category: :kubernetes_management do + get '/allowed_agents', urgency: :low, feature_category: :kubernetes_management do validate_current_authenticated_job status 200 diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb index 6030fe86f00..4b522f37524 100644 --- a/lib/api/ci/pipeline_schedules.rb +++ b/lib/api/ci/pipeline_schedules.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :continuous_integration + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 8d2c58dabdf..4253a9eb4d7 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -51,7 +51,7 @@ module API desc: 'Sort pipelines' optional :source, type: String, values: ::Ci::Pipeline.sources.keys end - get ':id/pipelines', feature_category: :continuous_integration do + get ':id/pipelines', urgency: :low, feature_category: :continuous_integration do authorize! :read_pipeline, user_project authorize! :read_build, user_project @@ -67,7 +67,7 @@ module API requires :ref, type: String, desc: 'Reference' optional :variables, Array, desc: 'Array of variables available in the pipeline' end - post ':id/pipeline', feature_category: :continuous_integration do + post ':id/pipeline', urgency: :low, feature_category: :continuous_integration do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711') authorize! :create_pipeline, user_project @@ -94,7 +94,7 @@ module API params do optional :ref, type: String, desc: 'branch ref of pipeline' end - get ':id/pipelines/latest', feature_category: :continuous_integration do + get ':id/pipelines/latest', urgency: :low, feature_category: :continuous_integration do authorize! :read_pipeline, latest_pipeline present latest_pipeline, with: Entities::Ci::Pipeline @@ -107,7 +107,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - get ':id/pipelines/:pipeline_id', feature_category: :continuous_integration do + get ':id/pipelines/:pipeline_id', urgency: :low, feature_category: :continuous_integration do authorize! :read_pipeline, pipeline present pipeline, with: Entities::Ci::Pipeline @@ -205,7 +205,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - delete ':id/pipelines/:pipeline_id', feature_category: :continuous_integration do + delete ':id/pipelines/:pipeline_id', urgency: :low, feature_category: :continuous_integration do authorize! :destroy_pipeline, pipeline destroy_conditionally!(pipeline) do @@ -220,7 +220,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - post ':id/pipelines/:pipeline_id/retry', feature_category: :continuous_integration do + post ':id/pipelines/:pipeline_id/retry', urgency: :low, feature_category: :continuous_integration do authorize! :update_pipeline, pipeline response = pipeline.retry_failed(current_user) @@ -239,7 +239,7 @@ module API params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' end - post ':id/pipelines/:pipeline_id/cancel', feature_category: :continuous_integration do + post ':id/pipelines/:pipeline_id/cancel', urgency: :low, feature_category: :continuous_integration do authorize! :update_pipeline, pipeline pipeline.cancel_running diff --git a/lib/api/ci/resource_groups.rb b/lib/api/ci/resource_groups.rb index 616bec499d4..e3fd887475a 100644 --- a/lib/api/ci/resource_groups.rb +++ b/lib/api/ci/resource_groups.rb @@ -3,14 +3,29 @@ module API module Ci class ResourceGroups < ::API::Base + include PaginationParams + before { authenticate! } feature_category :continuous_delivery + urgency :low params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get all resource groups for this project' do + success Entities::Ci::ResourceGroup + end + params do + use :pagination + end + get ':id/resource_groups' do + authorize! :read_resource_group, user_project + + present paginate(user_project.resource_groups), with: Entities::Ci::ResourceGroup + end + desc 'Get a single resource group' do success Entities::Ci::ResourceGroup end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 0e3b295396b..4381309fb9e 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -29,7 +29,7 @@ module API mutually_exclusive :maintainer_note, :maintainer_note mutually_exclusive :active, :paused end - post '/', feature_category: :runner do + post '/', urgency: :low, feature_category: :runner do attributes = attributes_for_keys(%i[description maintainer_note maintenance_note active paused locked run_untagged tag_list access_level maximum_timeout]) .merge(get_runner_details_from_request) @@ -54,7 +54,7 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) end - delete '/', feature_category: :runner do + delete '/', urgency: :low, feature_category: :runner do authenticate_runner! destroy_conditionally!(current_runner) { ::Ci::Runners::UnregisterRunnerService.new(current_runner, params[:token]).execute } @@ -66,7 +66,7 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) end - post '/verify', feature_category: :runner do + post '/verify', urgency: :low, feature_category: :runner do authenticate_runner! status 200 body "200" @@ -78,7 +78,7 @@ module API params do requires :token, type: String, desc: 'The current authentication token of the runner' end - post '/reset_authentication_token', feature_category: :runner do + post '/reset_authentication_token', urgency: :low, feature_category: :runner do authenticate_runner! current_runner.reset_token! @@ -212,7 +212,7 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) end - patch '/:id/trace', urgency: :default, feature_category: :continuous_integration do + patch '/:id/trace', urgency: :low, feature_category: :continuous_integration do job = authenticate_job!(heartbeat_runner: true) error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') @@ -305,6 +305,7 @@ module API result = ::Ci::JobArtifacts::CreateService.new(job).execute(artifacts, params, metadata_file: metadata) if result[:status] == :success + log_artifact_size(result[:artifact]) status :created body "201" else @@ -323,9 +324,13 @@ module API optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end get '/:id/artifacts', feature_category: :build_artifacts do - job = authenticate_job!(require_running: false) + if request_using_running_job_token? + authenticate_job_via_dependent_job! + else + authenticate_job!(require_running: false) + end - present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download]) + present_carrierwave_file!(current_job.artifacts_file, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 3c9e887e751..7863cfd1e79 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :runner + urgency :low resource :runners do desc 'Get runners available for user' do diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index ee39bdfd90c..6c7f502b428 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -62,13 +62,11 @@ module API params do requires :name, type: String, desc: 'The name of the file' requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded' - optional :permissions, type: String, desc: 'The file permissions', default: 'read_only', values: %w[read_only read_write execute] end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true post ':id/secure_files' do secure_file = user_project.secure_files.new( - name: params[:name], - permissions: params[:permissions] || :read_only + name: params[:name] ) secure_file.file = params[:file] @@ -96,11 +94,11 @@ module API helpers do def feature_flag_enabled? - service_unavailable! unless Feature.enabled?(:ci_secure_files, user_project, default_enabled: :yaml) + service_unavailable! unless Feature.enabled?(:ci_secure_files, user_project) end def read_only_feature_flag_enabled? - service_unavailable! if Feature.enabled?(:ci_secure_files_read_only, user_project, type: :ops, default_enabled: :yaml) + service_unavailable! if Feature.enabled?(:ci_secure_files_read_only, user_project, type: :ops) end end end diff --git a/lib/api/ci/triggers.rb b/lib/api/ci/triggers.rb index ae89b475ef8..c49f1c9e9e1 100644 --- a/lib/api/ci/triggers.rb +++ b/lib/api/ci/triggers.rb @@ -8,6 +8,7 @@ module API HTTP_GITLAB_EVENT_HEADER = "HTTP_#{::Gitlab::WebHooks::GITLAB_EVENT_HEADER}".underscore.upcase feature_category :continuous_integration + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb index ec9951aba0d..f9707960b9d 100644 --- a/lib/api/ci/variables.rb +++ b/lib/api/ci/variables.rb @@ -35,7 +35,7 @@ module API requires :key, type: String, desc: 'The key of the variable' end # rubocop: disable CodeReuse/ActiveRecord - get ':id/variables/:key' do + get ':id/variables/:key', urgency: :low do variable = find_variable(user_project, params) not_found!('Variable') unless variable diff --git a/lib/api/clusters/agent_tokens.rb b/lib/api/clusters/agent_tokens.rb new file mode 100644 index 00000000000..1e52790f26b --- /dev/null +++ b/lib/api/clusters/agent_tokens.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module API + module Clusters + class AgentTokens < ::API::Base + include PaginationParams + + before { authenticate! } + + feature_category :kubernetes_management + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :agent_id, type: Integer, desc: 'The ID of an agent' + end + resource ':id/cluster_agents/:agent_id' do + resource :tokens do + desc 'List agent tokens' do + detail 'This feature was introduced in GitLab 15.0.' + success Entities::Clusters::AgentTokenBasic + end + params do + use :pagination + end + get do + authorize! :read_cluster, user_project + + agent = user_project.cluster_agents.find(params[:agent_id]) + + present paginate(agent.agent_tokens), with: Entities::Clusters::AgentTokenBasic + end + + desc 'Get a single agent token' do + detail 'This feature was introduced in GitLab 15.0.' + success Entities::Clusters::AgentToken + end + params do + requires :token_id, type: Integer, desc: 'The ID of the agent token' + end + get ':token_id' do + authorize! :read_cluster, user_project + + agent = user_project.cluster_agents.find(params[:agent_id]) + token = agent.agent_tokens.find(params[:token_id]) + + present token, with: Entities::Clusters::AgentToken + end + + desc 'Create an agent token' do + detail 'This feature was introduced in GitLab 15.0.' + success Entities::Clusters::AgentTokenWithToken + end + params do + requires :name, type: String, desc: 'The name for the token' + optional :description, type: String, desc: 'The description for the token' + end + post do + authorize! :create_cluster, user_project + + token_params = declared_params(include_missing: false) + + agent = user_project.cluster_agents.find(params[:agent_id]) + + result = ::Clusters::AgentTokens::CreateService.new( + container: agent.project, current_user: current_user, params: token_params.merge(agent_id: agent.id) + ).execute + + bad_request!(result[:message]) if result[:status] == :error + + present result[:token], with: Entities::Clusters::AgentTokenWithToken + end + + desc 'Revoke an agent token' do + detail 'This feature was introduced in GitLab 15.0.' + end + params do + requires :token_id, type: Integer, desc: 'The ID of the agent token' + end + delete ':token_id' do + authorize! :admin_cluster, user_project + + agent = user_project.cluster_agents.find(params[:agent_id]) + token = agent.agent_tokens.find(params[:token_id]) + + # Skipping explicit error handling and relying on exceptions + token.revoked! + + status :no_content + end + end + end + end + end + end +end diff --git a/lib/api/clusters/agents.rb b/lib/api/clusters/agents.rb index 6c1bf21b952..0fa556d2da9 100644 --- a/lib/api/clusters/agents.rb +++ b/lib/api/clusters/agents.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :kubernetes_management + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index c89abf72e2d..5a6d06dcdd9 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -5,6 +5,7 @@ require 'mime/types' module API class CommitStatuses < ::API::Base feature_category :continuous_integration + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index c311b34a697..de59cb4a7c3 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -71,7 +71,7 @@ module API desc 'Composer packages endpoint at group level' route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true - get ':id/-/packages/composer/packages' do + get ':id/-/packages/composer/packages', urgency: :low do presenter.root end @@ -80,7 +80,7 @@ module API requires :sha, type: String, desc: 'Shasum of current json' end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true - get ':id/-/packages/composer/p/:sha' do + get ':id/-/packages/composer/p/:sha', urgency: :low do presenter.provider end @@ -89,7 +89,7 @@ module API requires :package_name, type: String, file_path: true, desc: 'The Composer package name' end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true - get ':id/-/packages/composer/p2/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do + get ':id/-/packages/composer/p2/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true, urgency: :low do not_found! if packages.empty? presenter.package_versions @@ -100,7 +100,7 @@ module API requires :package_name, type: String, file_path: true, desc: 'The Composer package name' end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true - get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do + get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true, urgency: :low do not_found! if packages.empty? not_found! if params[:sha].blank? @@ -122,7 +122,7 @@ module API optional :tag, type: String, desc: 'The name of the tag' exactly_one_of :tag, :branch end - post do + post urgency: :low do authorize_create_package!(authorized_user_project) if params[:branch].present? @@ -147,7 +147,7 @@ module API requires :package_name, type: String, file_path: true, desc: 'The Composer package name' end route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true - get 'archives/*package_name' do + get 'archives/*package_name', urgency: :default do authorize_read_package!(authorized_user_project) metadata = authorized_user_project diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index e241633fa8b..d1cc35b16d8 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -56,7 +56,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'ping' do + get 'ping', urgency: :default do header 'X-Conan-Server-Capabilities', [].join(',') end @@ -70,7 +70,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'conans/search' do + get 'conans/search', urgency: :low do service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute service.payload end @@ -89,7 +89,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'authenticate' do + get 'authenticate', urgency: :low do unauthorized! unless token token.to_jwt @@ -101,7 +101,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'check_credentials' do + get 'check_credentials', urgency: :default do authenticate! :ok end @@ -133,7 +133,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'packages/:conan_package_reference' do + get 'packages/:conan_package_reference', urgency: :low do authorize!(:read_package, project) presenter = ::Packages::Conan::PackagePresenter.new( @@ -152,7 +152,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get do + get urgency: :low do authorize!(:read_package, project) presenter = ::Packages::Conan::PackagePresenter.new(package, current_user, project) @@ -174,7 +174,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'packages/:conan_package_reference/digest' do + get 'packages/:conan_package_reference/digest', urgency: :low do present_package_download_urls end @@ -184,7 +184,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'digest' do + get 'digest', urgency: :low do present_recipe_download_urls end @@ -204,7 +204,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'packages/:conan_package_reference/download_urls' do + get 'packages/:conan_package_reference/download_urls', urgency: :low do present_package_download_urls end @@ -214,7 +214,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get 'download_urls' do + get 'download_urls', urgency: :low do present_recipe_download_urls end @@ -235,7 +235,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - post 'packages/:conan_package_reference/upload_urls' do + post 'packages/:conan_package_reference/upload_urls', urgency: :low do authorize!(:read_package, project) status 200 @@ -248,7 +248,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - post 'upload_urls' do + post 'upload_urls', urgency: :low do authorize!(:read_package, project) status 200 @@ -261,7 +261,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - delete do + delete urgency: :low do authorize!(:destroy_package, project) track_package_event('delete_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace) @@ -297,7 +297,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get do + get urgency: :low do download_package_file(:recipe_file) end @@ -311,7 +311,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - put do + put urgency: :low do upload_package_file(:recipe_file) end @@ -321,7 +321,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - put 'authorize' do + put 'authorize', urgency: :low do authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) end end @@ -338,7 +338,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - get do + get urgency: :low do download_package_file(:package_file) end @@ -348,7 +348,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - put 'authorize' do + put 'authorize', urgency: :low do authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) end @@ -362,7 +362,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - put do + put urgency: :low do upload_package_file(:package_file) end end diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb index ddc83d0f747..e01f3adbb06 100644 --- a/lib/api/concerns/packages/debian_distribution_endpoints.rb +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -10,6 +10,7 @@ module API include PaginationParams feature_category :package_registry + urgency :low helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index d083643f3d0..e8d27448f02 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -16,6 +16,7 @@ module API included do feature_category :package_registry + urgency :low helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index 208daeb3037..e0328e488c6 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -56,7 +56,7 @@ module API desc 'The NuGet Service Index' do detail 'This feature was introduced in GitLab 12.6' end - get 'index', format: :json do + get 'index', format: :json, urgency: :default do authorize_read_package!(project_or_group) track_package_event('cli_metadata', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) @@ -77,7 +77,7 @@ module API desc 'The NuGet Metadata Service - Package name level' do detail 'This feature was introduced in GitLab 12.8' end - get 'index', format: :json do + get 'index', format: :json, urgency: :low do present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages(params[:package_name])), with: ::API::Entities::Nuget::PackagesMetadata end @@ -88,7 +88,7 @@ module API params do requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX end - get '*package_version', format: :json do + get '*package_version', format: :json, urgency: :low do present ::Packages::Nuget::PackageMetadataPresenter.new(find_package(params[:package_name], params[:package_version])), with: ::API::Entities::Nuget::PackageMetadata end @@ -109,7 +109,7 @@ module API desc 'The NuGet Search Service' do detail 'This feature was introduced in GitLab 12.8' end - get format: :json do + get format: :json, urgency: :low do search_options = { include_prerelease_versions: params[:prerelease], per_page: params[:take], diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb index 9bad31f6661..66689f8d7c8 100644 --- a/lib/api/container_registry_event.rb +++ b/lib/api/container_registry_event.rb @@ -4,7 +4,8 @@ module API class ContainerRegistryEvent < ::API::Base DOCKER_DISTRIBUTION_EVENTS_V1_JSON = 'application/vnd.docker.distribution.events.v1+json' - feature_category :package_registry + feature_category :container_registry + urgency :low before { authenticate_registry_notification! } diff --git a/lib/api/container_repositories.rb b/lib/api/container_repositories.rb index 17d667fb6df..d4fa6153a92 100644 --- a/lib/api/container_repositories.rb +++ b/lib/api/container_repositories.rb @@ -10,6 +10,7 @@ module API before { authenticate! } feature_category :container_registry + urgency :low namespace 'registry' do params do diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb index 9d0b1bf4423..290a90934d7 100644 --- a/lib/api/dependency_proxy.rb +++ b/lib/api/dependency_proxy.rb @@ -5,6 +5,7 @@ module API helpers ::API::Helpers::PackagesHelpers feature_category :dependency_proxy + urgency :low after_validation do authorize! :admin_group, user_group diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 0ab9fe6644c..ca13db8701e 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -7,6 +7,7 @@ module API before { authenticate! } feature_category :continuous_delivery + urgency :low helpers do def add_deploy_keys_project(project, attrs = {}) diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 074c307e881..3e0411d2e91 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -5,6 +5,7 @@ module API include PaginationParams feature_category :continuous_delivery + urgency :low helpers do def scope_params diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 6939853c06b..8db5f54b45a 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :continuous_delivery + urgency :low params do requires :id, type: String, desc: 'The project ID' diff --git a/lib/api/entities/ci/job_request/dependency.rb b/lib/api/entities/ci/job_request/dependency.rb index 2672a4a245b..63c1552de8a 100644 --- a/lib/api/entities/ci/job_request/dependency.rb +++ b/lib/api/entities/ci/job_request/dependency.rb @@ -5,7 +5,12 @@ module API module Ci module JobRequest class Dependency < Grape::Entity - expose :id, :name, :token + expose :id, :name + + expose :token do |job, options| + options[:running_job]&.token + end + expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.available_artifacts? } end end diff --git a/lib/api/entities/ci/job_request/response.rb b/lib/api/entities/ci/job_request/response.rb index 86c945cb236..9de415ebacb 100644 --- a/lib/api/entities/ci/job_request/response.rb +++ b/lib/api/entities/ci/job_request/response.rb @@ -28,8 +28,10 @@ module API expose :artifacts, using: Entities::Ci::JobRequest::Artifacts expose :cache, using: Entities::Ci::JobRequest::Cache expose :credentials, using: Entities::Ci::JobRequest::Credentials - expose :all_dependencies, as: :dependencies, using: Entities::Ci::JobRequest::Dependency expose :features + expose :dependencies do |job, options| + Entities::Ci::JobRequest::Dependency.represent(job.all_dependencies, options.merge(running_job: job)) + end end end end diff --git a/lib/api/entities/ci/lint/result.rb b/lib/api/entities/ci/lint/result.rb index 39039868bba..b44a6e13463 100644 --- a/lib/api/entities/ci/lint/result.rb +++ b/lib/api/entities/ci/lint/result.rb @@ -9,6 +9,7 @@ module API expose :errors expose :warnings expose :merged_yaml + expose :includes expose :jobs, if: -> (result, options) { options[:include_jobs] } end end diff --git a/lib/api/entities/ci/secure_file.rb b/lib/api/entities/ci/secure_file.rb index b60a1a6ac90..639615e5779 100644 --- a/lib/api/entities/ci/secure_file.rb +++ b/lib/api/entities/ci/secure_file.rb @@ -6,7 +6,6 @@ module API class SecureFile < Grape::Entity expose :id expose :name - expose :permissions expose :checksum expose :checksum_algorithm expose :created_at diff --git a/lib/api/entities/clusters/agent_token.rb b/lib/api/entities/clusters/agent_token.rb new file mode 100644 index 00000000000..e8cc1009361 --- /dev/null +++ b/lib/api/entities/clusters/agent_token.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Clusters + class AgentToken < AgentTokenBasic + expose :last_used_at + end + end + end +end diff --git a/lib/api/entities/clusters/agent_token_basic.rb b/lib/api/entities/clusters/agent_token_basic.rb new file mode 100644 index 00000000000..793ec8188b7 --- /dev/null +++ b/lib/api/entities/clusters/agent_token_basic.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Entities + module Clusters + class AgentTokenBasic < Grape::Entity + expose :id + expose :name + expose :description + expose :agent_id + expose :status + expose :created_at + expose :created_by_user_id + end + end + end +end diff --git a/lib/api/entities/clusters/agent_token_with_token.rb b/lib/api/entities/clusters/agent_token_with_token.rb new file mode 100644 index 00000000000..8b84c80795f --- /dev/null +++ b/lib/api/entities/clusters/agent_token_with_token.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Clusters + class AgentTokenWithToken < AgentToken + expose :token + end + end + end +end diff --git a/lib/api/entities/environment.rb b/lib/api/entities/environment.rb index 91867f3403d..b1a720ac6bb 100644 --- a/lib/api/entities/environment.rb +++ b/lib/api/entities/environment.rb @@ -6,6 +6,7 @@ module API include RequestAwareEntity include Gitlab::Utils::StrongMemoize + expose :tier expose :project, using: Entities::BasicProjectDetails expose :last_deployment, using: Entities::Deployment, if: { last_deployment: true } expose :state diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index 9f4d1635998..94e50f19b35 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -3,6 +3,14 @@ module API module Entities class PlanLimit < Grape::Entity + expose :ci_pipeline_size + expose :ci_active_jobs + expose :ci_active_pipelines + expose :ci_project_subscriptions + expose :ci_pipeline_schedules + expose :ci_needs_size_limit + expose :ci_registered_group_runners + expose :ci_registered_project_runners expose :conan_max_file_size expose :generic_packages_max_file_size expose :helm_max_file_size @@ -11,6 +19,7 @@ module API expose :nuget_max_file_size expose :pypi_max_file_size expose :terraform_module_max_file_size + expose :storage_size_limit end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 60cc5167c41..9e216b0aed5 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -35,6 +35,10 @@ module API expose :members do |project| expose_url(api_v4_projects_members_path(id: project.id)) end + + expose :cluster_agents do |project| + expose_url(api_v4_projects_cluster_agents_path(id: project.id)) + end end expose :packages_enabled @@ -99,6 +103,7 @@ module API expose :ci_default_git_depth expose :ci_forward_deployment_enabled expose :ci_job_token_scope_enabled + expose :ci_separated_caches expose :public_builds, as: :public_jobs expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| project.build_allow_git_fetch ? 'fetch' : 'clone' @@ -121,6 +126,7 @@ module API expose :printing_merge_request_link_enabled expose :merge_method expose :squash_option + expose :enforce_auth_checks_on_uploads expose :suggestion_commit_message expose :merge_commit_template expose :squash_commit_template diff --git a/lib/api/entities/projects/topic.rb b/lib/api/entities/projects/topic.rb index d3d1cbec81c..976c307382a 100644 --- a/lib/api/entities/projects/topic.rb +++ b/lib/api/entities/projects/topic.rb @@ -6,6 +6,7 @@ module API class Topic < Grape::Entity expose :id expose :name + expose :title expose :description expose :total_projects_count expose :avatar_url do |topic, options| diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index ff711b4dec2..2366d137cc2 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -18,6 +18,9 @@ module API expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user| user.followees.size end + expose :is_followed, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) && opts[:current_user] } do |user, opts| + opts[:current_user].following?(user) + end expose :local_time do |user| local_time(user.timezone) end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 19b48c1e3cf..11f1cab0c72 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :continuous_delivery + urgency :low params do requires :id, type: String, desc: 'The project ID' @@ -29,6 +30,8 @@ module API environments = ::Environments::EnvironmentsFinder.new(user_project, current_user, params).execute present paginate(environments), with: Entities::Environment, current_user: current_user + rescue ::Environments::EnvironmentsFinder::InvalidStatesError => exception + bad_request!(exception.message) end desc 'Creates a new environment' do @@ -39,6 +42,7 @@ module API requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' optional :slug, absence: { message: "is automatically generated and cannot be changed" } + optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the environment to be created' end post ':id/environments' do authorize! :create_environment, user_project @@ -62,13 +66,14 @@ module API optional :name, type: String, desc: 'DEPRECATED: Renaming environment can lead to errors, this will be removed in 15.0' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' optional :slug, absence: { message: "is automatically generated and cannot be changed" } + optional :tier, type: String, values: Environment.tiers.keys, desc: 'The tier of the environment to be created' end put ':id/environments/:environment_id' do authorize! :update_environment, user_project environment = user_project.environments.find(params[:environment_id]) - update_params = declared_params(include_missing: false).extract!(:name, :external_url) + update_params = declared_params(include_missing: false).extract!(:name, :external_url, :tier) if environment.update(update_params) present environment, with: Entities::Environment, current_user: current_user else diff --git a/lib/api/error_tracking/client_keys.rb b/lib/api/error_tracking/client_keys.rb index e97df03b6f0..d92cf220433 100644 --- a/lib/api/error_tracking/client_keys.rb +++ b/lib/api/error_tracking/client_keys.rb @@ -43,6 +43,8 @@ module API delete '/client_keys/:key_id' do key = user_project.error_tracking_client_keys.find(params[:key_id]) key.destroy! + + present key, with: Entities::ErrorTracking::ClientKey end end end diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb index 22a4e04a91c..29b213eaffb 100644 --- a/lib/api/error_tracking/collector.rb +++ b/lib/api/error_tracking/collector.rb @@ -12,7 +12,7 @@ module API content_type :txt, 'text/plain' default_format :envelope - rescue_from ActiveRecord::RecordInvalid do |e| + rescue_from Gitlab::ErrorTracking::ErrorRepository::DatabaseError do |e| render_api_error!(e.message, 400) end diff --git a/lib/api/events.rb b/lib/api/events.rb index db5ed7b7e6e..0a0141484ef 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -9,6 +9,7 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } feature_category :users + urgency :low resource :events do desc "List currently authenticated user's events" do diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb index c1f958ac007..42050888c14 100644 --- a/lib/api/feature_flags.rb +++ b/lib/api/feature_flags.rb @@ -8,6 +8,7 @@ module API .merge(name: API::NO_SLASH_URL_PART_REGEX) feature_category :feature_flags + urgency :low before do authorize_read_feature_flags! diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb index 8577da173b1..854719db4a1 100644 --- a/lib/api/feature_flags_user_lists.rb +++ b/lib/api/feature_flags_user_lists.rb @@ -9,6 +9,7 @@ module API } feature_category :feature_flags + urgency :low before do authorize_admin_feature_flags_user_lists! diff --git a/lib/api/features.rb b/lib/api/features.rb index 398e57794c8..bff2817a2ec 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -5,6 +5,7 @@ module API before { authenticated_as_admin! } feature_category :feature_flags + urgency :low helpers do def gate_value(params) @@ -69,12 +70,14 @@ module API optional :feature_group, type: String, desc: 'A Feature group name' optional :user, type: String, desc: 'A GitLab username' optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'" + optional :namespace, type: String, desc: "A GitLab group or user namespace path, such as 'gitlab-org'" optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' optional :force, type: Boolean, desc: 'Skip feature flag validation checks, ie. YAML definition' mutually_exclusive :key, :feature_group mutually_exclusive :key, :user mutually_exclusive :key, :group + mutually_exclusive :key, :namespace mutually_exclusive :key, :project end post ':name' do diff --git a/lib/api/files.rb b/lib/api/files.rb index 41a8e899614..fd574ca865b 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -56,6 +56,16 @@ module API end end + def fetch_blame_range(blame_params) + return if blame_params[:range].blank? + + range = Range.new(blame_params[:range][:start], blame_params[:range][:end]) + + render_api_error!('range[start] must be less than or equal to range[end]', 400) if range.begin > range.end + + range + end + def blob_data { file_name: @blob.name, @@ -110,13 +120,19 @@ module API params do requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + optional :range, type: Hash do + requires :start, type: Integer, desc: 'The first line of the range to blame', allow_blank: false, values: ->(v) { v > 0 } + requires :end, type: Integer, desc: 'The last line of the range to blame', allow_blank: false, values: ->(v) { v > 0 } + end end get ":id/repository/files/:file_path/blame", requirements: FILE_ENDPOINT_REQUIREMENTS do + blame_params = declared_params(include_missing: false) + assign_file_vars! set_http_headers(blob_data) - blame_ranges = Gitlab::Blame.new(@blob, @commit).groups(highlight: false) + blame_ranges = Gitlab::Blame.new(@blob, @commit, range: fetch_blame_range(blame_params)).groups(highlight: false) present blame_ranges, with: Entities::BlameRange end diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb index d001ced8581..e69baeee97f 100644 --- a/lib/api/freeze_periods.rb +++ b/lib/api/freeze_periods.rb @@ -7,6 +7,7 @@ module API before { authenticate! } feature_category :continuous_delivery + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 97230976482..0b1c06b3c26 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -10,6 +10,7 @@ module API ALLOWED_STATUSES = %w[default hidden].freeze feature_category :package_registry + urgency :low before do require_packages_enabled! diff --git a/lib/api/geo.rb b/lib/api/geo.rb index 9fc610c9b32..85f242cd135 100644 --- a/lib/api/geo.rb +++ b/lib/api/geo.rb @@ -3,6 +3,7 @@ module API class Geo < ::API::Base feature_category :geo_replication + urgency :low helpers do # Overridden in EE diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index ea30f17522e..2d9c0cd6ce1 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -5,6 +5,7 @@ module API helpers ::API::Helpers::PackagesHelpers feature_category :package_registry + urgency :low # basic semver, except case encoded (A => !a) MODULE_VERSION_REGEX = /v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([-.!a-z0-9]+))?(?:\+([-.!a-z0-9]+))?/.freeze diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index e9350da555c..180b6110cf2 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -8,6 +8,7 @@ module API prepend_mod_with('API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule feature_category :team_planning + urgency :low before { authenticate! } diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index a5a60ce8741..edaa32c26c4 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -10,6 +10,7 @@ module API end feature_category :kubernetes_management + urgency :low params do requires :id, type: String, desc: 'The ID of the group' @@ -138,7 +139,7 @@ module API end def ensure_feature_enabled! - not_found! unless Feature.enabled?(:certificate_based_clusters, user_group, default_enabled: :yaml, type: :ops) + not_found! unless user_group.certificate_based_clusters_enabled? end end end diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index 55e18fd1370..b834d177a12 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -9,7 +9,8 @@ module API before { authorize_read_group_container_images! } - feature_category :package_registry + feature_category :container_registry + urgency :low REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) @@ -24,8 +25,6 @@ module API end params do use :pagination - optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' - optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included' end get ':id/registry/repositories' do repositories = ContainerRepositoriesFinder.new( @@ -34,7 +33,7 @@ module API track_package_event('list_repositories', :container, user: current_user, namespace: user_group) - present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: false, tags_count: false end end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 5754eceda97..2948960a9b4 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -7,6 +7,7 @@ module API end feature_category :importers + urgency :low params do requires :id, type: String, desc: 'The ID of a group' diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb index 4a752732652..abb8c10efc6 100644 --- a/lib/api/group_import.rb +++ b/lib/api/group_import.rb @@ -3,6 +3,7 @@ module API class GroupImport < ::API::Base feature_category :importers + urgency :low helpers Helpers::FileUploadHelpers diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 7c1f23be828..e4cbe442f58 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low params do requires :id, type: String, desc: 'The ID of a group' diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index b097022e9c1..0096e466bef 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low params do requires :id, type: String, desc: 'The ID of a group' diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index e396c9608cf..af6e2b1e422 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -9,6 +9,7 @@ module API end feature_category :package_registry + urgency :low helpers ::API::Helpers::PackagesHelpers diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index e726f9b61cc..2235746b254 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -21,7 +21,7 @@ module API params do use :pagination end - get ':id/variables' do + get ':id/variables', urgency: :low do variables = user_group.variables present paginate(variables), with: Entities::Ci::Variable end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 0ed14476c61..60bb51bf48f 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -196,7 +196,7 @@ module API use :optional_params end - post feature_category: :subgroups do + post feature_category: :subgroups, urgency: :low do parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? if parent_group authorize! :create_subgroup, parent_group @@ -229,7 +229,7 @@ module API use :optional_update_params use :optional_update_params_ee end - put ':id', feature_category: :subgroups do + put ':id', feature_category: :subgroups, urgency: :low do group = find_group!(params[:id]) group.preload_shared_group_links @@ -266,7 +266,7 @@ module API end desc 'Remove a group.' - delete ":id", feature_category: :subgroups do + delete ":id", feature_category: :subgroups, urgency: :low do group = find_group!(params[:id]) authorize! :admin_group, group check_subscription! group @@ -361,7 +361,7 @@ module API use :group_list_params use :with_custom_attributes end - get ":id/descendant_groups", feature_category: :subgroups do + get ":id/descendant_groups", feature_category: :subgroups, urgency: :low do finder_params = declared_params(include_missing: false).merge(include_parent_descendants: true) groups = find_groups(finder_params, params[:id]) present_groups params, groups @@ -418,7 +418,6 @@ module API optional :expires_at, type: Date, desc: 'Share expiration date' end post ":id/share", feature_category: :subgroups do - shared_group = find_group!(params[:id]) shared_with_group = find_group!(params[:group_id]) group_link_create_params = { @@ -426,11 +425,11 @@ module API expires_at: params[:expires_at] } - result = ::Groups::GroupLinks::CreateService.new(shared_group, shared_with_group, current_user, group_link_create_params).execute - shared_group.preload_shared_group_links + result = ::Groups::GroupLinks::CreateService.new(user_group, shared_with_group, current_user, group_link_create_params).execute + user_group.preload_shared_group_links if result[:status] == :success - present shared_group, with: Entities::GroupDetail, current_user: current_user + present user_group, with: Entities::GroupDetail, current_user: current_user else render_api_error!(result[:message], result[:http_status]) end diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index 4278d17e003..e0e4e02fa55 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -9,6 +9,7 @@ module API include ::API::Helpers::Authentication feature_category :package_registry + urgency :low PACKAGE_FILENAME = 'package.tgz' HELM_REQUIREMENTS = { diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index ee0520df8ff..a079c591519 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -144,7 +144,7 @@ module API return true unless job_token_authentication? return true unless route_authentication_setting[:job_token_scope] == :project - ::Feature.enabled?(:ci_job_token_scope, project, default_enabled: :yaml) && + ::Feature.enabled?(:ci_job_token_scope, project) && current_authenticated_job.project == project end @@ -160,7 +160,17 @@ module API def find_group!(id) group = find_group(id) + check_group_access(group) + end + + # rubocop: disable CodeReuse/ActiveRecord + def find_group_by_full_path!(full_path) + group = Group.find_by_full_path(full_path) + check_group_access(group) + end + # rubocop: enable CodeReuse/ActiveRecord + def check_group_access(group) return group if can?(current_user, :read_group, group) return unauthorized! if authenticate_non_public? @@ -384,7 +394,14 @@ module API end def order_options_with_tie_breaker - order_options = { params[:order_by] => params[:sort] } + order_by = if Feature.enabled?(:replace_order_by_created_at_with_id) && + params[:order_by] == 'created_at' + 'id' + else + params[:order_by] + end + + order_options = { order_by => params[:sort] } order_options['id'] ||= params[:sort] || 'asc' order_options end @@ -555,6 +572,8 @@ module API def present_carrierwave_file!(file, supports_direct_download: true) return not_found! unless file&.exists? + log_artifact_size(file) if file.is_a?(JobArtifactUploader) + if file.file_storage? present_disk_file!(file.path, file.filename) elsif supports_direct_download && file.class.direct_download_enabled? @@ -567,9 +586,6 @@ module API end def increment_counter(event_name) - feature_name = "usage_data_#{event_name}" - return unless Feature.enabled?(feature_name, default_enabled: :yaml) - Gitlab::UsageDataCounters.count(event_name) rescue StandardError => error Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") @@ -708,16 +724,23 @@ module API # Deprecated. Use `send_artifacts_entry` instead. def legacy_send_artifacts_entry(file, entry) header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) + log_artifact_size(file) body '' end def send_artifacts_entry(file, entry) header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) + header(*Gitlab::Workhorse.detect_content_type) + log_artifact_size(file) body '' end + def log_artifact_size(file) + Gitlab::ApplicationContext.push(artifact: file.model) + end + # The Grape Error Middleware only has access to `env` but not `params` nor # `request`. We workaround this by defining methods that returns the right # values. diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 46685df0989..e03f029a6ef 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -124,8 +124,7 @@ module API repository: repository.gitaly_repository.to_h, address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags(repository.project), - use_sidechannel: Feature.enabled?(:gitlab_shell_upload_pack_sidechannel, repository.project, default_enabled: :yaml) + features: Feature::Gitaly.server_feature_flags(repository.project) } end end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index f26ac1318b1..c91e153c7b9 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -63,7 +63,7 @@ module API def add_single_member_by_user_id(create_service_params) source = create_service_params[:source] - user_id = create_service_params[:user_ids] + user_id = create_service_params[:user_id] user = User.find_by(id: user_id) # rubocop: disable CodeReuse/ActiveRecord if user diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index f1125899f8c..7a9dd78e4ed 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -13,7 +13,6 @@ module API optional :build_git_strategy, type: String, values: %w(fetch clone), desc: 'The Git strategy. Defaults to `fetch`' optional :build_timeout, type: Integer, desc: 'Build timeout' optional :auto_cancel_pending_pipelines, type: String, values: %w(disabled enabled), desc: 'Auto-cancel pending pipelines' - optional :build_coverage_regex, type: String, desc: 'Test coverage parsing' optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`' optional :service_desk_enabled, type: Boolean, desc: 'Disable or enable the service desk' optional :keep_latest_artifact, type: Boolean, desc: 'Indicates if the latest artifact should be kept for this project.' @@ -41,8 +40,9 @@ module API optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis' optional :warn_about_potentially_unwanted_characters, type: Boolean, desc: 'Warn about Potentially Unwanted Characters' + optional :enforce_auth_checks_on_uploads, type: Boolean, desc: 'Enforce auth check on uploads' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' - optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' + optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diff threads on lines changed with a push' optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge' optional :container_registry_enabled, type: Boolean, desc: 'Deprecated: Use :container_registry_access_level instead. Flag indication if the container registry is enabled for that project' optional :container_expiration_policy_attributes, type: Hash do @@ -54,7 +54,7 @@ module API optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :allow_merge_on_skipped_pipeline, type: Boolean, desc: 'Allow to merge if pipeline is skipped' - optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all threads are resolved' optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :topics instead' optional :topics, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of topics for a project' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 @@ -124,7 +124,6 @@ module API :auto_devops_enabled, :auto_devops_deploy_strategy, :auto_cancel_pending_pipelines, - :build_coverage_regex, :build_git_strategy, :build_timeout, :builds_access_level, @@ -175,6 +174,7 @@ module API :service_desk_enabled, :keep_latest_artifact, :mr_default_target_self, + :enforce_auth_checks_on_uploads, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, @@ -192,8 +192,6 @@ module API def validate_git_import_url!(import_url) return if import_url.blank? - yield if block_given? - result = Import::ValidateRemoteGitEndpointService.new(url: import_url).execute # network call if result.error? diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb index 0f0d62dcbfb..0f2d6239d0d 100644 --- a/lib/api/import_bitbucket_server.rb +++ b/lib/api/import_bitbucket_server.rb @@ -3,10 +3,7 @@ module API class ImportBitbucketServer < ::API::Base feature_category :importers - - before do - forbidden! unless Gitlab::CurrentSettings.import_sources&.include?('bitbucket_server') - end + urgency :low helpers do def client diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index c91a7700f58..46ca8e4c428 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -3,19 +3,16 @@ module API class ImportGithub < ::API::Base feature_category :importers + urgency :low rescue_from Octokit::Unauthorized, with: :provider_unauthorized - before do - forbidden! unless Gitlab::CurrentSettings.import_sources&.include?('github') - end - helpers do def client @client ||= if Feature.enabled?(:remove_legacy_github_client) Gitlab::GithubImport::Client.new(params[:personal_access_token], host: params[:github_hostname]) else - Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options) + Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], **client_options) end end diff --git a/lib/api/integrations/jira_connect/subscriptions.rb b/lib/api/integrations/jira_connect/subscriptions.rb new file mode 100644 index 00000000000..fa19dc2be3f --- /dev/null +++ b/lib/api/integrations/jira_connect/subscriptions.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module API + class Integrations + module JiraConnect + class Subscriptions < ::API::Base + feature_category :integrations + + before { authenticate! } + + namespace :integrations do + namespace :jira_connect do + resource :subscriptions do + desc 'Subscribe a namespace to a JiraConnectInstallation' + params do + requires :jwt, type: String, desc: 'JWT token for authorization with the Jira Connect installation' + requires :namespace_path, type: String, desc: 'Path for the namespace that should be subscribed' + end + post do + not_found! unless Feature.enabled?(:jira_connect_oauth, current_user) + + jwt = Atlassian::JiraConnect::Jwt::Symmetric.new(params[:jwt]) + installation = JiraConnectInstallation.find_by_client_key(jwt.iss_claim) + + if !installation || !jwt.valid?(installation.shared_secret) || !jwt.verify_context_qsh_claim + unauthorized! + end + + jira_user = installation.client.user_info(jwt.sub_claim) + + result = ::JiraConnectSubscriptions::CreateService.new( + installation, + current_user, + namespace_path: params['namespace_path'], + jira_user: jira_user + ).execute + + if result[:status] == :success + status :created + { success: true } + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + end + end + end + end + end +end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 2ab5d482295..b53f855c3a2 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -43,7 +43,7 @@ module API # This is a separate method so that EE can alter its behaviour more # easily. - if Feature.enabled?(:rate_limit_gitlab_shell, default_enabled: :yaml) + if Feature.enabled?(:rate_limit_gitlab_shell) check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], actor.key_or_user]) end @@ -123,10 +123,19 @@ module API 'Could not find a user for the given key' unless actor.user end + # TODO: backwards compatibility; remove after https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/454 is merged def two_factor_otp_check { success: false, message: 'Feature is not available' } end + def two_factor_manual_otp_check + { success: false, message: 'Feature is not available' } + end + + def two_factor_push_otp_check + { success: false, message: 'Feature is not available' } + end + def with_admin_mode_bypass!(actor_id) return yield unless Gitlab::CurrentSettings.admin_mode @@ -320,10 +329,23 @@ module API end end + # TODO: backwards compatibility; remove after https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/454 is merged post '/two_factor_otp_check', feature_category: :authentication_and_authorization do status 200 - two_factor_otp_check + two_factor_manual_otp_check + end + + post '/two_factor_push_otp_check', feature_category: :authentication_and_authorization do + status 200 + + two_factor_push_otp_check + end + + post '/two_factor_manual_otp_check', feature_category: :authentication_and_authorization do + status 200 + + two_factor_manual_otp_check end end end diff --git a/lib/api/internal/container_registry/migration.rb b/lib/api/internal/container_registry/migration.rb index b84e14c6f31..c750db94dab 100644 --- a/lib/api/internal/container_registry/migration.rb +++ b/lib/api/internal/container_registry/migration.rb @@ -5,6 +5,7 @@ module API module ContainerRegistry class Migration < ::API::Base feature_category :container_registry + urgency :low STATUS_PRE_IMPORT_COMPLETE = 'pre_import_complete' STATUS_PRE_IMPORT_FAILED = 'pre_import_failed' @@ -36,23 +37,25 @@ module API requires :status, type: String, values: POSSIBLE_VALUES, desc: 'The migration step status' end put 'internal/registry/repositories/*repository_path/migration/status' do - repository = find_repository!(declared_params[:repository_path]) + ::Gitlab::Database::LoadBalancing::Session.current.use_primary do + repository = find_repository!(declared_params[:repository_path]) - unless repository.migration_in_active_state? - bad_request!("Wrong migration state (#{repository.migration_state})") - end - - case declared_params[:status] - when STATUS_PRE_IMPORT_COMPLETE - unless repository.finish_pre_import_and_start_import - bad_request!("Couldn't transition from pre_importing to importing") + unless repository.migration_in_active_state? + bad_request!("Wrong migration state (#{repository.migration_state})") end - when STATUS_IMPORT_COMPLETE - unless repository.finish_import - bad_request!("Couldn't transition from importing to import_done") + + case declared_params[:status] + when STATUS_PRE_IMPORT_COMPLETE + unless repository.finish_pre_import_and_start_import + bad_request!("Couldn't transition from pre_importing to importing") + end + when STATUS_IMPORT_COMPLETE + unless repository.finish_import + bad_request!("Couldn't transition from importing to import_done") + end + when STATUS_IMPORT_FAILED, STATUS_PRE_IMPORT_FAILED + repository.abort_import! end - when STATUS_IMPORT_FAILED, STATUS_PRE_IMPORT_FAILED - repository.abort_import end status 200 diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 59bc917a602..34acfac4cb1 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -5,7 +5,6 @@ module API module Internal class Kubernetes < ::API::Base feature_category :kubernetes_management - before do check_feature_enabled authenticate_gitlab_kas_request! @@ -48,7 +47,7 @@ module API end def check_feature_enabled - not_found! unless Feature.enabled?(:kubernetes_agent_internal_api, default_enabled: true, type: :ops) + not_found! unless Feature.enabled?(:kubernetes_agent_internal_api, type: :ops) end def check_agent_token @@ -68,7 +67,7 @@ module API detail 'Retrieves agent info for the given token' end route_setting :authentication, cluster_agent_token_allowed: true - get '/agent_info' do + get '/agent_info', urgency: :low do project = agent.project status 200 @@ -82,7 +81,7 @@ module API end end - namespace 'kubernetes/agent_configuration' do + namespace 'kubernetes/agent_configuration', urgency: :low do desc 'POST agent configuration' do detail 'Store configuration for an agent' end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 75f63a5d98f..6fb3eca0ba8 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -28,7 +28,7 @@ module API optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end - post ":id/invitations" do + post ":id/invitations", urgency: :low do ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/354016') bad_request!('Must provide either email or user_id as a parameter') if params[:email].blank? && params[:user_id].blank? @@ -36,7 +36,7 @@ module API source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id], source: source }) + create_service_params = params.merge(source: source) ::Members::InviteService.new(current_user, create_service_params).execute end diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 0e93a4adb65..cf075af8373 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -7,6 +7,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e9bb9fe7a97..971163c18db 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -9,6 +9,7 @@ module API before { authenticate_non_get! } feature_category :team_planning + urgency :low helpers do params :negatable_issue_filter_params do diff --git a/lib/api/labels.rb b/lib/api/labels.rb index e3253d15c15..e2d4f5d823a 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low LABEL_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( name: API::NO_SLASH_URL_PART_REGEX, diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 5245cd10564..2fed724f947 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -6,6 +6,7 @@ module API }.freeze feature_category :package_registry + urgency :low content_type :md5, 'text/plain' content_type :sha1, 'text/plain' diff --git a/lib/api/members.rb b/lib/api/members.rb index 01e859c94c4..e2045c6def7 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -6,12 +6,14 @@ module API before { authenticate! } - feature_category :authentication_and_authorization urgency :low helpers ::API::Helpers::MembersHelpers - %w[group project].each do |source_type| + { + "group" => :subgroups, + "project" => :projects + }.each do |source_type, feature_category| params do requires :id, type: String, desc: "The #{source_type} ID" end @@ -27,7 +29,7 @@ module API use :pagination end - get ":id/members" do + get ":id/members", feature_category: feature_category do source = find_source(source_type, params[:id]) members = paginate(retrieve_members(source, params: params)) @@ -46,7 +48,7 @@ module API use :pagination end - get ":id/members/all" do + get ":id/members/all", feature_category: feature_category do source = find_source(source_type, params[:id]) members = paginate(retrieve_members(source, params: params, deep: true)) @@ -61,7 +63,7 @@ module API requires :user_id, type: Integer, desc: 'The user ID of the member' end # rubocop: disable CodeReuse/ActiveRecord - get ":id/members/:user_id" do + get ":id/members/:user_id", feature_category: feature_category do source = find_source(source_type, params[:id]) members = source_members(source) @@ -78,7 +80,7 @@ module API requires :user_id, type: Integer, desc: 'The user ID of the member' end # rubocop: disable CodeReuse/ActiveRecord - get ":id/members/all/:user_id" do + get ":id/members/all/:user_id", feature_category: feature_category do source = find_source(source_type, params[:id]) members = find_all_members(source) @@ -100,16 +102,15 @@ module API optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end - post ":id/members" do + post ":id/members", feature_category: feature_category do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - user_id = params[:user_id].to_s - create_service_params = params.except(:user_id).merge({ user_ids: user_id, source: source }) + create_service_params = params.merge(source: source) - if add_multiple_members?(user_id) + if add_multiple_members?(params[:user_id].to_s) ::Members::CreateService.new(current_user, create_service_params).execute - elsif add_single_member?(user_id) + elsif add_single_member?(params[:user_id].to_s) add_single_member_by_user_id(create_service_params) end end @@ -123,7 +124,7 @@ module API optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end # rubocop: disable CodeReuse/ActiveRecord - put ":id/members/:user_id" do + put ":id/members/:user_id", feature_category: feature_category do source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) @@ -152,7 +153,7 @@ module API desc: 'Flag indicating if the removed member should be unassigned from any issues or merge requests within given group or project' end # rubocop: disable CodeReuse/ActiveRecord - delete ":id/members/:user_id" do + delete ":id/members/:user_id", feature_category: feature_category do source = find_source(source_type, params[:id]) member = source_members(source).find_by!(user_id: params[:user_id]) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index de9a2a198d9..730baae63a2 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -373,7 +373,7 @@ module API desc 'Get the merge request pipelines' do success Entities::Ci::PipelineBasic end - get ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do + get ':id/merge_requests/:merge_request_iid/pipelines', urgency: :low, feature_category: :continuous_integration do pipelines = merge_request_pipelines_with_access present paginate(pipelines), with: Entities::Ci::PipelineBasic @@ -382,7 +382,7 @@ module API desc 'Create a pipeline for merge request' do success ::API::Entities::Ci::Pipeline end - post ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do + post ':id/merge_requests/:merge_request_iid/pipelines', urgency: :low, feature_category: :continuous_integration do pipeline = ::MergeRequests::CreatePipelineService .new(project: user_project, current_user: current_user, params: { allow_duplicate: true }) .execute(find_merge_request_with_access(params[:merge_request_iid])) diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 1f3516e0667..4ff7096b5d9 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -30,7 +30,7 @@ module API use :pagination use :optional_list_params_ee end - get feature_category: :subgroups do + get feature_category: :subgroups, urgency: :low do owned_only = params[:owned_only] == true namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only) @@ -52,7 +52,7 @@ module API params do requires :id, type: String, desc: "Namespace's ID or path" end - get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups do + get ':id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do user_namespace = find_namespace!(params[:id]) present user_namespace, with: Entities::Namespace, current_user: current_user @@ -65,7 +65,7 @@ module API requires :namespace, type: String, desc: "Namespace's path" optional :parent_id, type: Integer, desc: "The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered." end - get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups do + get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do namespace_path = params[:namespace] exists = Namespace.without_project_namespaces.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists? diff --git a/lib/api/notes.rb b/lib/api/notes.rb index c12b3bf5562..2a854bd785e 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -94,7 +94,7 @@ module API note = create_note(noteable, opts) - if note.errors.attribute_names == [:commands_only] + if note.errors.attribute_names == [:commands_only, :command_names] status 202 present note, with: Entities::NoteCommands elsif note.valid? diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index 420eabb41db..8cd72d2ab15 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -6,6 +6,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low helpers ::API::Helpers::MembersHelpers diff --git a/lib/api/npm_instance_packages.rb b/lib/api/npm_instance_packages.rb index 12fc008e00f..e387dd65e41 100644 --- a/lib/api/npm_instance_packages.rb +++ b/lib/api/npm_instance_packages.rb @@ -4,6 +4,7 @@ module API helpers ::API::Helpers::Packages::Npm feature_category :package_registry + urgency :low rescue_from ActiveRecord::RecordInvalid do |e| render_api_error!(e.message, 400) diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index dbfc0a61577..21bb2e69799 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -4,6 +4,7 @@ module API helpers ::API::Helpers::Packages::Npm feature_category :package_registry + urgency :low rescue_from ActiveRecord::RecordInvalid do |e| render_api_error!(e.message, 400) diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 03d1492908d..1e630cffea1 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -105,7 +105,7 @@ module API params do use :file_params end - put do + put urgency: :low do upload_nuget_package_file do |package| track_package_event( 'push_package', @@ -121,7 +121,7 @@ module API forbidden! end - put 'authorize' do + put 'authorize', urgency: :low do authorize_nuget_upload end @@ -133,7 +133,7 @@ module API params do use :file_params end - put 'symbolpackage' do + put 'symbolpackage', urgency: :low do upload_nuget_package_file(symbol_package: true) do |package| track_package_event( 'push_symbol_package', @@ -149,7 +149,7 @@ module API forbidden! end - put 'symbolpackage/authorize' do + put 'symbolpackage/authorize', urgency: :low do authorize_nuget_upload end @@ -165,7 +165,7 @@ module API desc 'The NuGet Content Service - index request' do detail 'This feature was introduced in GitLab 12.8' end - get 'index', format: :json do + get 'index', format: :json, urgency: :low do present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages(params[:package_name])), with: ::API::Entities::Nuget::PackagesVersions end @@ -177,7 +177,7 @@ module API requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX end - get '*package_version/*package_filename', format: [:nupkg, :snupkg] do + get '*package_version/*package_filename', format: [:nupkg, :snupkg], urgency: :low do filename = "#{params[:package_filename]}.#{params[:format]}" package_file = ::Packages::PackageFileFinder.new(find_package(params[:package_name], params[:package_version]), filename, with_file_name_like: true) .execute diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 4861c0c740e..278dc4c2044 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -9,6 +9,7 @@ module API end feature_category :package_registry + urgency :low helpers ::API::Helpers::PackagesHelpers diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index 56590bb9a8f..40e6486dae9 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -39,6 +39,12 @@ module API def find_token(id) PersonalAccessToken.find(id) || not_found! end + + def revoke_token(token) + service = ::PersonalAccessTokens::RevokeService.new(current_user, token: token).execute + + service.success? ? no_content! : bad_request!(nil) + end end resources :personal_access_tokens do @@ -48,13 +54,14 @@ module API present paginate(tokens), with: Entities::PersonalAccessToken end + delete 'self' do + revoke_token(access_token) + end + delete ':id' do - service = ::PersonalAccessTokens::RevokeService.new( - current_user, - token: find_token(params[:id]) - ).execute + token = find_token(params[:id]) - service.success? ? no_content! : bad_request!(nil) + revoke_token(token) end end end diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 8bba67a53af..4644d38ea80 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -10,6 +10,7 @@ module API end feature_category :kubernetes_management + urgency :low params do requires :id, type: String, desc: 'The ID of the project' @@ -143,7 +144,9 @@ module API end def ensure_feature_enabled! - not_found! unless Feature.enabled?(:certificate_based_clusters, user_project, default_enabled: :yaml, type: :ops) + namespace = user_project.namespace + + not_found! unless namespace.certificate_based_clusters_enabled? end end end diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index d4efca6e8f2..6a6275ed02a 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -13,6 +13,7 @@ module API before { authorize_read_container_images! } feature_category :package_registry + urgency :low params do requires :id, type: String, desc: 'The ID of a project' @@ -91,7 +92,7 @@ module API # rubocop:disable CodeReuse/Worker CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id, - declared_params.except(:repository_id).merge(container_expiration_policy: false)) + declared_params.except(:repository_id)) # rubocop:enable CodeReuse/Worker track_package_event('delete_tag_bulk', :container, user: current_user, project: user_project, namespace: user_project.namespace) diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 8b27d8d2163..d610b5e4f95 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -3,6 +3,7 @@ module API class ProjectExport < ::API::Base feature_category :importers + urgency :low before do not_found! unless Gitlab::CurrentSettings.project_export_enabled? @@ -65,9 +66,13 @@ module API if export_strategy&.invalid? render_validation_error!(export_strategy) else - user_project.add_export_job(current_user: current_user, - after_export_strategy: export_strategy, - params: project_export_params) + begin + user_project.add_export_job(current_user: current_user, + after_export_strategy: export_strategy, + params: project_export_params) + rescue Project::ExportLimitExceeded => e + render_api_error!(e.message, 400) + end end accepted! @@ -75,7 +80,7 @@ module API resource do before do - not_found! unless ::Feature.enabled?(:bulk_import, default_enabled: :yaml) + not_found! unless ::Feature.enabled?(:bulk_import) end desc 'Start relations export' do diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index bd8faefa803..7a66044c5b6 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -8,6 +8,7 @@ module API helpers Helpers::FileUploadHelpers feature_category :importers + urgency :low before { authenticate! unless route.settings[:skip_authentication] } @@ -178,7 +179,7 @@ module API success Entities::ProjectImportStatus end post 'remote-import-s3' do - not_found! unless ::Feature.enabled?(:import_project_from_remote_file_s3, default_enabled: :yaml) + not_found! unless ::Feature.enabled?(:import_project_from_remote_file_s3) check_rate_limit! :project_import, scope: [current_user, :project_import] diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 435e4bed776..9f82dbf9813 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -8,6 +8,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index c997afea865..79a5ca531e1 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -9,6 +9,7 @@ module API end feature_category :package_registry + urgency :low helpers ::API::Helpers::PackagesHelpers diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 9f7b3f9b088..44b1acaca88 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -76,7 +76,7 @@ module API # Temporarily introduced for upload API: https://gitlab.com/gitlab-org/gitlab/-/issues/325788 def project_attachment_size(user_project) return PROJECT_ATTACHMENT_SIZE_EXEMPT if exempt_from_global_attachment_size?(user_project) - return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project, default_enabled: :yaml) + return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project) PROJECT_ATTACHMENT_SIZE_EXEMPT end @@ -90,10 +90,6 @@ module API Gitlab::AppLogger.info({ message: "File exceeds maximum size", file_bytes: file.size, project_id: user_project.id, project_path: user_project.full_path, upload_allowed: allowed }) end end - - def check_import_by_url_is_enabled - Gitlab::CurrentSettings.import_sources&.include?('git') || forbidden! - end end helpers do @@ -202,6 +198,11 @@ module API params[:builds_enabled] = params.delete(:jobs_enabled) if params.key?(:jobs_enabled) params end + + def add_import_params(params) + params[:import_type] = 'git' if params[:import_url]&.present? + params + end end resource :users, requirements: API::USER_REQUIREMENTS do @@ -214,7 +215,7 @@ module API use :statistics_params use :with_custom_attributes end - get ":user_id/projects", feature_category: :projects, urgency: :default do + get ":user_id/projects", feature_category: :projects, urgency: :low do user = find_user(params[:user_id]) not_found!('User') unless user @@ -231,7 +232,7 @@ module API use :collection_params use :statistics_params end - get ":user_id/starred_projects", feature_category: :projects do + get ":user_id/starred_projects", feature_category: :projects, urgency: :low do user = find_user(params[:user_id]) not_found!('User') unless user @@ -267,13 +268,14 @@ module API use :optional_create_project_params use :create_params end - post do + post urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/issues/21139') attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) + attrs = add_import_params(attrs) filter_attributes_using_license!(attrs) - validate_git_import_url!(params[:import_url]) { check_import_by_url_is_enabled } + validate_git_import_url!(params[:import_url]) project = ::Projects::CreateService.new(current_user, attrs).execute @@ -286,6 +288,8 @@ module API error!(project.errors[:limit_reached], 403) end + forbidden! if project.errors[:import_source_disabled].present? + render_validation_error!(project) end end @@ -311,6 +315,7 @@ module API attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) + attrs = add_import_params(attrs) filter_attributes_using_license!(attrs) validate_git_import_url!(params[:import_url]) @@ -321,6 +326,8 @@ module API user_can_admin_project: can?(current_user, :admin_project, project), current_user: current_user else + forbidden! if project.errors[:import_source_disabled].present? + render_validation_error!(project) end end @@ -342,7 +349,7 @@ module API desc: 'Include project license data' end # TODO: Set higher urgency https://gitlab.com/gitlab-org/gitlab/-/issues/357622 - get ":id", feature_category: :projects, urgency: :default do + get ":id", feature_category: :projects, urgency: :low do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, @@ -441,6 +448,7 @@ module API authorize! :change_visibility_level, user_project if user_project.visibility_attribute_present?(attrs) attrs = translate_params_for_compatibility(attrs) + attrs = add_import_params(attrs) filter_attributes_using_license!(attrs) verify_update_project_attrs!(user_project, attrs) @@ -469,7 +477,7 @@ module API desc 'Unarchive a project' do success Entities::Project end - post ':id/unarchive', feature_category: :projects do + post ':id/unarchive', feature_category: :projects, urgency: :default do authorize!(:archive_project, user_project) ::Projects::UpdateService.new(user_project, current_user, archived: false).execute @@ -575,14 +583,14 @@ module API end post ":id/share", feature_category: :authentication_and_authorization do authorize! :admin_project, user_project - group = Group.find_by_id(params[:group_id]) + shared_with_group = Group.find_by_id(params[:group_id]) unless user_project.allowed_to_share_with_group? break render_api_error!("The project sharing with group is disabled", 400) end - result = ::Projects::GroupLinks::CreateService.new(user_project, current_user, declared_params(include_missing: false)) - .execute(group) + result = ::Projects::GroupLinks::CreateService + .new(user_project, shared_with_group, current_user, declared_params(include_missing: false)).execute if result[:status] == :success present result[:link], with: Entities::ProjectGroupLink @@ -663,7 +671,7 @@ module API optional :skip_users, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Filter out users with the specified IDs' use :pagination end - get ':id/users', feature_category: :authentication_and_authorization do + get ':id/users', urgency: :low, feature_category: :authentication_and_authorization do users = DeclarativePolicy.subject_scope { user_project.team.users } users = users.search(params[:search]) if params[:search].present? users = users.where_not_in(params[:skip_users]) if params[:skip_users].present? @@ -706,6 +714,17 @@ module API end end + desc 'Start a task to recalculate repository size for a project' do + detail 'This feature was introduced in GitLab 15.0.' + end + post ':id/repository_size', feature_category: :source_code_management do + authorize_admin_project + + user_project.repository.expire_statistics_caches + + ::Projects::UpdateStatisticsService.new(user_project, nil, statistics: [:repository_size, :lfs_objects_size]).execute + end + desc 'Transfer a project to a new namespace' params do requires :namespace, type: String, desc: 'The ID or path of the new namespace' @@ -729,7 +748,7 @@ module API params do requires :id, type: String, desc: 'ID of a project' end - get ':id/storage', feature_category: :projects do + get ':id/storage', feature_category: :source_code_management do authenticated_as_admin! present user_project, with: Entities::ProjectRepositoryStorage, current_user: current_user diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index d4f51beb2e5..f11270457c9 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -14,6 +14,7 @@ module API include ::API::Helpers::Packages::BasicAuthHelpers::Constants feature_category :package_registry + urgency :low default_format :json diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index 52c73104bb4..bc5ffe5b21f 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -11,6 +11,7 @@ module API before { authorize! :read_release, user_project } feature_category :release_orchestration + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 9e085a91a7c..c69f45f1f38 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -9,6 +9,7 @@ module API RELEASE_CLI_USER_AGENT = 'GitLab-release-cli' feature_category :release_orchestration + urgency :low params do requires :id, type: String, desc: 'The ID of a group' @@ -29,8 +30,6 @@ module API use :pagination end get ":id/releases" do - not_found! unless Feature.enabled?(:group_releases_finder_inoperator) - finder_options = { sort: params[:sort] } diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 6ac5ad0518b..e6c54faebd9 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -8,6 +8,7 @@ module API helpers ::API::Helpers::PackagesHelpers feature_category :package_registry + urgency :low # The Marshal version can be found by "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}" # Updating the version should require a GitLab API version change. diff --git a/lib/api/search.rb b/lib/api/search.rb index 4ef8fef329c..fd4d46cf77d 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -11,6 +11,7 @@ module API end feature_category :global_search + urgency :low rescue_from ActiveRecord::QueryCanceled do |e| render_api_error!({ error: 'Request timed out' }, 408) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 774ab472f2d..c25a56d5f08 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -95,6 +95,7 @@ module API optional :invisible_captcha_enabled, type: Boolean, desc: 'Enable Invisible Captcha spam detection during signup.' optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' + optional :max_export_size, type: Integer, desc: 'Maximum export size in MB' optional :max_import_size, type: Integer, desc: 'Maximum import size in MB' optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' @@ -180,6 +181,7 @@ module API optional :runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for shared runners, in seconds' optional :group_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for group runners, in seconds' optional :project_runner_token_expiration_interval, type: Integer, desc: 'Token expiration interval for project runners, in seconds' + optional :pipeline_limit_per_project_user_sha, type: Integer, desc: "Maximum number of pipeline creation requests allowed per minute per user and commit. Set to 0 for unlimited requests per minute." Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index c30b9d7583a..bca1376d489 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -10,7 +10,8 @@ module API helpers do def queue_metrics - Sidekiq::Queue.all.each_with_object({}) do |queue, hash| + ::Gitlab::SidekiqConfig.routing_queues.each_with_object({}) do |queue_name, hash| + queue = Sidekiq::Queue.new(queue_name) hash[queue.name] = { backlog: queue.size, latency: queue.latency.to_i diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index 797b4aad033..8da77ba18ae 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -22,6 +22,7 @@ module API }.freeze feature_category :infrastructure_as_code + urgency :low after_validation do require_packages_enabled! diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index 29e71611092..7b111451b9f 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -8,6 +8,7 @@ module API include ::Gitlab::Utils::StrongMemoize feature_category :infrastructure_as_code + urgency :low default_format :json diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb index d3680323b9f..ca37c786666 100644 --- a/lib/api/terraform/state_version.rb +++ b/lib/api/terraform/state_version.rb @@ -6,6 +6,7 @@ module API default_format :json feature_category :infrastructure_as_code + urgency :low before do authenticate! diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 1bc3e25a46c..f1779df7cc6 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -7,6 +7,7 @@ module API before { authenticate! } feature_category :team_planning + urgency :low ISSUABLE_TYPES = { 'merge_requests' => ->(iid) { find_merge_request_with_access(iid) }, diff --git a/lib/api/topics.rb b/lib/api/topics.rb index e4a1fa2367e..15f79e75be3 100644 --- a/lib/api/topics.rb +++ b/lib/api/topics.rb @@ -38,7 +38,8 @@ module API success Entities::Projects::Topic end params do - requires :name, type: String, desc: 'Name' + requires :name, type: String, desc: 'Slug (name)' + requires :title, type: String, desc: 'Title' optional :description, type: String, desc: 'Description' optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic' end @@ -60,7 +61,8 @@ module API end params do requires :id, type: Integer, desc: 'ID of project topic' - optional :name, type: String, desc: 'Name' + optional :name, type: String, desc: 'Slug (name)' + optional :title, type: String, desc: 'Title' optional :description, type: String, desc: 'Description' optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for topic' end diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb index 6dadaf4fc54..37fe540cde1 100644 --- a/lib/api/unleash.rb +++ b/lib/api/unleash.rb @@ -30,7 +30,7 @@ module API end desc 'Get a list of features' - get 'client/features', urgency: :medium do + get 'client/features' do present :version, 1 present :features, feature_flags, with: ::API::Entities::UnleashFeature end diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 43c75206b88..6e81a578d4a 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -8,7 +8,7 @@ module API namespace 'usage_data' do before do - not_found! unless Feature.enabled?(:usage_data_api, default_enabled: :yaml, type: :ops) + not_found! unless Feature.enabled?(:usage_data_api, type: :ops) forbidden!('Invalid CSRF token is provided') unless verified_request? end @@ -40,7 +40,7 @@ module API desc 'Get a list of all metric definitions' do detail 'This feature was introduced in GitLab 13.11.' end - get 'metric_definitions' do + get 'metric_definitions', urgency: :low do content_type 'application/yaml' env['api.format'] = :binary diff --git a/lib/api/usage_data_non_sql_metrics.rb b/lib/api/usage_data_non_sql_metrics.rb index 983038e0263..41f369a43b8 100644 --- a/lib/api/usage_data_non_sql_metrics.rb +++ b/lib/api/usage_data_non_sql_metrics.rb @@ -5,10 +5,11 @@ module API before { authenticated_as_admin! } feature_category :service_ping + urgency :low namespace 'usage_data' do before do - not_found! unless Feature.enabled?(:usage_data_non_sql_metrics, default_enabled: :yaml, type: :ops) + not_found! unless Feature.enabled?(:usage_data_non_sql_metrics, type: :ops) end desc 'Get Non SQL usage ping metrics' do diff --git a/lib/api/usage_data_queries.rb b/lib/api/usage_data_queries.rb index 3432e71eb28..fe972942111 100644 --- a/lib/api/usage_data_queries.rb +++ b/lib/api/usage_data_queries.rb @@ -5,10 +5,11 @@ module API before { authenticated_as_admin! } feature_category :service_ping + urgency :low namespace 'usage_data' do before do - not_found! unless Feature.enabled?(:usage_data_queries_api, default_enabled: :yaml, type: :ops) + not_found! unless Feature.enabled?(:usage_data_queries_api, type: :ops) end desc 'Get raw SQL queries for usage data SQL metrics' do diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index e5dfac3b1a1..756901c5717 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -19,7 +19,7 @@ module API todos: current_user.todos_pending_count } - if Feature.enabled?(:mr_attention_requests, default_enabled: :yaml) + if current_user&.mr_attention_requests_enabled? counts[:attention_requests] = current_user.attention_requested_open_merge_requests_count end diff --git a/lib/api/users.rb b/lib/api/users.rb index b26611cfe03..b10458c4358 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -10,6 +10,8 @@ module API feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] + urgency :high, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] + resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do include CustomAttributesEndpoints @@ -99,7 +101,7 @@ module API use :optional_index_params_ee end # rubocop: disable CodeReuse/ActiveRecord - get feature_category: :users, urgency: :default do + get feature_category: :users, urgency: :low do authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present? unless current_user&.admin? @@ -781,7 +783,7 @@ module API optional :type, type: String, values: %w[Project Namespace] use :pagination end - get ":user_id/memberships", feature_category: :users do + get ":user_id/memberships", feature_category: :users, urgency: :high do authenticated_as_admin! user = find_user_by_id(params) @@ -1078,7 +1080,7 @@ module API params do use :pagination end - get "emails", feature_category: :users do + get "emails", feature_category: :users, urgency: :high do present paginate(current_user.emails), with: Entities::Email end @@ -1120,7 +1122,7 @@ module API optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs end - put "preferences", feature_category: :users do + put "preferences", feature_category: :users, urgency: :high do authenticate! preferences = current_user.user_preference diff --git a/lib/atlassian/jira_connect/asymmetric_jwt.rb b/lib/atlassian/jira_connect/asymmetric_jwt.rb deleted file mode 100644 index a5668701965..00000000000 --- a/lib/atlassian/jira_connect/asymmetric_jwt.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -module Atlassian - module JiraConnect - # See documentation about Atlassian asymmetric JWT verification: - # https://developer.atlassian.com/cloud/jira/platform/understanding-jwt-for-connect-apps/#verifying-a-asymmetric-jwt-token-for-install-callbacks - - class AsymmetricJwt - include Gitlab::Utils::StrongMemoize - - KeyFetchError = Class.new(StandardError) - - ALGORITHM = 'RS256' - PUBLIC_KEY_CDN_URL = 'https://connect-install-keys.atlassian.com/' - UUID4_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/.freeze - - def initialize(token, verification_claims) - @token = token - @verification_claims = verification_claims - end - - def valid? - claims.present? && claims['qsh'] == verification_qsh - end - - def iss_claim - return unless claims - - claims['iss'] - end - - private - - def claims - strong_memoize(:claims) do - _, jwt_headers = decode_token - public_key = retrieve_public_key(jwt_headers['kid']) - decoded_claims, _ = decode_token(public_key, true, **relevant_claims, verify_aud: true, verify_iss: true, algorithm: ALGORITHM) - - decoded_claims - rescue JWT::DecodeError, OpenSSL::PKey::PKeyError, KeyFetchError - end - end - - def decode_token(key = nil, verify = false, **claims) - Atlassian::Jwt.decode(@token, key, verify, **claims) - end - - def retrieve_public_key(key_id) - raise KeyFetchError unless UUID4_REGEX.match?(key_id) - - public_key = Gitlab::HTTP.try_get(PUBLIC_KEY_CDN_URL + key_id).try(:body) - - raise KeyFetchError if public_key.blank? - - OpenSSL::PKey.read(public_key) - end - - def relevant_claims - @verification_claims.slice(:aud, :iss) - end - - def verification_qsh - @verification_claims[:qsh] - end - end - end -end diff --git a/lib/atlassian/jira_connect/jwt/asymmetric.rb b/lib/atlassian/jira_connect/jwt/asymmetric.rb new file mode 100644 index 00000000000..0611a17c005 --- /dev/null +++ b/lib/atlassian/jira_connect/jwt/asymmetric.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Jwt + # See documentation about Atlassian asymmetric JWT verification: + # https://developer.atlassian.com/cloud/jira/platform/understanding-jwt-for-connect-apps/#verifying-a-asymmetric-jwt-token-for-install-callbacks + + class Asymmetric + include Gitlab::Utils::StrongMemoize + + KeyFetchError = Class.new(StandardError) + + ALGORITHM = 'RS256' + PUBLIC_KEY_CDN_URL = 'https://connect-install-keys.atlassian.com/' + UUID4_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/.freeze + + def initialize(token, verification_claims) + @token = token + @verification_claims = verification_claims + end + + def valid? + claims.present? && claims['qsh'] == verification_qsh + end + + def iss_claim + return unless claims + + claims['iss'] + end + + private + + def claims + strong_memoize(:claims) do + _, jwt_headers = decode_token + public_key = retrieve_public_key(jwt_headers['kid']) + + decoded_claims(public_key) + rescue JWT::DecodeError, OpenSSL::PKey::PKeyError, KeyFetchError + end + end + + def decoded_claims(public_key) + decode_token( + public_key, + true, + **relevant_claims, + verify_aud: true, + verify_iss: true, + algorithm: ALGORITHM + ).first + end + + def decode_token(key = nil, verify = false, **claims) + Atlassian::Jwt.decode(@token, key, verify, **claims) + end + + def retrieve_public_key(key_id) + raise KeyFetchError unless UUID4_REGEX.match?(key_id) + + public_key = Gitlab::HTTP.try_get(PUBLIC_KEY_CDN_URL + key_id).try(:body) + + raise KeyFetchError if public_key.blank? + + OpenSSL::PKey.read(public_key) + end + + def relevant_claims + @verification_claims.slice(:aud, :iss) + end + + def verification_qsh + @verification_claims[:qsh] + end + end + end + end +end diff --git a/lib/atlassian/jira_connect/jwt/symmetric.rb b/lib/atlassian/jira_connect/jwt/symmetric.rb new file mode 100644 index 00000000000..61e5bd923a4 --- /dev/null +++ b/lib/atlassian/jira_connect/jwt/symmetric.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Jwt + class Symmetric + include Gitlab::Utils::StrongMemoize + + CONTEXT_QSH_STRING = 'context-qsh' + + def initialize(jwt) + @jwt = jwt + end + + def iss_claim + jwt_headers['iss'] + end + + def sub_claim + jwt_headers['sub'] + end + + def valid?(shared_secret) + Atlassian::Jwt.decode(@jwt, shared_secret).present? + rescue JWT::DecodeError + false + end + + def verify_qsh_claim(url_with_query, method, url) + qsh_claim == Atlassian::Jwt.create_query_string_hash(url_with_query, method, url) + rescue StandardError + false + end + + def verify_context_qsh_claim + qsh_claim == CONTEXT_QSH_STRING + end + + private + + def qsh_claim + jwt_headers['qsh'] + end + + def jwt_headers + strong_memoize(:jwt_headers) do + Atlassian::Jwt.decode(@jwt, nil, false).first + rescue JWT::DecodeError + {} + end + end + end + end + end +end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index 93342e789e9..a8b0e7ad157 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -31,7 +31,7 @@ module Backup args = [] args += ['-parallel', @max_parallelism.to_s] if @max_parallelism args += ['-parallel-storage', @storage_parallelism.to_s] if @storage_parallelism - if Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml) + if Feature.enabled?(:incremental_repository_backup) args += ['-layout', 'pointer'] if type == :create args += ['-incremental'] if @incremental diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 403b2d9f16c..0991177d044 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -9,6 +9,11 @@ module Backup # if some of these files are still there, we don't need them in the backup LEGACY_PAGES_TMP_PATH = '@pages.tmp' + LIST_ENVS = { + skipped: 'SKIP', + repositories_storages: 'REPOSITORIES_STORAGES' + }.freeze + TaskDefinition = Struct.new( :enabled, # `true` if the task can be used. Treated as `true` when not specified. :human_name, # Name of the task used for logging. @@ -29,20 +34,23 @@ module Backup @progress = progress @incremental = Feature.feature_flags_available? && - Feature.enabled?(:incremental_repository_backup, default_enabled: :yaml) && + Feature.enabled?(:incremental_repository_backup) && Gitlab::Utils.to_boolean(ENV['INCREMENTAL'], default: false) - @definitions = definitions || build_definitions + @definitions = definitions end def create if incremental? - unpack + unpack(ENV.fetch('PREVIOUS_BACKUP', ENV['BACKUP'])) read_backup_information verify_backup_version + update_backup_information end - @definitions.keys.each do |task_name| + build_backup_information + + definitions.keys.each do |task_name| run_create_task(task_name) end @@ -64,10 +72,10 @@ module Backup end def run_create_task(task_name) - definition = @definitions[task_name] - build_backup_information + definition = definitions[task_name] + unless definition.enabled? puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan) return @@ -79,7 +87,7 @@ module Backup end puts_time "Dumping #{definition.human_name} ... ".color(:blue) - definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), backup_id) + definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), full_backup_id) puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "done".color(:green) rescue Backup::DatabaseBackupError, Backup::FileBackupError => e @@ -87,11 +95,11 @@ module Backup end def restore - cleanup_required = unpack + cleanup_required = unpack(ENV['BACKUP']) read_backup_information verify_backup_version - @definitions.keys.each do |task_name| + definitions.keys.each do |task_name| run_restore_task(task_name) if !skipped?(task_name) && enabled_task?(task_name) end @@ -110,7 +118,9 @@ module Backup end def run_restore_task(task_name) - definition = @definitions[task_name] + read_backup_information + + definition = definitions[task_name] unless definition.enabled? puts_time "Restoring #{definition.human_name} ... ".color(:blue) + "[DISABLED]".color(:cyan) @@ -142,6 +152,10 @@ module Backup private + def definitions + @definitions ||= build_definitions + end + def build_definitions { 'db' => TaskDefinition.new( @@ -211,7 +225,7 @@ module Backup max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence strategy = Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - Repositories.new(progress, strategy: strategy) + Repositories.new(progress, strategy: strategy, storages: repositories_storages) end def build_files_task(app_files_dir, excludes: []) @@ -244,10 +258,24 @@ module Backup gitlab_version: Gitlab::VERSION, tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, - skipped: ENV["SKIP"] + skipped: ENV['SKIP'], + repositories_storages: ENV['REPOSITORIES_STORAGES'] } end + def update_backup_information + @backup_information.merge!( + full_backup_id: full_backup_id, + db_version: ActiveRecord::Migrator.current_version.to_s, + backup_created_at: Time.zone.now, + gitlab_version: Gitlab::VERSION, + tar_version: tar_version, + installation_type: Gitlab::INSTALLATION_TYPE, + skipped: list_env(:skipped).join(','), + repositories_storages: list_env(:repositories_storages).join(',') + ) + end + def backup_information raise Backup::Error, "#{MANIFEST_NAME} not yet loaded" unless @backup_information @@ -297,7 +325,7 @@ module Backup puts_time "Deleting tar staging files ... ".color(:blue) remove_backup_path(MANIFEST_NAME) - @definitions.each do |_, definition| + definitions.each do |_, definition| remove_backup_path(definition.cleanup_path || definition.destination_path) end @@ -374,8 +402,8 @@ module Backup end end - def unpack - if ENV['BACKUP'].blank? && non_tarred_backup? + def unpack(source_backup_id) + if source_backup_id.blank? && non_tarred_backup? puts_time "Non tarred backup found in #{backup_path}, using that" return false @@ -387,14 +415,14 @@ module Backup puts_time "No backups found in #{backup_path}" puts_time "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 - elsif backup_file_list.many? && ENV["BACKUP"].nil? + elsif backup_file_list.many? && source_backup_id.nil? puts_time 'Found more than one backup:' # print list of available backups puts_time " " + available_timestamps.join("\n ") if incremental? puts_time 'Please specify which one you want to create an incremental backup for:' - puts_time 'rake gitlab:backup:create INCREMENTAL=true BACKUP=timestamp_of_backup' + puts_time 'rake gitlab:backup:create INCREMENTAL=true PREVIOUS_BACKUP=timestamp_of_backup' else puts_time 'Please specify which one you want to restore:' puts_time 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' @@ -403,8 +431,8 @@ module Backup exit 1 end - tar_file = if ENV['BACKUP'].present? - File.basename(ENV['BACKUP']) + FILE_NAME_SUFFIX + tar_file = if source_backup_id.present? + File.basename(source_backup_id) + FILE_NAME_SUFFIX else backup_file_list.first end @@ -431,12 +459,26 @@ module Backup end def skipped?(item) - ENV.fetch('SKIP', '').include?(item) || - backup_information[:skipped] && backup_information[:skipped].include?(item) + skipped.include?(item) + end + + def skipped + @skipped ||= list_env(:skipped) + end + + def repositories_storages + @repositories_storages ||= list_env(:repositories_storages) + end + + def list_env(name) + list = ENV.fetch(LIST_ENVS[name], '').split(',') + list += backup_information[name].split(',') if backup_information[name] + list.uniq! + list end def enabled_task?(task_name) - @definitions[task_name].enabled? + definitions[task_name].enabled? end def backup_file?(file) @@ -491,7 +533,7 @@ module Backup end def backup_contents - [MANIFEST_NAME] + @definitions.reject do |name, definition| + [MANIFEST_NAME] + definitions.reject do |name, definition| skipped?(name) || !enabled_task?(name) || (definition.destination_optional && !File.exist?(File.join(backup_path, definition.destination_path))) end.values.map(&:destination_path) @@ -501,12 +543,19 @@ module Backup @tar_file ||= "#{backup_id}#{FILE_NAME_SUFFIX}" end + def full_backup_id + full_backup_id = backup_information[:full_backup_id] + full_backup_id ||= File.basename(ENV['PREVIOUS_BACKUP']) if ENV['PREVIOUS_BACKUP'].present? + full_backup_id ||= backup_id + full_backup_id + end + def backup_id - @backup_id ||= if ENV['BACKUP'].present? - File.basename(ENV['BACKUP']) - else - "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}" - end + if ENV['BACKUP'].present? + File.basename(ENV['BACKUP']) + else + "#{backup_information[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{backup_information[:gitlab_version]}" + end end def create_attributes diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 11bed84e356..4a31e87b969 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -6,10 +6,11 @@ module Backup class Repositories < Task extend ::Gitlab::Utils::Override - def initialize(progress, strategy:) + def initialize(progress, strategy:, storages: []) super(progress) @strategy = strategy + @storages = storages end override :dump @@ -35,7 +36,7 @@ module Backup private - attr_reader :strategy + attr_reader :strategy, :storages def enqueue_consecutive enqueue_consecutive_projects @@ -49,7 +50,7 @@ module Backup end def enqueue_consecutive_snippets - Snippet.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } + snippet_relation.find_each(batch_size: 1000) { |snippet| enqueue_snippet(snippet) } end def enqueue_project(project) @@ -63,7 +64,15 @@ module Backup end def project_relation - Project.includes(:route, :group, namespace: :owner) + scope = Project.includes(:route, :group, namespace: :owner) + scope = scope.id_in(ProjectRepository.for_repository_storage(storages).select(:project_id)) if storages.any? + scope + end + + def snippet_relation + scope = Snippet.all + scope = scope.id_in(SnippetRepository.for_repository_storage(storages).select(:snippet_id)) if storages.any? + scope end def restore_object_pools @@ -88,7 +97,7 @@ module Backup def cleanup_snippets_without_repositories invalid_snippets = [] - Snippet.find_each(batch_size: 1000).each do |snippet| + snippet_relation.find_each(batch_size: 1000).each do |snippet| response = Snippets::RepositoryValidationService.new(nil, snippet).execute next if response.success? diff --git a/lib/banzai/filter/image_lazy_load_filter.rb b/lib/banzai/filter/image_lazy_load_filter.rb index 916c135b777..a8a275d2039 100644 --- a/lib/banzai/filter/image_lazy_load_filter.rb +++ b/lib/banzai/filter/image_lazy_load_filter.rb @@ -4,13 +4,15 @@ module Banzai module Filter # HTML filter that moves the value of image `src` attributes to `data-src` - # so they can be lazy loaded. + # so they can be lazy loaded. Also sets decoding to 'async' so that the + # decoding of images doesn't block the loading of other content. class ImageLazyLoadFilter < HTML::Pipeline::Filter CSS = 'img' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze def call doc.xpath(XPATH).each do |img| + img['decoding'] = 'async' img.add_class('lazy') img['data-src'] = img['src'] img['src'] = LazyImageTagHelper.placeholder_image diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index a34519799d5..521fd7bf4cc 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -206,6 +206,7 @@ module Banzai link_content: !!link_content, link_reference: link_reference) data_attributes[:reference_format] = matches[:format] if matches.names.include?("format") + data_attributes.merge!(additional_object_attributes(object)) data = data_attribute(data_attributes) @@ -294,6 +295,10 @@ module Banzai placeholder_data[Regexp.last_match(1).to_i] end end + + def additional_object_attributes(object) + {} + end end end end diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb index 1053501de7b..337075b7ff8 100644 --- a/lib/banzai/filter/references/issue_reference_filter.rb +++ b/lib/banzai/filter/references/issue_reference_filter.rb @@ -31,6 +31,10 @@ module Banzai private + def additional_object_attributes(issue) + { issue_type: issue.issue_type } + end + def issue_path(issue, project) Gitlab::Routing.url_helpers.namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: issue.iid) end diff --git a/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb b/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb new file mode 100644 index 00000000000..01ee3f5d9e8 --- /dev/null +++ b/lib/banzai/pipeline/incident_management/timeline_event_pipeline.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + module IncidentManagement + class TimelineEventPipeline < PlainMarkdownPipeline + ALLOWLIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( + elements: %w(p b i strong em pre code a img) + ).freeze + + def self.filters + @filters ||= FilterArray[ + *super, + *Banzai::Pipeline::GfmPipeline.reference_filters, + Filter::EmojiFilter, + Filter::SanitizationFilter, + Filter::ExternalLinkFilter, + Filter::ImageLinkFilter + ] + end + + def self.transform_context(context) + Filter::AssetProxyFilter.transform_context(context).merge( + only_path: true, + no_sourcepos: true, + allowlist: ALLOWLIST, + link_replaces_image: true + ) + end + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb index fa4780dd8de..8d5b15e299a 100644 --- a/lib/bitbucket/representation/repo.rb +++ b/lib/bitbucket/representation/repo.rb @@ -3,10 +3,6 @@ module Bitbucket module Representation class Repo < Representation::Base - def initialize(raw) - super(raw) - end - def owner_and_slug @owner_and_slug ||= full_name.split('/', 2) end diff --git a/lib/bitbucket_server/representation/repo.rb b/lib/bitbucket_server/representation/repo.rb index 4cd5b75bbed..a3c5f387941 100644 --- a/lib/bitbucket_server/representation/repo.rb +++ b/lib/bitbucket_server/representation/repo.rb @@ -3,10 +3,6 @@ module BitbucketServer module Representation class Repo < Representation::Base - def initialize(raw) - super(raw) - end - def project_key raw.dig('project', 'key') end diff --git a/lib/bulk_imports/common/extractors/json_extractor.rb b/lib/bulk_imports/common/extractors/json_extractor.rb new file mode 100644 index 00000000000..45a48cedce8 --- /dev/null +++ b/lib/bulk_imports/common/extractors/json_extractor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Extractors + class JsonExtractor + def initialize(relation:) + @relation = relation + @tmpdir = Dir.mktmpdir + end + + def extract(context) + download_service(context).execute + decompression_service.execute + + attributes = ndjson_reader.consume_attributes(relation) + + BulkImports::Pipeline::ExtractedData.new(data: attributes) + end + + def remove_tmpdir + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + attr_reader :relation, :tmpdir + + def filename + "#{relation}.json.gz" + end + + def download_service(context) + @download_service ||= BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(relation), + tmpdir: tmpdir, + filename: filename + ) + end + + def decompression_service + @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: filename) + end + + def ndjson_reader + @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmpdir) + end + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 0f4def3b17a..915dcf1b455 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -4,7 +4,7 @@ module BulkImports module Common module Pipelines class EntityFinisher - def self.ndjson_pipeline? + def self.file_extraction_pipeline? false end diff --git a/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb new file mode 100644 index 00000000000..d4f5901e971 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/group_attributes_pipeline.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class GroupAttributesPipeline + include Pipeline + + file_extraction_pipeline! + + relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION + + extractor ::BulkImports::Common::Extractors::JsonExtractor, relation: relation + + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + + def transform(_context, data) + return unless data + + data.symbolize_keys.slice(:membership_lock) + end + + def load(_context, data) + return unless data + + ::Groups::UpdateService.new(portable, current_user, data).execute + end + + def after_run(_context) + extractor.remove_tmpdir + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/namespace_settings_pipeline.rb b/lib/bulk_imports/groups/pipelines/namespace_settings_pipeline.rb new file mode 100644 index 00000000000..1bd6486b413 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/namespace_settings_pipeline.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class NamespaceSettingsPipeline + include Pipeline + + file_extraction_pipeline! + + relation_name 'namespace_settings' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + + def transform(_context, data) + return unless data + + data.first.symbolize_keys.slice(*allowed_attributes) + end + + def load(_context, data) + return unless data + + ::Groups::UpdateService.new(portable, current_user, data).execute + end + + def after_run(_context) + extractor.remove_tmpdir + end + + private + + def allowed_attributes + Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h.dig(:included_attributes, :namespace_settings) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 97a423b6ea9..c4db53424fd 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -11,10 +11,18 @@ module BulkImports pipeline: BulkImports::Groups::Pipelines::GroupPipeline, stage: 0 }, + group_attributes: { + pipeline: BulkImports::Groups::Pipelines::GroupAttributesPipeline, + stage: 1 + }, subgroups: { pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, stage: 1 }, + namespace_settings: { + pipeline: BulkImports::Groups::Pipelines::NamespaceSettingsPipeline, + stage: 1 + }, members: { pipeline: BulkImports::Common::Pipelines::MembersPipeline, stage: 1 @@ -69,9 +77,9 @@ module BulkImports if destination_namespace.present? root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor - ::Feature.enabled?(:bulk_import_projects, root_ancestor, default_enabled: :yaml) + ::Feature.enabled?(:bulk_import_projects, root_ancestor) else - ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml) + ::Feature.enabled?(:bulk_import_projects) end end end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index 23e898a7bb2..df27275b664 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -8,7 +8,6 @@ module BulkImports import_entity = context.entity data - .then { |data| transform_name(import_entity, data) } .then { |data| transform_path(import_entity, data) } .then { |data| transform_full_path(data) } .then { |data| transform_parent(context, import_entity, data) } @@ -19,11 +18,6 @@ module BulkImports private - def transform_name(import_entity, data) - data['name'] = import_entity.destination_name - data - end - def transform_path(import_entity, data) data['path'] = import_entity.destination_name.parameterize data diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb index 676a6ca8d2a..d8fb937ecd2 100644 --- a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb +++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb @@ -8,7 +8,7 @@ module BulkImports { source_type: :group_entity, source_full_path: entry['full_path'], - destination_name: entry['name'], + destination_name: entry['path'], destination_namespace: context.entity.group.full_path, parent_id: context.entity.id } diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index d85e51984df..05d724a5e42 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -7,7 +7,7 @@ module BulkImports include Pipeline included do - ndjson_pipeline! + file_extraction_pipeline! def transform(context, data) return unless data diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 6798936576b..dc2ebdddd14 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -170,12 +170,12 @@ module BulkImports class_attributes[:abort_on_failure] end - def ndjson_pipeline! - class_attributes[:ndjson_pipeline] = true + def file_extraction_pipeline! + class_attributes[:file_extraction_pipeline] = true end - def ndjson_pipeline? - class_attributes[:ndjson_pipeline] + def file_extraction_pipeline? + class_attributes[:file_extraction_pipeline] end def relation_name(name) diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb index 2492a023cbe..1754f27137c 100644 --- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -6,6 +6,10 @@ module BulkImports class ProjectAttributesPipeline include Pipeline + file_extraction_pipeline! + + relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer def extract(_context) @@ -55,7 +59,7 @@ module BulkImports def download_service @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), + relative_url: context.entity.relation_download_url_path(self.class.relation), tmpdir: tmpdir, filename: compressed_filename ) @@ -70,7 +74,7 @@ module BulkImports end def filename - "#{BulkImports::FileTransfer::BaseConfig::SELF_RELATION}.json" + "#{self.class.relation}.json" end def json_decode(string) diff --git a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb new file mode 100644 index 00000000000..8f9c6a5749f --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ReleasesPipeline + include NdjsonPipeline + + relation_name 'releases' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index b920c1bf355..229df9c410d 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -63,10 +63,6 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline, stage: 4 }, - ci_pipelines: { - pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline, - stage: 4 - }, project_feature: { pipeline: BulkImports::Projects::Pipelines::ProjectFeaturePipeline, stage: 4 @@ -79,6 +75,14 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::ServiceDeskSettingPipeline, stage: 4 }, + releases: { + pipeline: BulkImports::Projects::Pipelines::ReleasesPipeline, + stage: 4 + }, + ci_pipelines: { + pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline, + stage: 5 + }, wiki: { pipeline: BulkImports::Common::Pipelines::WikiPipeline, stage: 5 diff --git a/lib/constraints/feature_constrainer.rb b/lib/constraints/feature_constrainer.rb deleted file mode 100644 index cd246cf37a4..00000000000 --- a/lib/constraints/feature_constrainer.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Constraints - class FeatureConstrainer - attr_reader :args - - def initialize(*args) - @args = args - end - - def matches?(_request) - Feature.enabled?(*args) - end - end -end diff --git a/lib/container_registry/base_client.rb b/lib/container_registry/base_client.rb index bb9422ae048..66bc934d1ef 100644 --- a/lib/container_registry/base_client.rb +++ b/lib/container_registry/base_client.rb @@ -31,9 +31,6 @@ module ContainerRegistry end }.freeze - # Taken from: FaradayMiddleware::FollowRedirects - REDIRECT_CODES = Set.new [301, 302, 303, 307] - class << self private @@ -98,23 +95,10 @@ module ContainerRegistry conn.adapter :net_http end - def response_body(response, allow_redirect: false) - if allow_redirect && REDIRECT_CODES.include?(response.status) - response = redirect_response(response.headers['location']) - end - + def response_body(response) response.body if response && response.success? end - def redirect_response(location) - return unless location - - uri = URI(@base_uri).merge(location) - raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme) - - faraday_redirect.get(uri) - end - def configure_connection(conn) conn.headers['Accept'] = ACCEPTED_TYPES @@ -125,18 +109,6 @@ module ContainerRegistry conn.response :json, content_type: OCI_MANIFEST_V1_TYPE end - # Create a new request to make sure the Authorization header is not inserted - # via the Faraday middleware - def faraday_redirect - @faraday_redirect ||= faraday_base do |conn| - conn.request :json - - conn.request(:retry, RETRY_OPTIONS) - conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) - conn.adapter :net_http - end - end - def delete_if_exists(path) result = faraday.delete(path) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 4b2250d089d..498bc11b168 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -130,7 +130,7 @@ module ContainerRegistry def blob(name, digest, type = nil) type ||= 'application/octet-stream' - response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true + response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type) end def delete_blob(name, digest) @@ -152,9 +152,7 @@ module ContainerRegistry @faraday_blob ||= faraday_base do |conn| initialize_connection(conn, @options) - if Feature.enabled?(:container_registry_follow_redirects_middleware, default_enabled: :yaml) - conn.use ::FaradayMiddleware::FollowRedirects, REDIRECT_OPTIONS - end + conn.use ::FaradayMiddleware::FollowRedirects, REDIRECT_OPTIONS end end end diff --git a/lib/container_registry/migration.rb b/lib/container_registry/migration.rb index 005ef880034..8377190c83c 100644 --- a/lib/container_registry/migration.rb +++ b/lib/container_registry/migration.rb @@ -20,6 +20,8 @@ module ContainerRegistry delegate :container_registry_import_max_step_duration, to: ::Gitlab::CurrentSettings delegate :container_registry_import_target_plan, to: ::Gitlab::CurrentSettings delegate :container_registry_import_created_before, to: ::Gitlab::CurrentSettings + delegate :container_registry_pre_import_timeout, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_timeout, to: ::Gitlab::CurrentSettings alias_method :max_tags_count, :container_registry_import_max_tags_count alias_method :max_retries, :container_registry_import_max_retries @@ -27,6 +29,8 @@ module ContainerRegistry alias_method :max_step_duration, :container_registry_import_max_step_duration alias_method :target_plan_name, :container_registry_import_target_plan alias_method :created_before, :container_registry_import_created_before + alias_method :pre_import_timeout, :container_registry_pre_import_timeout + alias_method :import_timeout, :container_registry_import_timeout end def self.enabled? @@ -52,6 +56,8 @@ module ContainerRegistry # return 25 if Feature.enabled?(:container_registry_migration_phase2_capacity_25) return 10 if Feature.enabled?(:container_registry_migration_phase2_capacity_10) + return 5 if Feature.enabled?(:container_registry_migration_phase2_capacity_5) + return 2 if Feature.enabled?(:container_registry_migration_phase2_capacity_2) return 1 if Feature.enabled?(:container_registry_migration_phase2_capacity_1) 0 @@ -64,5 +70,13 @@ module ContainerRegistry def self.all_plans? Feature.enabled?(:container_registry_migration_phase2_all_plans) end + + def self.enqueue_twice? + Feature.enabled?(:container_registry_migration_phase2_enqueue_twice) + end + + def self.enqueuer_loop? + Feature.enabled?(:container_registry_migration_phase2_enqueuer_loop) + end end end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 8833207dd1d..8c3377fdb80 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -15,6 +15,8 @@ class EventFilter WIKI = 'wiki' DESIGNS = 'designs' + PROJECT_ONLY_EVENT_TYPES = [PUSH, MERGED, TEAM, ISSUE, DESIGNS].freeze + def initialize(filter) # Split using comma to maintain backward compatibility Ex/ "filter1,filter2" filter = filter.to_s.split(',')[0].to_s @@ -49,13 +51,15 @@ class EventFilter # rubocop: disable Metrics/CyclomaticComplexity # This method build specialized in-operator optimized queries based on different # filter parameters. All queries will benefit from the index covering the following columns: - # author_id target_type action id + # * author_id target_type action id + # * project_id target_type action id + # * group_id target_type action id # # More context: https://docs.gitlab.com/ee/development/database/efficient_in_operator_queries.html#the-inoperatoroptimization-module - def in_operator_query_builder_params(user_ids) + def in_operator_query_builder_params(array_data) case filter when ALL - in_operator_params(array_scope_ids: user_ids) + in_operator_params(array_data: array_data) when PUSH # Here we need to add an order hint column to force the correct index usage. # Without the order hint, the following conditions will use the `index_events_on_author_id_and_id` @@ -66,25 +70,25 @@ class EventFilter # to use the correct index: # > target_type IS NULL AND action = 5 AND author_id = X ORDER BY target_type DESC, id DESC in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.where(target_type: nil).pushed_action, order_hint_column: :target_type ) when MERGED in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.where(target_type: MergeRequest.to_s).merged_action ) when COMMENTS in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.commented_action, in_column: :target_type, in_values: [Note, *Note.descendants].map(&:name) # To make the query efficient we need to list all Note classes ) when TEAM in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.where(target_type: nil), order_hint_column: :target_type, in_column: :action, @@ -92,34 +96,34 @@ class EventFilter ) when ISSUE in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.where(target_type: Issue.name), in_column: :action, in_values: Event.actions.values_at(*Event::ISSUE_ACTIONS) ) when WIKI in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.for_wiki_page, in_column: :action, in_values: Event.actions.values_at(*Event::WIKI_ACTIONS) ) when DESIGNS in_operator_params( - array_scope_ids: user_ids, + array_data: array_data, scope: Event.for_design, in_column: :action, in_values: Event.actions.values_at(*Event::DESIGN_ACTIONS) ) else - in_operator_params(array_scope_ids: user_ids) + in_operator_params(array_data: array_data) end end # rubocop: enable Metrics/CyclomaticComplexity private - def in_operator_params(array_scope_ids:, scope: nil, in_column: nil, in_values: nil, order_hint_column: nil) + def in_operator_params(array_data:, scope: nil, in_column: nil, in_values: nil, order_hint_column: nil) base_scope = Event.all base_scope = base_scope.merge(scope) if scope @@ -146,8 +150,8 @@ class EventFilter base_scope = base_scope.reorder(order) array_params = in_operator_array_params( - array_scope_ids: array_scope_ids, scope: base_scope, + array_data: array_data, in_column: in_column, in_values: in_values ) @@ -161,22 +165,30 @@ class EventFilter # This method builds the array_ parameters # without in_column parameter: uses one IN filter: author_id # with in_column: two IN filters: author_id, (target_type OR action) - def in_operator_array_params(scope:, array_scope_ids:, in_column: nil, in_values: nil) + # @param array_data [Hash] Must contain the scope_ids, scope_model, mapping_column keys + def in_operator_array_params(scope:, array_data:, in_column: nil, in_values: nil) + array_scope_ids = array_data[:scope_ids] + array_scope_model = array_data[:scope_model] + array_mapping_column = array_data[:mapping_column] + + # Adding non-existent record to generate valid SQL if array_scope_ids is empty + array_scope_ids << 0 if array_scope_ids.empty? + if in_column - # Builds Carthesian product of the in_values and the array_scope_ids (in this case: user_ids). + # Builds Cartesian product of the in_values and the array_scope_ids (in this case: user_ids). # The process is described here: https://docs.gitlab.com/ee/development/database/efficient_in_operator_queries.html#multiple-in-queries # VALUES ((array_scope_ids[0], in_values[0]), (array_scope_ids[1], in_values[0]) ...) cartesian = array_scope_ids.product(in_values) - user_with_column_list = Arel::Nodes::ValuesList.new(cartesian) + column_list = Arel::Nodes::ValuesList.new(cartesian) as = "array_ids(id, #{Event.connection.quote_column_name(in_column)})" - from = Arel::Nodes::Grouping.new(user_with_column_list).as(as) + from = Arel::Nodes::Grouping.new(column_list).as(as) { - array_scope: User.select(:id, in_column).from(from), - array_mapping_scope: -> (author_id_expression, in_column_expression) do + array_scope: array_scope_model.select(:id, in_column).from(from), + array_mapping_scope: -> (primary_id_expression, in_column_expression) do Event .merge(scope) - .where(Event.arel_table[:author_id].eq(author_id_expression)) + .where(Event.arel_table[array_mapping_column].eq(primary_id_expression)) .where(Event.arel_table[in_column].eq(in_column_expression)) end } @@ -186,11 +198,11 @@ class EventFilter array_ids_list = Arel::Nodes::ValuesList.new(array_scope_ids.map { |id| [id] }) from = Arel::Nodes::Grouping.new(array_ids_list).as('array_ids(id)') { - array_scope: User.select(:id).from(from), - array_mapping_scope: -> (author_id_expression) do + array_scope: array_scope_model.select(:id).from(from), + array_mapping_scope: -> (primary_id_expression) do Event .merge(scope) - .where(Event.arel_table[:author_id].eq(author_id_expression)) + .where(Event.arel_table[array_mapping_column].eq(primary_id_expression)) end } end diff --git a/lib/feature.rb b/lib/feature.rb index 47fee23c7ea..b5a97ee8f9b 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -42,8 +42,10 @@ class Feature flipper.features.to_a end + RecursionError = Class.new(RuntimeError) + def get(key) - flipper.feature(key) + with_feature(key, &:itself) end def persisted_names @@ -65,34 +67,29 @@ class Feature persisted_names.include?(feature_name.to_s) end - # use `default_enabled: true` to default the flag to being `enabled` - # unless set explicitly. The default is `disabled` - # TODO: remove the `default_enabled:` and read it from the `definition_yaml` - # check: https://gitlab.com/gitlab-org/gitlab/-/issues/30228 - def enabled?(key, thing = nil, type: :development, default_enabled: false) + # The default state of feature flag is read from `YAML`: + # 1. If feature flag does not have YAML it will fallback to `default_enabled: false` + # in production environment, but raise exception in development or tests. + # 2. The `default_enabled_if_undefined:` is tech debt related to Gitaly flags + # and should not be used outside of Gitaly's `lib/feature/gitaly.rb` + def enabled?(key, thing = nil, type: :development, default_enabled_if_undefined: nil) if check_feature_flags_definition? if thing && !thing.respond_to?(:flipper_id) raise InvalidFeatureFlagError, "The thing '#{thing.class.name}' for feature flag '#{key}' needs to include `FeatureGate` or implement `flipper_id`" end - Feature::Definition.valid_usage!(key, type: type, default_enabled: default_enabled) + Feature::Definition.valid_usage!(key, type: type) end - # If `default_enabled: :yaml` we fetch the value from the YAML definition instead. - default_enabled = Feature::Definition.default_enabled?(key) if default_enabled == :yaml - - # During setup the database does not exist yet. So we haven't stored a value - # for the feature yet and return the default. - return default_enabled unless ApplicationRecord.database.exists? + default_enabled = Feature::Definition.default_enabled?(key, default_enabled_if_undefined: default_enabled_if_undefined) - feature = get(key) + feature_value = with_feature(key) do |feature| + feature_value = current_feature_value(feature, thing, default_enabled: default_enabled) + end - # If we're not default enabling the flag or the feature has been set, always evaluate. - # `persisted?` can potentially generate DB queries and also checks for inclusion - # in an array of feature names (177 at last count), possibly reducing performance by half. - # So we only perform the `persisted` check if `default_enabled: true` - feature_value = !default_enabled || Feature.persisted_name?(feature.name) ? feature.enabled?(thing) : true + # If not yielded, then either recursion is happening, or the database does not exist yet, so use default_enabled. + feature_value = default_enabled if feature_value.nil? # If we don't filter out this flag here we will enter an infinite loop log_feature_flag_state(key, feature_value) if log_feature_flag_states?(key) @@ -100,46 +97,46 @@ class Feature feature_value end - def disabled?(key, thing = nil, type: :development, default_enabled: false) + def disabled?(key, thing = nil, type: :development, default_enabled_if_undefined: nil) # we need to make different method calls to make it easy to mock / define expectations in test mode - thing.nil? ? !enabled?(key, type: type, default_enabled: default_enabled) : !enabled?(key, thing, type: type, default_enabled: default_enabled) + thing.nil? ? !enabled?(key, type: type, default_enabled_if_undefined: default_enabled_if_undefined) : !enabled?(key, thing, type: type, default_enabled_if_undefined: default_enabled_if_undefined) end def enable(key, thing = true) log(key: key, action: __method__, thing: thing) - get(key).enable(thing) + with_feature(key) { _1.enable(thing) } end def disable(key, thing = false) log(key: key, action: __method__, thing: thing) - get(key).disable(thing) + with_feature(key) { _1.disable(thing) } end def enable_percentage_of_time(key, percentage) log(key: key, action: __method__, percentage: percentage) - get(key).enable_percentage_of_time(percentage) + with_feature(key) { _1.enable_percentage_of_time(percentage) } end def disable_percentage_of_time(key) log(key: key, action: __method__) - get(key).disable_percentage_of_time + with_feature(key, &:disable_percentage_of_time) end def enable_percentage_of_actors(key, percentage) log(key: key, action: __method__, percentage: percentage) - get(key).enable_percentage_of_actors(percentage) + with_feature(key) { _1.enable_percentage_of_actors(percentage) } end def disable_percentage_of_actors(key) log(key: key, action: __method__) - get(key).disable_percentage_of_actors + with_feature(key, &:disable_percentage_of_actors) end def remove(key) return unless persisted_name?(key) log(key: key, action: __method__) - get(key).remove + with_feature(key, &:remove) end def reset @@ -181,6 +178,52 @@ class Feature private + # Evaluate if `default enabled: false` or the feature has been persisted. + # `persisted_name?` can potentially generate DB queries and also checks for inclusion + # in an array of feature names (177 at last count), possibly reducing performance by half. + # So we only perform the `persisted` check if `default_enabled: true` + def current_feature_value(feature, thing, default_enabled:) + return true if default_enabled && !Feature.persisted_name?(feature.name) + + feature.enabled?(thing) + end + + # NOTE: it is not safe to call `Flipper::Feature#enabled?` outside the block + def with_feature(key) + feature = unsafe_get(key) + yield feature if feature.present? + ensure + pop_recursion_stack + end + + def unsafe_get(key) + # During setup the database does not exist yet. So we haven't stored a value + # for the feature yet and return the default. + return unless ApplicationRecord.database.exists? + + flag_stack = ::Thread.current[:feature_flag_recursion_check] || [] + Thread.current[:feature_flag_recursion_check] = flag_stack + + # Prevent more than 10 levels of recursion. This limit was chosen as a fairly + # low limit while allowing some nesting of flag evaluation. We have not seen + # this limit hit in production. + if flag_stack.size > 10 + Gitlab::ErrorTracking.track_exception(RecursionError.new('deep recursion'), stack: flag_stack) + return + elsif flag_stack.include?(key) + Gitlab::ErrorTracking.track_exception(RecursionError.new('self recursion'), stack: flag_stack) + return + end + + flag_stack.push(key) + flipper.feature(key) + end + + def pop_recursion_stack + flag_stack = Thread.current[:feature_flag_recursion_check] + flag_stack.pop if flag_stack + end + def flipper if Gitlab::SafeRequestStore.active? Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance(memoize: true) diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index 61f7e395769..1551af730db 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -63,19 +63,13 @@ class Feature end end - def valid_usage!(type_in_code:, default_enabled_in_code:) + def valid_usage!(type_in_code:) unless Array(type).include?(type_in_code.to_s) # Raise exception in test and dev raise Feature::InvalidFeatureFlagError, "The `type:` of `#{key}` is not equal to config: " \ "#{type_in_code} vs #{type}. Ensure to use valid type in #{path} or ensure that you use " \ "a valid syntax: #{TYPES.dig(type, :example)}" end - - unless default_enabled_in_code == :yaml || default_enabled == default_enabled_in_code - # Raise exception in test and dev - raise Feature::InvalidFeatureFlagError, "The `default_enabled:` of `#{key}` is not equal to config: " \ - "#{default_enabled_in_code} vs #{default_enabled}. Ensure to update #{path}" - end end def to_h @@ -124,9 +118,9 @@ class Feature feature.force_log_state_changes? || feature.for_upcoming_milestone? end - def valid_usage!(key, type:, default_enabled:) + def valid_usage!(key, type:) if definition = get(key) - definition.valid_usage!(type_in_code: type, default_enabled_in_code: default_enabled) + definition.valid_usage!(type_in_code: type) elsif type_definition = self::TYPES[type] raise InvalidFeatureFlagError, "Missing feature definition for `#{key}`" unless type_definition[:optional] else @@ -134,9 +128,11 @@ class Feature end end - def default_enabled?(key) + def default_enabled?(key, default_enabled_if_undefined: nil) if definition = get(key) definition.default_enabled + elsif !default_enabled_if_undefined.nil? + default_enabled_if_undefined else Gitlab::ErrorTracking.track_and_raise_for_dev_exception( InvalidFeatureFlagError.new("The feature flag YAML definition for '#{key}' does not exist")) diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index a1f7dc0ee39..04ed78b8a51 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -8,7 +8,7 @@ class Feature def enabled?(feature_flag, project = nil) return false unless Feature::FlipperFeature.table_exists? - Feature.enabled?("#{PREFIX}#{feature_flag}", project) + Feature.enabled?("#{PREFIX}#{feature_flag}", project, type: :undefined, default_enabled_if_undefined: false) rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 2ce078b2f02..40f21fc4f50 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -28,8 +28,8 @@ class Feature }, ops: { description: "Long-lived feature flags that control operational aspects of GitLab's behavior", - optional: true, - rollout_issue: false, + optional: false, + rollout_issue: true, ee_only: false, default_enabled: false, example: <<-EOS @@ -37,6 +37,14 @@ class Feature push_frontend_feature_flag(:my_ops_flag, project, type: :ops) EOS }, + undefined: { + description: "Feature flags that are undefined in GitLab codebase (should not be used)", + optional: true, + rollout_issue: false, + ee_only: false, + default_enabled: false, + example: '' + }, experiment: { description: 'Short lived, used specifically to run A/B/n experiments.', optional: true, diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index 1b67b91e839..de34a0f5d47 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -4,8 +4,7 @@ module Gitlab module AlertManagement module Payload MONITORING_TOOLS = { - prometheus: 'Prometheus', - cilium: 'Cilium' + prometheus: 'Prometheus' }.freeze class << self @@ -48,5 +47,3 @@ module Gitlab end end end - -Gitlab::AlertManagement::Payload.prepend_mod_with('Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index af695c5cfa4..d0d8d68362e 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -107,7 +107,7 @@ module Gitlab def use_aggregated_backend? group.present? && # for now it's only available on the group-level aggregation.enabled && - Feature.enabled?(:use_vsa_aggregated_tables, group, default_enabled: :yaml) + Feature.enabled?(:use_vsa_aggregated_tables, group) end def aggregation_attributes diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index b10330914ca..6ef5a1e2cd8 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -19,7 +19,8 @@ module Gitlab :job_id, :pipeline_id, :related_class, - :feature_category + :feature_category, + :artifact_size ].freeze private_constant :KNOWN_KEYS @@ -32,7 +33,8 @@ module Gitlab Attribute.new(:remote_ip, String), Attribute.new(:job, ::Ci::Build), Attribute.new(:related_class, String), - Attribute.new(:feature_category, String) + Attribute.new(:feature_category, String), + Attribute.new(:artifact, ::Ci::JobArtifact) ].freeze def self.known_keys @@ -74,6 +76,8 @@ module Gitlab assign_attributes(args) end + # rubocop: disable Metrics/CyclomaticComplexity + # rubocop: disable Metrics/PerceivedComplexity def to_lazy_hash {}.tap do |hash| hash[:user] = -> { username } if include_user? @@ -86,8 +90,11 @@ module Gitlab hash[:feature_category] = feature_category if set_values.include?(:feature_category) hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) hash[:job_id] = -> { job&.id } if set_values.include?(:job) + hash[:artifact_size] = -> { artifact&.size } if set_values.include?(:artifact) end end + # rubocop: enable Metrics/CyclomaticComplexity + # rubocop: enable Metrics/PerceivedComplexity def use Labkit::Context.with_context(to_lazy_hash) { yield } diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 09775297def..41a6cbc2543 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -59,6 +59,8 @@ module Gitlab def throttled?(key, scope:, threshold: nil, users_allowlist: nil, peek: false) raise InvalidKeyError unless rate_limits[key] + ::Gitlab::Instrumentation::RateLimitingGates.track(key) + return false if scoped_user_in_allowlist?(scope, users_allowlist) threshold_value = threshold || threshold(key) diff --git a/lib/gitlab/audit/deploy_token_author.rb b/lib/gitlab/audit/deploy_token_author.rb new file mode 100644 index 00000000000..69b42034826 --- /dev/null +++ b/lib/gitlab/audit/deploy_token_author.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + class DeployTokenAuthor < Gitlab::Audit::NullAuthor + def initialize(name: nil) + super(id: -2, name: name) + end + + # Events that are authored by a deploy token, should be + # shown as authored by `Deploy Token` in the UI. + def name + @name || _('Deploy Token') + end + end + end +end diff --git a/lib/gitlab/audit/null_author.rb b/lib/gitlab/audit/null_author.rb index 80e0c4ddf58..08be6ae6d9f 100644 --- a/lib/gitlab/audit/null_author.rb +++ b/lib/gitlab/audit/null_author.rb @@ -13,8 +13,8 @@ module Gitlab # # @param [Integer] id # @param [String] name - # - # @return [Gitlab::Audit::UnauthenticatedAuthor, Gitlab::Audit::DeletedAuthor, Gitlab::Audit::CiRunnerTokenAuthor] + # rubocop: disable Layout/LineLength + # @return [Gitlab::Audit::UnauthenticatedAuthor, Gitlab::Audit::DeletedAuthor, Gitlab::Audit::CiRunnerTokenAuthor, Gitlab::Audit::DeployTokenAuthor] def self.for(id, audit_event) name = audit_event[:author_name] || audit_event.details[:author_name] @@ -22,6 +22,8 @@ module Gitlab Gitlab::Audit::CiRunnerTokenAuthor.new(audit_event) elsif id == -1 Gitlab::Audit::UnauthenticatedAuthor.new(name: name) + elsif id == -2 + Gitlab::Audit::DeployTokenAuthor.new(name: name) else Gitlab::Audit::DeletedAuthor.new(id: id, name: name) end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator.rb deleted file mode 100644 index c1433f05db2..00000000000 --- a/lib/gitlab/auth/otp/strategies/forti_authenticator.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Auth - module Otp - module Strategies - class FortiAuthenticator < Base - def validate(otp_code) - body = { username: user.username, - token_code: otp_code } - - response = Gitlab::HTTP.post( - auth_url, - headers: { 'Content-Type': 'application/json' }, - body: body.to_json, - basic_auth: api_credentials) - - # Successful authentication results in HTTP 200: OK - # https://docs.fortinet.com/document/fortiauthenticator/6.2.0/rest-api-solution-guide/704555/authentication-auth - response.ok? ? success : error_from_response(response) - rescue StandardError => ex - Gitlab::AppLogger.error(ex) - error(ex.message) - end - - private - - def auth_url - host = ::Gitlab.config.forti_authenticator.host - port = ::Gitlab.config.forti_authenticator.port - path = 'api/v1/auth/' - - "https://#{host}:#{port}/#{path}" - end - - def api_credentials - { username: ::Gitlab.config.forti_authenticator.username, - password: ::Gitlab.config.forti_authenticator.access_token } - end - end - end - end - end -end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb new file mode 100644 index 00000000000..9cf1b2247a7 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + module FortiAuthenticator + class ManualOtp < Base + def validate(otp_code) + @otp_code = otp_code + + response = Gitlab::HTTP.post( + auth_url, + headers: { 'Content-Type': 'application/json' }, + body: body.to_json, + basic_auth: api_credentials) + + # Successful authentication results in HTTP 200: OK + # Manual OTP - https://docs.fortinet.com/document/fortiauthenticator/6.2.0/rest-api-solution-guide/704555/authentication-auth + response.ok? ? success : error_from_response(response) + rescue StandardError => ex + Gitlab::AppLogger.error(ex) + error(ex.message) + end + + private + + def auth_url + host = ::Gitlab.config.forti_authenticator.host + port = ::Gitlab.config.forti_authenticator.port + path = 'api/v1/auth/' + + "https://#{host}:#{port}/#{path}" + end + + def body + { username: user.username, + token_code: @otp_code } + end + + def api_credentials + { username: ::Gitlab.config.forti_authenticator.username, + password: ::Gitlab.config.forti_authenticator.access_token } + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator/push_otp.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator/push_otp.rb new file mode 100644 index 00000000000..03cc648f7b0 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator/push_otp.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + module FortiAuthenticator + class PushOtp < Base + def validate + response = Gitlab::HTTP.post( + auth_url, + headers: { 'Content-Type': 'application/json' }, + body: body.to_json, + basic_auth: api_credentials) + + # Successful authentication results in HTTP 200: OK + # Push - https://docs.fortinet.com/document/fortiauthenticator/6.2.1/rest-api-solution-guide/943094/push-authentication-pushauth + response.ok? ? success : error_from_response(response) + rescue StandardError => ex + Gitlab::AppLogger.error(ex) + error(ex.message) + end + + private + + def auth_url + host = ::Gitlab.config.forti_authenticator.host + port = ::Gitlab.config.forti_authenticator.port + path = 'api/v1/pushauth/' + + "https://#{host}:#{port}/#{path}" + end + + def body + { username: user.username } + end + + def api_credentials + { username: ::Gitlab.config.forti_authenticator.username, + password: ::Gitlab.config.forti_authenticator.access_token } + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 3f13a264b0a..815130aeee2 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -5,6 +5,10 @@ module Gitlab module Saml class Config class << self + def enabled? + ::AuthHelper.saml_providers.any? + end + def options Gitlab::Auth::OAuth::Provider.config_for('saml') end diff --git a/lib/gitlab/auth/saml/identity_linker.rb b/lib/gitlab/auth/saml/identity_linker.rb index 93195c3189f..a44a9c2fca5 100644 --- a/lib/gitlab/auth/saml/identity_linker.rb +++ b/lib/gitlab/auth/saml/identity_linker.rb @@ -32,3 +32,5 @@ module Gitlab end end end + +Gitlab::Auth::Saml::IdentityLinker.prepend_mod diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 50112a51675..116c84c3759 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -50,3 +50,12 @@ Style/FrozenStringLiteralComment: Enabled: true Details: >- This removes the need for calling "freeze", reducing noise in the code. + +Migration/BackgroundMigrationBaseClass: + Enabled: true + Exclude: + - 'batching_strategies/**/*.rb' + - 'job_coordinator.rb' + - 'base_job.rb' + - 'batched_migration_job.rb' + - 'logger.rb' diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb deleted file mode 100644 index f6b36571c90..00000000000 --- a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill expire_at for a range of Ci::JobArtifact - class BackfillArtifactExpiryDate - include Gitlab::Utils::StrongMemoize - - SWITCH_DATE = Date.new(2020, 06, 22).freeze - OLD_ARTIFACT_AGE = 15.months - BATCH_SIZE = 1_000 - OLD_ARTIFACT_EXPIRY_OFFSET = 3.months - RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year - - # Ci::JobArtifact model - class Ci::JobArtifact < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'ci_job_artifacts' - - scope :without_expiry_date, -> { where(expire_at: nil) } - scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) } - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) } - scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) } - end - - def perform(start_id, end_id) - Ci::JobArtifact - .without_expiry_date.before_switch - .between(start_id, end_id) - .each_batch(of: BATCH_SIZE) do |batch| - batch.old.update_all(expire_at: old_artifact_expiry_date) - batch.recent.update_all(expire_at: recent_artifact_expiry_date) - end - end - - private - - def offset_date - strong_memoize(:offset_date) do - current_date = Time.current - target_date = Time.zone.local(current_date.year, current_date.month, 22, 0, 0, 0) - - current_date.day < 22 ? target_date : target_date.next_month - end - end - - def old_artifact_expiry_date - offset_date + OLD_ARTIFACT_EXPIRY_OFFSET - end - - def recent_artifact_expiry_date - offset_date + RECENT_ARTIFACT_EXPIRY_OFFSET - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb new file mode 100644 index 00000000000..b9151343d6a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill draft column on open merge requests based on regex parsing of + # their titles. + # + class BackfillDraftStatusOnMergeRequestsWithCorrectedRegex # rubocop:disable Migration/BackgroundMigrationBaseClass + # Migration only version of MergeRequest table + class MergeRequest < ::ApplicationRecord + include EachBatch + + CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" + + self.table_name = 'merge_requests' + + def self.eligible + where(state_id: 1) + .where(draft: false) + .where("title ~* ?", CORRECTED_REGEXP_STR) + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) + + eligible_mrs.each_slice(10) do |slice| + MergeRequest.where(id: slice).update_all(draft: true) + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillDraftStatusOnMergeRequestsWithCorrectedRegex', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb index 084c788c8cb..4c3af7be319 100644 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -3,34 +3,19 @@ module Gitlab module BackgroundMigration # Backfill group_features for an array of groups - class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BaseJob - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, batch_size) - pause_ms = 0 if pause_ms < 0 - - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| - batch_metrics.time_operation(:upsert_group_features) do - upsert_group_features(sub_batch, batch_size) - end - - sleep(pause_ms * 0.001) + class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform(batch_size) + each_sub_batch( + operation_name: :upsert_group_features, + batching_arguments: { order_hint: :type }, + batching_scope: ->(relation) { relation.where(type: 'Group') } + ) do |sub_batch| + upsert_group_features(sub_batch, batch_size) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - .where(type: 'Group') - end - def upsert_group_features(relation, batch_size) connection.execute( <<~SQL diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb new file mode 100644 index 00000000000..de52629522b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Enable SSL verification for CI integrations with known-good hostnames. + class BackfillIntegrationsEnableSslVerification + INTEGRATIONS = { + # This matches the logic in `Integrations::DroneCi#url_is_saas?` + # - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/drone_ci.rb#L122-127 + # - https://docs.drone.io/pipeline/environment/reference/drone-system-hostname/ + 'Integrations::DroneCi' => [ + :drone_url, + /\Acloud\.drone\.io\z/i.freeze + ], + # This matches the logic in `Integrations::Teamcity#url_is_saas?` + # - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/teamcity.rb#L117-122 + # - https://www.jetbrains.com/help/teamcity/cloud/migrate-from-teamcity-on-premises-to-teamcity-cloud.html#Migration+Process + 'Integrations::Teamcity' => [ + :teamcity_url, + /\A[^\.]+\.teamcity\.com\z/i.freeze + ] + + # Other CI integrations which don't seem to have a SaaS offering: + # - Atlassian Bamboo (the SaaS offering is Bitbucket Pipelines) + # - Jenkins (self-hosted only) + # - MockCi (development only) + }.freeze + + # Define the `Integration` model + class Integration < ::ApplicationRecord + include IgnorableColumns + + self.table_name = :integrations + self.inheritance_column = :_type_disabled + + ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22' + ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22' + ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22' + + scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) } + + attr_encrypted :properties, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + + # Handle assignment of props with symbol keys. + # To do this correctly, we need to call the method generated by attr_encrypted. + alias_method :attr_encrypted_props=, :properties= + private :attr_encrypted_props= + + def properties=(props) + self.attr_encrypted_props = props&.with_indifferent_access&.freeze + end + end + + def perform(start_id, stop_id) + integration_ids = Integration + .affected + .where(id: (start_id..stop_id)) + .pluck(:id) + + integration_ids.each do |id| + Integration.transaction do + integration = Integration.lock.find(id) + process_integration(integration) + end + end + + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def process_integration(integration) + url_field, known_hostnames = INTEGRATIONS.fetch(integration.type_new) + + url = integration.properties[url_field.to_s] if integration.properties.present? + return unless url.present? + + parsed_url = Addressable::URI.parse(url) + return unless parsed_url.scheme == 'https' && parsed_url.hostname =~ known_hostnames + + integration.properties = integration.properties.merge('enable_ssl_verification' => true) + + integration.save!(touch: false) + rescue Addressable::URI::InvalidURIError, ActiveRecord::RecordInvalid + # Don't change the configuration if the record is invalid, in this case + # they will just keep having SSL verification disabled. + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb index a234cebfce5..6f33472af7d 100644 --- a/lib/gitlab/background_migration/backfill_integrations_type_new.rb +++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb @@ -22,7 +22,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ApplicationRecord.connection end def process_sub_batch(sub_batch) diff --git a/lib/gitlab/background_migration/backfill_issue_search_data.rb b/lib/gitlab/background_migration/backfill_issue_search_data.rb index ec206cbfd41..e408fd0cda6 100644 --- a/lib/gitlab/background_migration/backfill_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_issue_search_data.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) - define_batchable_model(batch_table, connection: ActiveRecord::Base.connection).where(batch_column => start_id..stop_id).each_batch(of: sub_batch_size) do |sub_batch| + define_batchable_model(batch_table, connection: ApplicationRecord.connection).where(batch_column => start_id..stop_id).each_batch(of: sub_batch_size) do |sub_batch| update_search_data(sub_batch) sleep(pause_ms * 0.001) diff --git a/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb index 1ed147d67c7..5f3d830c48d 100644 --- a/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb +++ b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb @@ -26,7 +26,7 @@ module Gitlab private def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + define_batchable_model(source_table, connection: ApplicationRecord.connection) .joins('INNER JOIN namespaces ON members.source_id = namespaces.id') .where(source_key_column => start_id..stop_id) .where(type: 'GroupMember') diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb index fe3edd3322b..0585924cb7b 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb @@ -27,7 +27,7 @@ module Gitlab private def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + define_batchable_model(source_table, connection: ApplicationRecord.connection) .joins('inner join namespaces on routes.source_id = namespaces.id') .where(source_key_column => start_id..stop_id) .where(namespace_id: nil) diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb index f6c8fb060f8..0282531ae17 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -13,7 +13,7 @@ module Gitlab cleanup_gin_index('routes') batch_metrics.time_operation(:update_all) do - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{sub_batch.to_sql} ) @@ -48,7 +48,7 @@ module Gitlab end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + define_batchable_model(source_table, connection: ApplicationRecord.connection) .joins('INNER JOIN projects ON routes.source_id = projects.id') .where(source_key_column => start_id..stop_id) .where(namespace_id: nil) diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb index 79e7a2f2279..587de1bcb5a 100644 --- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb @@ -29,7 +29,7 @@ module Gitlab WHERE namespaces.id = calculated_ids.id AND namespaces.traversal_ids = '{}' SQL - ActiveRecord::Base.connection.execute(update_sql) + ApplicationRecord.connection.execute(update_sql) sleep PAUSE_SECONDS end diff --git a/lib/gitlab/background_migration/backfill_note_discussion_id.rb b/lib/gitlab/background_migration/backfill_note_discussion_id.rb new file mode 100644 index 00000000000..da2c31ebd11 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_note_discussion_id.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes notes with NULL discussion_ids due to a bug when importing from GitHub + # Bug was fixed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76517 + class BackfillNoteDiscussionId + SUB_BATCH_SIZE = 300 + + # Migration only version of notes model + class Note < ApplicationRecord + include EachBatch + + self.table_name = 'notes' + + # Based on https://gitlab.com/gitlab-org/gitlab/blob/117c14d0c79403e169cf52922b48f69d1dcf6a85/app/models/discussion.rb#L62-74 + def generate_discussion_id + Digest::SHA1.hexdigest( + [:discussion, noteable_type.try(:underscore), noteable_id || commit_id, SecureRandom.hex].join('-') + ) + end + end + + def perform(start_id, stop_id) + notes = Note.select(:id, :noteable_type, :noteable_id, :commit_id) + .where(discussion_id: nil, id: start_id..stop_id) + + notes.each_batch(of: SUB_BATCH_SIZE) do |relation| + update_discussion_ids(relation) + end + end + + private + + def update_discussion_ids(notes) + mapping = notes.each_with_object({}) do |note, hash| + hash[note] = { discussion_id: note.generate_discussion_id } + end + + Gitlab::Database::BulkUpdate.execute(%i(discussion_id), mapping) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_settings.rb b/lib/gitlab/background_migration/backfill_project_settings.rb new file mode 100644 index 00000000000..7ede8de7bd6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_settings.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill project settings for projects that do not yet have one. + class BackfillProjectSettings + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + insert_sql = <<~SQL + INSERT INTO project_settings (project_id, created_at, updated_at) + #{sub_batch.where(project_settings: { project_id: nil }) + .select('projects.id, NOW(), NOW()') + .to_sql} + ON CONFLICT (project_id) DO NOTHING + SQL + + connection.execute(insert_sql) + + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) + end + end + + private + + def connection + ApplicationRecord.connection + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(:projects, connection: connection) + .where(source_key_column => start_id..stop_id) + .joins("LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id") + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_topics_title.rb b/lib/gitlab/background_migration/backfill_topics_title.rb new file mode 100644 index 00000000000..19a1eff5b58 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_topics_title.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to backfill the topic title + class BackfillTopicsTitle + # Temporary AR model for topics + class Topic < ActiveRecord::Base + self.table_name = 'topics' + end + + def perform(start_id, end_id) + Topic.where(id: start_id..end_id).where(title: nil).update_all('title = name') + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb index 170af90805a..3bf6bf993dd 100644 --- a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb +++ b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb @@ -16,7 +16,7 @@ module Gitlab private def execute(sql) - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection @connection.execute(sql) end diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb index ab569e236fb..df6b1f083c3 100644 --- a/lib/gitlab/background_migration/backfill_user_namespace.rb +++ b/lib/gitlab/background_migration/backfill_user_namespace.rb @@ -25,7 +25,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ApplicationRecord.connection end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb new file mode 100644 index 00000000000..442eab0673e --- /dev/null +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Base class for batched background migrations. Subclasses should implement the `#perform` + # method as the entry point for the job's execution, which will be called with the migration + # arguments (if any). + class BatchedMigrationJob + include Gitlab::Database::DynamicModelHelpers + + def initialize(start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, connection:) + @start_id = start_id + @end_id = end_id + @batch_table = batch_table + @batch_column = batch_column + @sub_batch_size = sub_batch_size + @pause_ms = pause_ms + @connection = connection + end + + def perform(*job_arguments) + raise NotImplementedError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection + + def each_sub_batch(operation_name: :default, batching_arguments: {}, batching_scope: nil) + all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) + + parent_relation = parent_batch_relation(batching_scope) + + parent_relation.each_batch(**all_batching_arguments) do |relation| + batch_metrics.instrument_operation(operation_name) do + yield relation + end + + sleep([pause_ms, 0].max * 0.001) + end + end + + def parent_batch_relation(batching_scope) + parent_relation = define_batchable_model(batch_table, connection: connection) + .where(batch_column => start_id..end_id) + + return parent_relation unless batching_scope + + batching_scope.call(parent_relation) + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 137b4d4bc4e..826845935b8 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -13,50 +13,25 @@ module Gitlab # - We skip the NULL checks as they may result in not using an index scan # - The table that is migrated does _not_ need `id` as the primary key # We use the provided primary_key column to perform the update. - class CopyColumnUsingBackgroundMigrationJob < BaseJob - include Gitlab::Database::DynamicModelHelpers + class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob + def perform(copy_from, copy_to) + assignment_clauses = build_assignment_clauses(copy_from, copy_to) - # start_id - The start ID of the range of rows to update. - # end_id - The end ID of the range of rows to update. - # batch_table - The name of the table that contains the columns. - # batch_column - The name of the column we use to batch over the table. - # sub_batch_size - We don't want updates to take more than ~100ms - # This allows us to run multiple smaller batches during - # the minimum 2.minute interval that we can schedule jobs - # pause_ms - The number of milliseconds to sleep between each subbatch execution. - # copy_from - List of columns containing the data to copy. - # copy_to - List of columns to copy the data to. Order must match the order in `copy_from`. - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, copy_from, copy_to) - copy_from = Array.wrap(copy_from) - copy_to = Array.wrap(copy_to) - - raise ArgumentError, 'number of source and destination columns must match' unless copy_from.count == copy_to.count - - assignment_clauses = column_assignment_clauses(copy_from, copy_to) - - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all(assignment_clauses) - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) + each_sub_batch(operation_name: :update_all) do |relation| + relation.update_all(assignment_clauses) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection).where(source_key_column => start_id..stop_id) - end + def build_assignment_clauses(copy_from, copy_to) + copy_from = Array.wrap(copy_from) + copy_to = Array.wrap(copy_to) + + unless copy_from.count == copy_to.count + raise ArgumentError, 'number of source and destination columns must match' + end - def column_assignment_clauses(copy_from, copy_to) assignments = copy_from.zip(copy_to).map do |from_column, to_column| from_column = connection.quote_column_name(from_column) to_column = connection.quote_column_name(to_column) diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb index 5d41a46c8cd..4a3a12ab53d 100644 --- a/lib/gitlab/background_migration/delete_orphaned_deployments.rb +++ b/lib/gitlab/background_migration/delete_orphaned_deployments.rb @@ -15,7 +15,7 @@ module Gitlab end def orphaned_deployments - define_batchable_model('deployments', connection: ActiveRecord::Base.connection) + define_batchable_model('deployments', connection: ApplicationRecord.connection) .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb index 9a88eb8ea06..dad5da875ab 100644 --- a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb +++ b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb @@ -32,7 +32,7 @@ module Gitlab private def execute(sql) - ActiveRecord::Base + ApplicationRecord .connection .execute(sql) end diff --git a/lib/gitlab/background_migration/expire_o_auth_tokens.rb b/lib/gitlab/background_migration/expire_o_auth_tokens.rb new file mode 100644 index 00000000000..595e4ac9dc8 --- /dev/null +++ b/lib/gitlab/background_migration/expire_o_auth_tokens.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Add expiry to all OAuth access tokens + class ExpireOAuthTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :update_oauth_tokens, + batching_scope: ->(relation) { relation.where(expires_in: nil) } + ) do |sub_batch| + update_oauth_tokens(sub_batch) + end + end + + private + + def update_oauth_tokens(relation) + relation.update_all(expires_in: 7_200) + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb index defd9ea832b..3772430d0b7 100644 --- a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb +++ b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb @@ -21,7 +21,7 @@ module Gitlab backfill_project_namespaces_service.cleanup_gin_index('projects') project_ids.each_slice(SUB_BATCH_SIZE) do |ids| - ActiveRecord::Base.connection.execute(update_projects_name_and_path_sql(ids)) + ApplicationRecord.connection.execute(update_projects_name_and_path_sql(ids)) end backfill_project_namespaces_service.backfill_project_namespaces diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index 83c01afa432..c21f9c1d50f 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -14,7 +14,7 @@ module Gitlab private def create_missing!(from_id, to_id) - result = ActiveRecord::Base.connection.select_one(sql(from_id, to_id)) + result = ApplicationRecord.connection.select_one(sql(from_id, to_id)) return 0 unless result result['number_of_created_records'] diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index b8e4562b3bf..496ec0bd0a1 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -120,14 +120,14 @@ module Gitlab end def create_missing(from_id, to_id) - result = ActiveRecord::Base.connection.select_one(create_sql(from_id, to_id)) + result = ApplicationRecord.connection.select_one(create_sql(from_id, to_id)) return unless result logger.info(message: "#{self.class}: created missing services for #{result['number_of_created_records']} projects in id=#{from_id}...#{to_id}") end def update_inconsistent(from_id, to_id) - result = ActiveRecord::Base.connection.select_one(update_sql(from_id, to_id)) + result = ApplicationRecord.connection.select_one(update_sql(from_id, to_id)) return unless result logger.info(message: "#{self.class}: updated inconsistent services for #{result['number_of_updated_records']} projects in id=#{from_id}...#{to_id}") diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index acbb5f76ad8..c440db58b94 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -14,7 +14,7 @@ module Gitlab worker_class = worker_for_tracking_database[tracking_database] if worker_class.nil? - raise ArgumentError, "tracking_database must be one of [#{worker_for_tracking_database.keys.join(', ')}]" + raise ArgumentError, "The '#{tracking_database}' must be one of #{worker_for_tracking_database.keys.to_a}" end new(worker_class) diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb index ec4631d1e34..d7d24960a41 100644 --- a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb +++ b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, end_id) - define_batchable_model('integrations', connection: ::ActiveRecord::Base.connection) + define_batchable_model('integrations', connection: ApplicationRecord.connection) .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) .update_all(category: 'third_party_wiki') diff --git a/lib/gitlab/background_migration/migrate_stage_status.rb b/lib/gitlab/background_migration/migrate_stage_status.rb deleted file mode 100644 index 6a29a632577..00000000000 --- a/lib/gitlab/background_migration/migrate_stage_status.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Metrics/AbcSize -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateStageStatus - STATUSES = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze - - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - - scope :latest, -> { where(retried: [false, nil]) } - scope :created, -> { where(status: 'created') } - scope :running, -> { where(status: 'running') } - scope :pending, -> { where(status: 'pending') } - scope :success, -> { where(status: 'success') } - scope :failed, -> { where(status: 'failed') } - scope :canceled, -> { where(status: 'canceled') } - scope :skipped, -> { where(status: 'skipped') } - scope :manual, -> { where(status: 'manual') } - - scope :failed_but_allowed, -> do - where(allow_failure: true, status: [:failed, :canceled]) - end - - scope :exclude_ignored, -> do - where("allow_failure = ? OR status IN (?)", - false, %w[created pending running success skipped]) - end - - def self.status_sql - scope_relevant = latest.exclude_ignored - scope_warnings = latest.failed_but_allowed - - builds = scope_relevant.select('count(*)').to_sql - created = scope_relevant.created.select('count(*)').to_sql - success = scope_relevant.success.select('count(*)').to_sql - manual = scope_relevant.manual.select('count(*)').to_sql - pending = scope_relevant.pending.select('count(*)').to_sql - running = scope_relevant.running.select('count(*)').to_sql - skipped = scope_relevant.skipped.select('count(*)').to_sql - canceled = scope_relevant.canceled.select('count(*)').to_sql - warnings = scope_warnings.select('count(*) > 0').to_sql - - <<-SQL.strip_heredoc - (CASE - WHEN (#{builds}) = (#{skipped}) AND (#{warnings}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{skipped}) THEN #{STATUSES[:skipped]} - WHEN (#{builds}) = (#{success}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{created}) THEN #{STATUSES[:created]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) THEN #{STATUSES[:success]} - WHEN (#{builds}) = (#{success}) + (#{skipped}) + (#{canceled}) THEN #{STATUSES[:canceled]} - WHEN (#{builds}) = (#{created}) + (#{skipped}) + (#{pending}) THEN #{STATUSES[:pending]} - WHEN (#{running}) + (#{pending}) > 0 THEN #{STATUSES[:running]} - WHEN (#{manual}) > 0 THEN #{STATUSES[:manual]} - WHEN (#{created}) > 0 THEN #{STATUSES[:running]} - ELSE #{STATUSES[:failed]} - END) - SQL - end - end - - def perform(start_id, stop_id) - status_sql = Build - .where('ci_builds.commit_id = ci_stages.pipeline_id') - .where('ci_builds.stage = ci_stages.name') - .status_sql - - sql = <<-SQL - UPDATE ci_stages SET status = (#{status_sql}) - WHERE ci_stages.status IS NULL - AND ci_stages.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - SQL - - ActiveRecord::Base.connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index c01545e5dca..06422ed282f 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab private def process_batch(from_id, to_id) - ActiveRecord::Base.connection.execute(update_sql(from_id, to_id)) + ApplicationRecord.connection.execute(update_sql(from_id, to_id)) logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}") end diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb index 78e897d9ae1..36d4e649271 100644 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb @@ -26,7 +26,7 @@ module Gitlab private def connection - ActiveRecord::Base.connection + ::Ci::ApplicationRecord.connection end def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb index 9e102ea1517..a9611e9814c 100644 --- a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb +++ b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb @@ -33,7 +33,7 @@ module Gitlab private def connection - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection end def execute(sql) diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb index 769ca4be7f3..1f2b55004e4 100644 --- a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb +++ b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb @@ -15,7 +15,7 @@ module Gitlab def perform(start_id, stop_id) Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ActiveRecord::Base.connection.execute(<<~SQL) + ApplicationRecord.connection.execute(<<~SQL) WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) UPDATE topics SET non_private_projects_count = ( diff --git a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb index 1d96872d445..2495cb51364 100644 --- a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb +++ b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb @@ -15,7 +15,7 @@ module Gitlab def perform(start_id, stop_id) Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ActiveRecord::Base.connection.execute(<<~SQL) + ApplicationRecord.connection.execute(<<~SQL) WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) UPDATE topics SET total_projects_count = (SELECT COUNT(*) FROM project_topics WHERE topic_id = batched_relation.id) diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb index 7b6d4c1ff81..5e6475a3d1a 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_reads.rb @@ -26,7 +26,7 @@ module Gitlab end def connection - ActiveRecord::Base.connection + ApplicationRecord.connection end def insert_query(start_id, end_id) diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index c13dbd76630..2b27bad3497 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -58,7 +58,7 @@ module Gitlab index_names = ApplicationRecord.connection.select_values("select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%using gin%'") index_names.each do |index_name| - ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')") + ApplicationRecord.connection.execute("select gin_clean_pending_list('#{index_name}')") end end @@ -77,7 +77,7 @@ module Gitlab projects = IsolatedModels::Project.where(id: project_ids) .select("projects.id, projects.name, projects.path, projects.namespace_id, projects.visibility_level, shared_runners_enabled, '#{PROJECT_NAMESPACE_STI_NAME}', now(), now()") - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL INSERT INTO namespaces (tmp_project_id, name, path, parent_id, visibility_level, shared_runners_enabled, type, created_at, updated_at) #{projects.to_sql} ON CONFLICT DO NOTHING; @@ -89,7 +89,7 @@ module Gitlab .joins("INNER JOIN namespaces ON projects.id = namespaces.tmp_project_id") .select("namespaces.id, namespaces.tmp_project_id") - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL WITH cte(project_namespace_id, project_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{projects.to_sql} ) @@ -105,7 +105,7 @@ module Gitlab .joins("INNER JOIN namespaces n2 ON namespaces.parent_id = n2.id") .select("namespaces.id as project_namespace_id, n2.traversal_ids") - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL UPDATE namespaces SET traversal_ids = array_append(project_namespaces.traversal_ids, project_namespaces.project_namespace_id) FROM (#{namespaces.to_sql}) as project_namespaces(project_namespace_id, traversal_ids) diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index c1b8de1f6aa..db7afd59f4d 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -79,7 +79,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength def perform(start_id, end_id) - unless Feature.enabled?(:migrate_vulnerability_finding_uuids, default_enabled: true) + unless Feature.enabled?(:migrate_vulnerability_finding_uuids) return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id) end diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb index 323f109449b..4acef9029f9 100644 --- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb +++ b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb @@ -10,7 +10,7 @@ module Gitlab include Gitlab::Database::DynamicModelHelpers def perform(start_id, stop_id) - define_batchable_model('vulnerability_finding_links', connection: ActiveRecord::Base.connection) + define_batchable_model('vulnerability_finding_links', connection: ApplicationRecord.connection) .where(id: start_id..stop_id) .delete_all end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb index 80ca76ef37f..190e2fc22fb 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb @@ -5,24 +5,24 @@ module Gitlab # A job to nullify duplicate runners_token_encrypted values in projects table in batches class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include ::EachBatch + include EachBatch self.table_name = 'projects' - scope :base_query, -> do - where.not(runners_token_encrypted: nil) - end + scope :base_query, -> { where.not(runners_token_encrypted: nil) } end def perform(start_id, end_id) # Reset duplicate runner tokens that would prevent creating an unique index. + batch_records = Project.base_query.where(id: start_id..end_id) + duplicate_tokens = Project.base_query - .where(id: start_id..end_id) + .where(runners_token_encrypted: batch_records.select(:runners_token_encrypted).distinct) .group(:runners_token_encrypted) .having('COUNT(*) > 1') .pluck(:runners_token_encrypted) - Project.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? + batch_records.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? mark_job_as_succeeded(start_id, end_id) end @@ -30,7 +30,10 @@ module Gitlab private def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects', arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) end end end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb index d87ce6c88d3..b58eefa0ab3 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb @@ -5,24 +5,24 @@ module Gitlab # A job to nullify duplicate ci_runners_token values in projects table in batches class ResetDuplicateCiRunnersTokenValuesOnProjects class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include ::EachBatch + include EachBatch self.table_name = 'projects' - scope :base_query, -> do - where.not(runners_token: nil) - end + scope :base_query, -> { where.not(runners_token: nil) } end def perform(start_id, end_id) # Reset duplicate runner tokens that would prevent creating an unique index. + batch_records = Project.base_query.where(id: start_id..end_id) + duplicate_tokens = Project.base_query - .where(id: start_id..end_id) + .where(runners_token: batch_records.select(:runners_token).distinct) .group(:runners_token) .having('COUNT(*) > 1') .pluck(:runners_token) - Project.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? + batch_records.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? mark_job_as_succeeded(start_id, end_id) end @@ -30,7 +30,10 @@ module Gitlab private def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnerValuesTokensOnProjects', arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) end end end diff --git a/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports.rb b/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports.rb new file mode 100644 index 00000000000..83a7eb0b4cc --- /dev/null +++ b/lib/gitlab/background_migration/reset_too_many_tags_skipped_registry_imports.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to reset container_repositories that were skipped in the phase 2 registry + # migration due to too many tags. + class ResetTooManyTagsSkippedRegistryImports # rubocop:disable Migration/BackgroundMigrationBaseClass + class ContainerRepository < ::ApplicationRecord # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'container_repositories' + + scope :base_query, -> { where(migration_state: 'import_skipped', migration_skipped_reason: 2) } + end + + def perform(start_id, end_id) + ContainerRepository.base_query.where(id: start_id..end_id).each_batch(of: 100) do |sub_batch| + sub_batch.update_all( + migration_pre_import_started_at: nil, + migration_pre_import_done_at: nil, + migration_import_started_at: nil, + migration_import_done_at: nil, + migration_aborted_at: nil, + migration_skipped_at: nil, + migration_retries_count: 0, + migration_skipped_reason: nil, + migration_state: 'default', + migration_aborted_in_state: nil + ) + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb index f54bb8256d0..38932e52bb0 100644 --- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb +++ b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb @@ -28,7 +28,7 @@ module Gitlab end def connection - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection end def execute(sql) diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb index 24c9967b88e..69bb5cf6e6d 100644 --- a/lib/gitlab/background_migration/update_timelogs_project_id.rb +++ b/lib/gitlab/background_migration/update_timelogs_project_id.rb @@ -36,7 +36,7 @@ module Gitlab end def execute(sql) - @connection ||= ::ActiveRecord::Base.connection + @connection ||= ApplicationRecord.connection @connection.execute(sql) end end diff --git a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb index f5ba9e63333..10db9f5064a 100644 --- a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb +++ b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb @@ -5,7 +5,7 @@ module Gitlab module BackgroundMigration class UpdateUsersWhereTwoFactorAuthRequiredFromGroup # rubocop:disable Metrics/ClassLength def perform(start_id, stop_id) - ActiveRecord::Base.connection.execute <<~SQL + ApplicationRecord.connection.execute <<~SQL UPDATE users SET diff --git a/lib/gitlab/backtrace_cleaner.rb b/lib/gitlab/backtrace_cleaner.rb index caea05c720d..d2ca2057eb6 100644 --- a/lib/gitlab/backtrace_cleaner.rb +++ b/lib/gitlab/backtrace_cleaner.rb @@ -17,7 +17,6 @@ module Gitlab lib/gitlab/profiler.rb lib/gitlab/query_limiting/ lib/gitlab/request_context.rb - lib/gitlab/request_profiler/ lib/gitlab/sidekiq_logging/ lib/gitlab/sidekiq_middleware/ lib/gitlab/sidekiq_status/ diff --git a/lib/gitlab/chat.rb b/lib/gitlab/chat.rb index 23d4fb36b66..30e9989d270 100644 --- a/lib/gitlab/chat.rb +++ b/lib/gitlab/chat.rb @@ -4,7 +4,7 @@ module Gitlab module Chat # Returns `true` if Chatops is available for the current instance. def self.available? - ::Feature.enabled?(:chatops, default_enabled: true) + ::Feature.enabled?(:chatops) end end end diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 84c01cf4baf..2e469aabeb2 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -3,6 +3,8 @@ module Gitlab module Checks class ChangesAccess + include Gitlab::Utils::StrongMemoize + ATTRIBUTES = %i[user_access project protocol changes logger].freeze attr_reader(*ATTRIBUTES) @@ -33,29 +35,37 @@ module Gitlab # changes. This set may also contain commits which are not referenced by # any of the new revisions. def commits - allow_quarantine = true + strong_memoize(:commits) do + allow_quarantine = true + + newrevs = @changes.map do |change| + oldrev = change[:oldrev] + newrev = change[:newrev] - newrevs = @changes.map do |change| - oldrev = change[:oldrev] - newrev = change[:newrev] + next if blank_rev?(newrev) - next if blank_rev?(newrev) + # In case any of the old revisions is blank, then we cannot reliably + # detect which commits are new for a given change when enumerating + # objects via the object quarantine directory given that the client + # may have pushed too many commits, and we don't know when to + # terminate the walk. We thus fall back to using `git rev-list --not + # --all`, which is a lot less efficient but at least can only ever + # returns commits which really are new. + allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) - # In case any of the old revisions is blank, then we cannot reliably - # detect which commits are new for a given change when enumerating - # objects via the object quarantine directory given that the client - # may have pushed too many commits, and we don't know when to - # terminate the walk. We thus fall back to using `git rev-list --not - # --all`, which is a lot less efficient but at least can only ever - # returns commits which really are new. - allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) + newrev + end.compact - newrev - end.compact + next [] if newrevs.empty? - return [] if newrevs.empty? + # When filtering quarantined commits we can enable usage of the object + # quarantine no matter whether we have an `oldrev` or not. + if Feature.enabled?(:filter_quarantined_commits) + allow_quarantine = true + end - @commits ||= project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) + project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) + end end # All commits which have been newly introduced via the given revision. diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 84069a1249b..1d1d24c8fcc 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -9,7 +9,7 @@ module Gitlab def validate! # This feature flag is used for disabling integrity check on some envs # because these costy calculations may cause performance issues - return unless Feature.enabled?(:lfs_check, project, default_enabled: :yaml) + return unless Feature.enabled?(:lfs_check, project) return unless project.lfs_enabled? diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 2fd48dfbfe2..8e12801daee 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -35,7 +35,8 @@ module Gitlab end def commits - @commits ||= project.repository.new_commits(newrev) + @commits ||= project.repository.new_commits(newrev, + allow_quarantine: Feature.enabled?(:filter_quarantined_commits)) end protected diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index f12b4f2dbfb..18db4861dc9 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -23,13 +23,11 @@ module Gitlab::Ci MIN_MEDIUM_DEFAULT = 75 def initialize(badge) - @entity = badge.entity @status = badge.status - @key_text = badge.customization.dig(:key_text) - @key_width = badge.customization.dig(:key_width) @min_good = badge.customization.dig(:min_good) @min_acceptable = badge.customization.dig(:min_acceptable) @min_medium = badge.customization.dig(:min_medium) + super end def value_text diff --git a/lib/gitlab/ci/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index c39f96e4a34..417fff252a3 100644 --- a/lib/gitlab/ci/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -22,10 +22,8 @@ module Gitlab::Ci }.freeze def initialize(badge) - @entity = badge.entity @status = badge.status - @key_text = badge.customization.dig(:key_text) - @key_width = badge.customization.dig(:key_width) + super end def value_text diff --git a/lib/gitlab/ci/badge/release/template.rb b/lib/gitlab/ci/badge/release/template.rb index 65bff4371cf..354be6276fa 100644 --- a/lib/gitlab/ci/badge/release/template.rb +++ b/lib/gitlab/ci/badge/release/template.rb @@ -13,10 +13,8 @@ module Gitlab::Ci VALUE_WIDTH_DEFAULT = 54 def initialize(badge) - @entity = badge.entity @tag = badge.tag || "none" - @key_width = badge.customization.dig(:key_width) - @key_text = badge.customization.dig(:key_text) + super end def key_text diff --git a/lib/gitlab/ci/badge/template.rb b/lib/gitlab/ci/badge/template.rb index d514a8577bd..b185fadc3a2 100644 --- a/lib/gitlab/ci/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -12,7 +12,8 @@ module Gitlab::Ci def initialize(badge) @entity = badge.entity - @status = badge.status + @key_text = badge.customization.dig(:key_text) + @key_width = badge.customization.dig(:key_width) end def key_text diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 2c9524c89ff..15a4ff91c1b 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -26,11 +26,8 @@ module Gitlab @source_ref_path = pipeline&.source_ref_path @project = project - if use_config_variables? - pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) - end - @context = self.logger.instrument(:config_build_context) do + pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) end @@ -94,7 +91,8 @@ module Gitlab def metadata { - includes: @context.includes + includes: @context.includes, + merged_yaml: @config&.deep_stringify_keys&.to_yaml } end @@ -148,46 +146,15 @@ module Gitlab sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: build_variables(project: project, pipeline: pipeline), + variables: build_variables(pipeline: pipeline), logger: logger) end - def build_variables(project:, pipeline:) + def build_variables(pipeline:) logger.instrument(:config_build_variables) do - build_variables_without_instrumentation( - project: project, - pipeline: pipeline - ) - end - end - - def build_variables_without_instrumentation(project:, pipeline:) - if use_config_variables? - return pipeline.variables_builder.config_variables - end - - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless project - - # The order of the following lines is important as priority of CI variables is - # defined globally within GitLab. - # - # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence - variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) if pipeline - variables.concat(secret_variables(project: project, pipeline: pipeline)) - variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group - variables.concat(project.ci_variables_for(ref: source_ref_path)) - variables.concat(pipeline.variables) if pipeline - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule - end - end - - def secret_variables(project:, pipeline:) - if pipeline - pipeline.variables_builder.secret_instance_variables - else - Gitlab::Ci::Variables::Builder::Instance.new.secret_variables + pipeline + .variables_builder + .config_variables end end @@ -195,12 +162,6 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, @context.sentry_payload) end - def use_config_variables? - strong_memoize(:use_config_variables) do - ::Feature.enabled?(:ci_variables_builder_config_variables, @project, default_enabled: :yaml) - end - end - # Overridden in EE def rescue_errors RESCUE_ERRORS diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 2066e9be3b1..bc39abfe977 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -44,7 +44,7 @@ module Gitlab validates :action, type: String, - inclusion: { in: %w[start stop prepare], message: 'should be start, stop or prepare' }, + inclusion: { in: %w[start stop prepare verify access], message: 'should be start, stop, prepare, verify, or access' }, allow_nil: true validates :deployment_tier, diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 06c81fd65dd..7513936a18a 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze - ALLOWED_KEYS = %i[tags script type image services start_in artifacts + ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout release].freeze @@ -55,11 +55,6 @@ module Gitlab description: 'Commands that will be executed in this job.', inherit: false - entry :type, Entry::Stage, - description: 'Deprecated: stage this job will be executed into.', - inherit: false, - deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0' } - entry :after_script, Entry::Commands, description: 'Commands that will be executed when finishing job.', inherit: true @@ -135,19 +130,6 @@ module Gitlab true end - def compose!(deps = nil) - super do - # The type keyword will be removed in 15.0: - # https://gitlab.com/gitlab-org/gitlab/-/issues/346823 - if type_defined? && !stage_defined? - @entries[:stage] = @entries[:type] - log_and_warn_deprecated_entry(@entries[:type]) - end - - @entries.delete(:type) - end - end - def delayed? self.when == 'delayed' end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index f8fce1abc06..d5d204bb995 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -15,7 +15,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif - dotenv cobertura terraform accessibility cluster_applications + dotenv terraform accessibility requirements coverage_fuzzing api_fuzzing cluster_image_scanning coverage_report].freeze @@ -45,14 +45,10 @@ module Gitlab validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true validates :dotenv, array_of_strings_or_string: true - validates :cobertura, array_of_strings_or_string: true validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true - validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end - - validates :config, mutually_exclusive_keys: [:coverage_report, :cobertura] end def value diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 7b58ef0b8ab..ff11c757dfa 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable ALLOWED_KEYS = %i[default include before_script image services - after_script variables stages types cache workflow].freeze + after_script variables stages cache workflow].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -57,11 +57,6 @@ module Gitlab description: 'Configuration of stages for this pipeline.', reserved: true - entry :types, Entry::Stages, - description: 'Deprecated: stages for this pipeline.', - reserved: true, - deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0' } - entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', reserved: true @@ -100,7 +95,6 @@ module Gitlab def compose!(_deps = nil) super(self) do - compose_deprecated_entries! compose_jobs! end end @@ -118,21 +112,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def compose_deprecated_entries! - ## - # Deprecated `:types` key workaround - if types are defined and - # stages are not defined we use types definition as stages. - # This keyword will be removed in 15.0: - # https://gitlab.com/gitlab-org/gitlab/-/issues/346823 - # - if types_defined? - @entries[:stages] = @entries[:types] unless stages_defined? - log_and_warn_deprecated_entry(@entries[:types]) - end - - @entries.delete(:types) - end - def filter_jobs! return unless @config.is_a?(Hash) diff --git a/lib/gitlab/ci/config/extendable/entry.rb b/lib/gitlab/ci/config/extendable/entry.rb index 0001a259281..169d329fe02 100644 --- a/lib/gitlab/ci/config/extendable/entry.rb +++ b/lib/gitlab/ci/config/extendable/entry.rb @@ -99,7 +99,7 @@ module Gitlab end def circular_dependency? - ancestors.include?(key) + ancestors.include?(key) # rubocop:disable Performance/AncestorsInclude end def unknown_extensions diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index ee9cc1552fe..feb2cbb19ad 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -23,6 +23,8 @@ module Gitlab super.merge( type: :local, location: masked_location, + blob: masked_blob, + raw: masked_raw, extra: {} ) end @@ -57,6 +59,24 @@ module Gitlab variables: context.variables } end + + def masked_blob + strong_memoize(:masked_blob) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_blob_url(context.project, ::File.join(context.sha, location)) + ) + end + end + + def masked_raw + return unless context.project + + strong_memoize(:masked_raw) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_raw_url(context.project, ::File.join(context.sha, location)) + ) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 3d4436530a8..09c36a1bcb6 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -31,6 +31,8 @@ module Gitlab super.merge( type: :file, location: masked_location, + blob: masked_blob, + raw: masked_raw, extra: { project: masked_project_name, ref: masked_ref_name } ) end @@ -69,6 +71,8 @@ module Gitlab end def sha + return unless project + strong_memoize(:sha) do project.commit(ref_name).try(:sha) end @@ -96,6 +100,26 @@ module Gitlab context.mask_variables_from(ref_name) end end + + def masked_blob + return unless project + + strong_memoize(:masked_blob) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_blob_url(project, ::File.join(sha, location)) + ) + end + end + + def masked_raw + return unless project + + strong_memoize(:masked_raw) do + context.mask_variables_from( + Gitlab::Routing.url_helpers.project_raw_url(project, ::File.join(sha, location)) + ) + end + end end end end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index e7b007b4d8d..7d3a2362246 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -22,6 +22,8 @@ module Gitlab super.merge( type: :remote, location: masked_location, + blob: nil, + raw: masked_location, extra: {} ) end diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 9469f09ce13..58b81b259cb 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -9,6 +9,7 @@ module Gitlab attr_reader :location SUFFIX = '.gitlab-ci.yml' + HOST = 'https://gitlab.com/gitlab-org/gitlab/-/raw/master' def initialize(params, context) @location = params[:template] @@ -24,6 +25,8 @@ module Gitlab super.merge( type: :template, location: masked_location, + blob: nil, + raw: masked_raw, extra: {} ) end @@ -51,6 +54,14 @@ module Gitlab def fetch_template_content Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content end + + def masked_raw + strong_memoize(:masked_raw) do + context.mask_variables_from( + "#{HOST}/#{Gitlab::Template::GitlabCiYmlTemplate::BASE_DIR}/#{location}" + ) + end + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 3fb86b8b3e8..97774bc5e13 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -73,7 +73,7 @@ module Gitlab def key @key ||= begin - key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project) Gitlab::CurrentSettings.ci_jwt_signing_key else Rails.application.secrets.openid_connect_signing_key diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 5591ed62436..51743a1f273 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -4,18 +4,23 @@ module Gitlab module Ci class Lint class Result - attr_reader :jobs, :merged_yaml, :errors, :warnings + attr_reader :jobs, :merged_yaml, :errors, :warnings, :includes - def initialize(jobs:, merged_yaml:, errors:, warnings:) + def initialize(jobs:, merged_yaml:, errors:, warnings:, includes:) @jobs = jobs @merged_yaml = merged_yaml @errors = errors @warnings = warnings + @includes = includes end def valid? @errors.empty? end + + def status + valid? ? :valid : :invalid + end end LOG_MAX_DURATION_THRESHOLD = 2.seconds @@ -44,9 +49,10 @@ module Gitlab Result.new( jobs: dry_run_convert_to_jobs(pipeline.stages), - merged_yaml: pipeline.merged_yaml, + merged_yaml: pipeline.config_metadata.try(:[], :merged_yaml), errors: pipeline.error_messages.map(&:content), - warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content) + warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content), + includes: pipeline.config_metadata.try(:[], :includes) ) end @@ -57,9 +63,10 @@ module Gitlab Result.new( jobs: static_validation_convert_to_jobs(result), - merged_yaml: result.merged_yaml, + merged_yaml: result.config_metadata[:merged_yaml], errors: result.errors, - warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord + warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT), # rubocop: disable CodeReuse/ActiveRecord + includes: result.config_metadata[:includes] ) ensure logger.commit(pipeline: ::Ci::Pipeline.new, caller: self.class.name) diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index cef029bd749..4460843545e 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -6,39 +6,28 @@ module Gitlab module Security module Validators class SchemaValidator - # https://docs.gitlab.com/ee/update/deprecations.html#147 SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2] }.freeze - # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags - PREVIOUS_RELEASES = %w[10.0.0 12.0.0 12.1.0 13.0.0 - 13.1.0 2.3.0-rc1 2.3.0-rc1 2.3.1-rc1 2.3.2-rc1 2.3.3-rc1 - 2.4.0-rc1 3.0.0 3.0.0-rc1 3.1.0-rc1 4.0.0-rc1 5.0.0-rc1 - 5.0.1-rc1 6.0.0-rc1 6.0.1-rc1 6.1.0-rc1 7.0.0-rc1 7.0.1-rc1 - 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze - - # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 - KNOWN_VERSIONS_TO_REMOVE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze - - VERSIONS_TO_REMOVE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_REMOVE).freeze + VERSIONS_TO_REMOVE_IN_16_0 = [].freeze DEPRECATED_VERSIONS = { - cluster_image_scanning: VERSIONS_TO_REMOVE_IN_15_0, - container_scanning: VERSIONS_TO_REMOVE_IN_15_0, - coverage_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, - dast: VERSIONS_TO_REMOVE_IN_15_0, - api_fuzzing: VERSIONS_TO_REMOVE_IN_15_0, - dependency_scanning: VERSIONS_TO_REMOVE_IN_15_0, - sast: VERSIONS_TO_REMOVE_IN_15_0, - secret_detection: VERSIONS_TO_REMOVE_IN_15_0 + cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0, + container_scanning: VERSIONS_TO_REMOVE_IN_16_0, + coverage_fuzzing: VERSIONS_TO_REMOVE_IN_16_0, + dast: VERSIONS_TO_REMOVE_IN_16_0, + api_fuzzing: VERSIONS_TO_REMOVE_IN_16_0, + dependency_scanning: VERSIONS_TO_REMOVE_IN_16_0, + sast: VERSIONS_TO_REMOVE_IN_16_0, + secret_detection: VERSIONS_TO_REMOVE_IN_16_0 }.freeze class Schema @@ -165,7 +154,6 @@ module Gitlab def handle_unsupported_report_version(treat_as:) if report_version.nil? message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}" - add_message_as(level: treat_as, message: message) else message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}" end diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..31840a7e914 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json @@ -0,0 +1,977 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json new file mode 100644 index 00000000000..c70628a0949 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json @@ -0,0 +1,911 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..fbc7b4ea733 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json new file mode 100644 index 00000000000..3c9db0546b1 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json @@ -0,0 +1,1287 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + }, + "discovered_at": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", + "examples": [ + "2020-01-28T03:26:02.956" + ] + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json new file mode 100644 index 00000000000..c7459216faf --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json @@ -0,0 +1,968 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.2" + }, + "required": [ + "dependency_files", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json index a7159be0190..20818792652 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/sast-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json @@ -325,7 +325,7 @@ } }, "self": { - "version": "14.0.0" + "version": "14.1.2" }, "required": [ "version", @@ -384,6 +384,68 @@ } } }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, "scanner": { "type": "object", "description": "Object defining the scanner used to perform the scan.", @@ -426,8 +488,8 @@ ] }, "vendor": { - "type": "object", "description": "The vendor/maintainer of the scanner.", + "type": "object", "required": [ "name" ], @@ -484,7 +546,7 @@ "description": "Array of vulnerability objects.", "items": { "type": "object", - "description": "Describes the vulnerability.", + "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ "category", "cve", @@ -629,6 +691,107 @@ "details": { "$ref": "#/definitions/named_list/properties/items" }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, "location": { "type": "object", "description": "Identifies the vulnerability's location.", diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json index 462e23a151c..12386d2c1d4 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/secret-detection-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json @@ -325,7 +325,7 @@ } }, "self": { - "version": "14.0.0" + "version": "14.1.2" }, "required": [ "version", @@ -384,6 +384,68 @@ } } }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, "scanner": { "type": "object", "description": "Object defining the scanner used to perform the scan.", @@ -426,8 +488,8 @@ ] }, "vendor": { - "type": "object", "description": "The vendor/maintainer of the scanner.", + "type": "object", "required": [ "name" ], @@ -484,7 +546,7 @@ "description": "Array of vulnerability objects.", "items": { "type": "object", - "description": "Describes the vulnerability.", + "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ "category", "cve", @@ -629,6 +691,107 @@ "details": { "$ref": "#/definitions/named_list/properties/items" }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, "location": { "required": [ "commit" diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/dependency-scanning-report-format.json deleted file mode 120000 index 11e0a6846fb..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/dependency-scanning-report-format.json +++ /dev/null @@ -1 +0,0 @@ -14.0.0/dependency-scanning-report-format.json
\ No newline at end of file diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index c466b8b36d0..0a6f6fd740c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -96,7 +96,7 @@ module Gitlab step = step_class.name.underscore.parameterize(separator: '_') logger.observe("pipeline_step_#{step}_duration_s", duration) - if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops) metrics.pipeline_creation_step_duration_histogram .observe({ step: step_class.name }, duration.seconds) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 64d1b001e3c..5548fca320f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -35,7 +35,7 @@ module Gitlab error(result.errors.first, config_error: true) end - @pipeline.merged_yaml = result.merged_yaml + @pipeline.config_metadata = result.config_metadata rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, diff --git a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb index cb02f09f819..17ebf56985b 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb @@ -54,15 +54,13 @@ module Gitlab def throttle_enabled? ::Feature.enabled?( :ci_throttle_pipelines_creation, - project, - default_enabled: :yaml) + project) end def dry_run? ::Feature.enabled?( :ci_throttle_pipelines_creation_dry_run, - project, - default_enabled: :yaml) + project) end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 4d65b914d8d..6efb3a4f16a 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -11,8 +11,15 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return false unless regexp + if ::Feature.enabled?(:ci_fix_rules_if_comparison_with_regexp_variable) + # All variables are evaluated as strings, even if they are regexp strings. + # So, we need to convert them to regexp objects. + regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) + end + regexp.scan(text.to_s).present? end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 29c5aa5d753..a72e5dbc822 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -11,8 +11,15 @@ module Gitlab def evaluate(variables = {}) text = @left.evaluate(variables) regexp = @right.evaluate(variables) + return true unless regexp + if ::Feature.enabled?(:ci_fix_rules_if_comparison_with_regexp_variable) + # All variables are evaluated as strings, even if they are regexp strings. + # So, we need to convert them to regexp objects. + regexp = Lexeme::Pattern.build_and_evaluate(regexp, variables) + end + regexp.scan(text.to_s).empty? end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index c7106f3ec39..cd4106b16bb 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -35,6 +35,18 @@ module Gitlab def self.build(string) new(string) end + + def self.build_and_evaluate(data, variables = {}) + return data if data.is_a?(Gitlab::UntrustedRegexp) + + begin + new_pattern = build(data) + rescue Lexer::SyntaxError + return data + end + + new_pattern.evaluate(variables) + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index e90e764bcd9..798cea34db6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -8,10 +8,6 @@ module Gitlab class String < Lexeme::Value PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze - def initialize(value) - super(value) - end - def evaluate(variables = {}) @value.to_s end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index 6d872fee39d..fa82bbe3275 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -10,6 +10,8 @@ module Gitlab :value end + attr_reader :value + def initialize(value) @value = value end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index ee6c3898592..44d905faced 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -110,7 +110,7 @@ module Gitlab def enabled? strong_memoize(:enabled) do - ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops, default_enabled: :yaml) + ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops) end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index b5e48f210ad..33b9ac9b641 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -46,7 +46,7 @@ module Gitlab name = :gitlab_ci_active_jobs comment = 'Total amount of active jobs' labels = { plan: nil } - buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000] + buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000, 15_000, 20_000, 30_000, 40_000] ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 54fb1d19ea8..7d8303214a5 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -74,7 +74,7 @@ module Gitlab end def observe_queue_depth(queue, size) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) raise ArgumentError, "unknown queue depth label: #{queue}" @@ -84,7 +84,7 @@ module Gitlab end def observe_queue_size(size_proc, runner_type) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) size = size_proc.call.to_f self.class.queue_size_total.observe({ runner_type: runner_type }, size) @@ -96,7 +96,7 @@ module Gitlab result = yield - return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time @@ -121,7 +121,7 @@ module Gitlab end def self.observe_active_runners(runners_proc) - return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics) queue_active_runners_total.observe({}, runners_proc.call.to_f) end @@ -250,7 +250,7 @@ module Gitlab end def running_jobs_relation(job) - if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data) ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) else job.project.builds.running.where(runner: ::Ci::Runner.instance_type) diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 365864d3317..68c911d3dbb 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -25,7 +25,7 @@ module Gitlab amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64" }, install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh", - runner_executable: "sudo gitlab-runner" + runner_executable: "gitlab-runner" }, windows: { human_readable_name: "Windows", diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index baf041fc358..46b41ed3c6c 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -5,12 +5,19 @@ module Gitlab class RunnerUpgradeCheck include Singleton + STATUSES = { + invalid: 'Runner version is not valid.', + not_available: 'Upgrade is not available for the runner.', + available: 'Upgrade is available for the runner.', + recommended: 'Upgrade is available and recommended for the runner.' + }.freeze + def initialize reset! end def check_runner_upgrade_status(runner_version) - return :unknown unless runner_version + return :invalid unless runner_version releases = RunnerReleases.instance.releases parsed_runner_version = runner_version.is_a?(::Gitlab::VersionInfo) ? runner_version : ::Gitlab::VersionInfo.parse(runner_version) diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index eaa87157716..263fd9d1052 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -16,7 +16,7 @@ module Gitlab def details_path return unless can?(user, :read_pipeline, downstream_pipeline) - if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project, default_enabled: :yaml) + if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project) project_job_path(subject.project, subject) else project_pipeline_path(downstream_project, downstream_pipeline) diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 8020ffee36f..fddcc1492a8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -178,7 +178,6 @@ include: - template: Jobs/Helm-2to3.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml - template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml - - template: Security/Cluster-Image-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index f3d2e293c86..8c63019d743 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index f3d2e293c86..8c63019d743 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.14.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 0cc5090f85e..04b1c4a6f73 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index d41182ec9be..5c56594da78 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -12,10 +12,9 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" - DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" - DS_MAJOR_VERSION: 2 + DS_MAJOR_VERSION: 3 dependency_scanning: stage: test @@ -52,6 +51,18 @@ dependency_scanning: paths: - "**/cyclonedx-*.json" +.gemnasium-shared-rule: + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + gemnasium-dependency_scanning: extends: - .ds-analyzer @@ -66,17 +77,20 @@ gemnasium-dependency_scanning: when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ - exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - - '{composer.lock,*/composer.lock,*/*/composer.lock}' - - '{gems.locked,*/gems.locked,*/*/gems.locked}' - - '{go.sum,*/go.sum,*/*/go.sum}' - - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' - - '{conan.lock,*/conan.lock,*/*/conan.lock}' + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + +.gemnasium-maven-shared-rule: + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' gemnasium-maven-dependency_scanning: extends: @@ -84,9 +98,6 @@ gemnasium-maven-dependency_scanning: - .cyclone-dx-reports variables: DS_ANALYZER_NAME: "gemnasium-maven" - # Stop reporting Gradle as "maven". - # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 - DS_REPORT_PACKAGE_MANAGER_MAVEN_WHEN_JAVA: "false" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -94,12 +105,22 @@ gemnasium-maven-dependency_scanning: when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ - exists: - - '{build.gradle,*/build.gradle,*/*/build.gradle}' - - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - - '{build.sbt,*/build.sbt,*/*/build.sbt}' - - '{pom.xml,*/pom.xml,*/*/pom.xml}' + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + +.gemnasium-python-shared-rule: + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' gemnasium-python-dependency_scanning: extends: @@ -107,9 +128,6 @@ gemnasium-python-dependency_scanning: - .cyclone-dx-reports variables: DS_ANALYZER_NAME: "gemnasium-python" - # Stop reporting Pipenv and Setuptools as "pip". - # See https://gitlab.com/gitlab-org/gitlab/-/issues/338252 - DS_REPORT_PACKAGE_MANAGER_PIP_WHEN_PYTHON: "false" rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never @@ -117,46 +135,39 @@ gemnasium-python-dependency_scanning: when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ - exists: - - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' - - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' - - '{Pipfile,*/Pipfile,*/*/Pipfile}' - - '{requires.txt,*/requires.txt,*/*/requires.txt}' - - '{setup.py,*/setup.py,*/*/setup.py}' - # Support passing of $PIP_REQUIREMENTS_FILE - # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ && $PIP_REQUIREMENTS_FILE bundler-audit-dependency_scanning: extends: .ds-analyzer - variables: - DS_ANALYZER_NAME: "bundler-audit" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/347491" + - exit 1 rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ - exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - when: never retire-js-dependency_scanning: extends: .ds-analyzer - variables: - DS_ANALYZER_NAME: "retire.js" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/289830" + - exit 1 rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/ - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && - $DS_DEFAULT_ANALYZERS =~ /retire.js/ - exists: - - '{package.json,*/package.json,*/*/package.json}' + - when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 89eb91c981f..c29b5b74bfc 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 78f28b59aa5..d09bb53a5b1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.23.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.25.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index 89a44eddefd..f7945b46a59 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. - LICENSE_MANAGEMENT_VERSION: 3 + LICENSE_MANAGEMENT_VERSION: 4 license_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml new file mode 100644 index 00000000000..b6358eb0831 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml @@ -0,0 +1,41 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/iac_scanning/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/iac_scanning/index.html + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" + + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + +iac-sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + variables: + SEARCH_MAX_DEPTH: 4 + allow_failure: true + script: + - /analyzer run + +kics-iac-sast: + extends: iac-sast + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ + when: never + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 488e7ec72fd..b6358eb0831 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -31,7 +31,7 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 1 + SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 7415fa3104c..be41553450c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -55,7 +55,7 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -92,7 +92,7 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -113,7 +113,7 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -144,7 +144,7 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" mobsf-android-sast: @@ -178,7 +178,7 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -194,7 +194,7 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -210,7 +210,7 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -226,22 +226,14 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: '2' + SAST_ANALYZER_IMAGE_TAG: '3' SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ when: never - # This rule shim will be removed in %15.0, - # See https://gitlab.com/gitlab-org/gitlab/-/issues/350935 - - if: $CI_COMMIT_BRANCH && $CI_SERVER_VERSION_MAJOR == '14' - exists: - - '**/*.csproj' - - '**/*.vbproj' - if: $CI_COMMIT_BRANCH - variables: - SAST_ANALYZER_IMAGE_TAG: '3' exists: - '**/*.csproj' - '**/*.vbproj' @@ -252,7 +244,7 @@ semgrep-sast: name: "$SAST_ANALYZER_IMAGE" variables: SEARCH_MAX_DEPTH: 20 - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - if: $SAST_DISABLED @@ -275,7 +267,7 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -291,7 +283,7 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f8e6e152ab9 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -0,0 +1,407 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ +# +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SAST_IMAGE_SUFFIX: "" + + SAST_EXCLUDED_ANALYZERS: "" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" + SCAN_KUBERNETES_MANIFESTS: "false" + +sast: + stage: test + artifacts: + reports: + sast: gl-sast-report.json + rules: + - when: never + variables: + SEARCH_MAX_DEPTH: 4 + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + +.sast-analyzer: + extends: sast + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +bandit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.py' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.py' + +brakeman-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.rb' + - '**/Gemfile' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.rb' + - '**/Gemfile' + +eslint-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 2 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.html' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + +flawfinder-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.c++' + - '**/*.cp' + - '**/*.cxx' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.c' + - '**/*.cc' + - '**/*.cpp' + - '**/*.c++' + - '**/*.cp' + - '**/*.cxx' + +kubesec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_MERGE_REQUEST_IID && + $SCAN_KUBERNETES_MANIFESTS == 'true' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH && + $SCAN_KUBERNETES_MANIFESTS == 'true' + +gosec-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.go' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.go' + +.mobsf-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + +mobsf-android-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_MERGE_REQUEST_IID && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.apk' + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .mobsf-sast + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_MERGE_REQUEST_IID && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.ipa' + - '**/*.xcodeproj/*' + +nodejs-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/package.json' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/package.json' + +phpcs-security-audit-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.php' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.php' + +pmd-apex-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.cls' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.cls' + +security-code-scan-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.csproj' + - '**/*.vbproj' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.csproj' + - '**/*.vbproj' + +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SERACH_MAX_DEPTH: 20 + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - '**/*.c' + - '**/*.go' + - '**/*.java' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' + - '**/*.c' + - '**/*.go' + - '**/*.java' + +sobelow-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - 'mix.exs' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - 'mix.exs' + +spotbugs-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never + - if: $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never + - if: $SAST_DISABLED + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + exists: + - '**/*.groovy' + - '**/*.java' + - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 6aacd082fd7..3f18237a525 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECRET_DETECTION_IMAGE_SUFFIX: "" - SECRETS_ANALYZER_VERSION: "3" + SECRETS_ANALYZER_VERSION: "4" SECRET_DETECTION_EXCLUDED_PATHS: "" .secret-analyzer: @@ -31,37 +31,4 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH script: - - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - # Historic scan - - if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ]; then echo "Running Secret Detection Historic Scan"; /analyzer run; exit; fi - # Default branch scan - - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi - # Push event - - | - if [ "$CI_COMMIT_BEFORE_SHA" == "0000000000000000000000000000000000000000" ]; - then - # first commit on a new branch - echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt - git fetch --depth=2 origin $CI_COMMIT_REF_NAME - else - # determine commit range so that we can fetch the appropriate depth - # check the exit code to determine if we need to limit the commit_list.txt to CI_COMMIT_SHA. - if ! git log --pretty=format:"%H" ${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt; - then - echo "unable to determine commit range, limiting to ${CI_COMMIT_SHA}" - echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt - else - # append newline to to list since `git log` does not end with a - # newline, this is to keep the log messages consistent - echo >> ${CI_COMMIT_SHA}_commit_list.txt - fi - - # we need to extend the git fetch depth to the number of commits + 1 for the following reasons: - # to include the parent commit of the base commit in this MR/Push event. This is needed because - # `git diff -p` needs something to compare changes in that commit against - git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) origin $CI_COMMIT_REF_NAME - fi - echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt))) commits for a push event" - export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt - /analyzer run - - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml new file mode 100644 index 00000000000..e81e06d1a1d --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -0,0 +1,36 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" + SECRET_DETECTION_IMAGE_SUFFIX: "" + SECRETS_ANALYZER_VERSION: "4" + SECRET_DETECTION_EXCLUDED_PATHS: "" + +.secret-analyzer: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION$SECRET_DETECTION_IMAGE_SUFFIX" + services: [] + allow_failure: true + variables: + GIT_DEPTH: "50" + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + +secret_detection: + extends: .secret-analyzer + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_MERGE_REQUEST_IID # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. + script: + - /analyzer run diff --git a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml index 67c69115948..64a063388b2 100644 --- a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml @@ -6,7 +6,7 @@ # Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template has three jobs: # - `command`: Run MATLAB scripts, functions, and statements. # - `test`: Run tests authored using the MATLAB unit testing framework or Simulink Test. -# - `test_artifacts_job`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. +# - `test_artifacts`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. # # You can copy and paste one or more jobs in this template into your `.gitlab-ci.yml` file. # You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. @@ -17,20 +17,20 @@ # - The jobs in this template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported # in MATLAB R2019a and later. -# The `command` runs MATLAB scripts, functions, and statements. To use the job in your pipeline, -# substitute `command` with the code you want to run. +# The `command` job runs MATLAB scripts, functions, and statements. To use the job in your pipeline, +# substitute `mycommand` with the code you want to run. # command: - script: matlab -batch command + script: matlab -batch mycommand -# If the value of `command` is the name of a MATLAB script or function, do not specify the file extension. -# For example, to run a script named `myscript.m` in the root of your repository, specify the `command` like this: +# If the value of `mycommand` is the name of a MATLAB script or function, do not specify the file extension. +# For example, to run a script named `myscript.m` in the root of your repository, specify `mycommand` like this: # # "myscript" # # If you specify more than one script, function, or statement, use a comma or semicolon to separate them. # For example, to run `myscript.m` in a folder named `myfolder` located in the root of the repository, -# you can specify the `command` like this: +# you can specify `mycommand` like this: # # "addpath('myfolder'), myscript" # @@ -41,7 +41,7 @@ command: # [1] https://www.mathworks.com/help/matlab/ref/assert.html # [2] https://www.mathworks.com/help/matlab/ref/error.html -# The `test` runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function +# The `test` job runs the MATLAB and Simulink tests in your project. It calls the [`runtests`][3] function # to run the tests and then the [`assertSuccess`][4] method to fail the job if any of the tests fail. # test: @@ -55,12 +55,12 @@ test: # [4] https://www.mathworks.com/help/matlab/ref/matlab.unittest.testresult.assertsuccess.html # [5] https://www.mathworks.com/help/matlab/projects.html -# The `test_artifacts_job` runs your tests and additionally generates test and coverage artifacts. +# The `test_artifacts` job runs your tests and additionally generates test and coverage artifacts. # It uses the plugin classes in the [`matlab.unittest.plugins`][6] package to generate a JUnit test results -# report and a Cobertura code coverage report. Like the `run_tests` job, this job runs all the tests in your +# report and a Cobertura code coverage report. Like the `test` job, this job runs all the tests in your # project and fails the build if any of the tests fail. # -test_artifacts_job: +test_artifacts: script: | matlab -batch " import matlab.unittest.TestRunner @@ -84,11 +84,13 @@ test_artifacts_job: artifacts: reports: junit: "./artifacts/results.xml" - cobertura: "./artifacts/cobertura.xml" + coverage_report: + coverage_format: cobertura + path: "./artifacts/cobertura.xml" paths: - "./artifacts" -# You can modify the contents of the `test_artifacts_job` depending on your goals. For more +# You can modify the contents of the `test_artifacts` job depending on your goals. For more # information on how to customize the test runner and generate various test and coverage artifacts, # see [Generate Artifacts Using MATLAB Unit Test Plugins][7]. # diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml deleted file mode 100644 index ca63e942130..00000000000 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ /dev/null @@ -1,32 +0,0 @@ -################################################################################ -# WARNING -################################################################################ -# -# This template is DEPRECATED and scheduled for removal in GitLab 15.0 -# See https://gitlab.com/gitlab-org/gitlab/-/issues/333610 for more context. -# -# To get started with a Cluster Management Project, we instead recommend -# using the updated project template: -# -# - Documentation: https://docs.gitlab.com/ee/user/clusters/management_project_template.html -# - Source code: https://gitlab.com/gitlab-org/project-templates/cluster-management/ -# -################################################################################ - -apply: - stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.43.1" - environment: - name: production - variables: - TILLER_NAMESPACE: gitlab-managed-apps - GITLAB_MANAGED_APPS_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/config.yaml - script: - - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - artifacts: - when: on_failure - paths: - - tiller.log diff --git a/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml b/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml index 6dbd0ce9561..60707dd0df0 100644 --- a/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Qualys-IaC-Security.gitlab-ci.yml @@ -3,7 +3,8 @@ # # This template shows how to use Qualys IaC Scan with a GitLab CI/CD pipeline. # Qualys and GitLab users can use this to scan their IaC templates for misconfigurations. -# Documentation about this integration: https://www.qualys.com/documentation/qualys-iac-gitlab-integration.pdf +# The IaC templates are uploaded to Qualys Platform for scanning, which returns the results to GitLab for reporting. +# Documentation about this integration: https://www.qualys.com/docs/qualys-iac-security-integration-gitlab.pdf # # This template should not need editing to work in your project. # It is not designed to be included in an existing CI/CD configuration with the "include:" keyword. diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 33c0928db6f..44f959468a8 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,7 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set --local deployment true # Install dependencies into ./vendor/ruby - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index aff8b6cb7fa..2fd5b409f5e 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -3,19 +3,36 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: API-Fuzzing.gitlab-ci.yml +# +# You also need to add a `fuzz` stage to your `stages:` configuration. A sample configuration for API Fuzzing: +# +# stages: +# - build +# - test +# - deploy +# - fuzz + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ # -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# Configure API Fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: - FUZZAPI_VERSION: "1" + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" - FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} + # + FUZZAPI_VERSION: "2" + FUZZAPI_IMAGE_SUFFIX: "" + FUZZAPI_IMAGE: api-security apifuzzer_fuzz: stage: fuzz - image: $FUZZAPI_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX allow_failure: true rules: - if: $API_FUZZING_DISABLED @@ -23,6 +40,10 @@ apifuzzer_fuzz: - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + FUZZAPI_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-fuzz-api diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index b6e811aa84f..450969fcdab 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -26,9 +26,9 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # - FUZZAPI_VERSION: "1" + FUZZAPI_VERSION: "2" FUZZAPI_IMAGE_SUFFIX: "" - FUZZAPI_IMAGE: api-fuzzing + FUZZAPI_IMAGE: api-security apifuzzer_fuzz: stage: fuzz diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml deleted file mode 100644 index 6b861510eef..00000000000 --- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Use this template to enable cluster image scanning in your project. -# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` -# keyword. -# The template should work without modifications but you can customize the template settings if -# needed: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#customize-the-container-scanning-settings -# -# Requirements: -# - A `test` stage to be present in the pipeline. -# - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities. -# -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables - -variables: - CIS_ANALYZER_IMAGE: registry.gitlab.com/security-products/cluster-image-scanning:0 - -cluster_image_scanning: - image: "$CIS_ANALYZER_IMAGE" - stage: test - allow_failure: true - artifacts: - reports: - cluster_image_scanning: gl-cluster-image-scanning-report.json - paths: [gl-cluster-image-scanning-report.json] - dependencies: [] - script: - - /analyzer run - rules: - - if: $CLUSTER_IMAGE_SCANNING_DISABLED - when: never - - if: '($KUBECONFIG == null || $KUBECONFIG == "") && ($CIS_KUBECONFIG == null || $CIS_KUBECONFIG == "")' - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcluster_image_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 66db311f897..bec269e2933 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:4 + CS_ANALYZER_IMAGE: registry.gitlab.com/security-products/container-scanning:5 container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" @@ -47,10 +47,8 @@ container_scanning: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && $CI_GITLAB_FIPS_MODE == "true" && $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ variables: CS_IMAGE_SUFFIX: -fips - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index d82f9f06f8d..893098d33c4 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -26,12 +26,13 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # - DAST_API_VERSION: "1" - DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + DAST_API_VERSION: "2" + DAST_API_IMAGE_SUFFIX: "" + DAST_API_IMAGE: api-security dast_api: stage: dast - image: $DAST_API_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true rules: - if: $DAST_API_DISABLED @@ -39,6 +40,10 @@ dast_api: - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DAST_API_IMAGE_SUFFIX: "-fips" - if: $CI_COMMIT_BRANCH script: - /peach/analyzer-dast-api @@ -50,3 +55,5 @@ dast_api: - gl-*.log reports: dast: gl-dast-api-report.json + +# end diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index b491b3e3c0c..3acc3b06031 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.latest.gitlab-ci.yml # To use this template, add the following to your .gitlab-ci.yml file: # @@ -26,9 +26,9 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # - DAST_API_VERSION: "1" + DAST_API_VERSION: "2" DAST_API_IMAGE_SUFFIX: "" - DAST_API_IMAGE: api-fuzzing + DAST_API_IMAGE: api-security dast_api: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 998425aa141..c71a1b1873a 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 2 + DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index e8e7fe62e70..3bc44fe5e1b 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 2 + DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index c755211ec11..e5ac5099546 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 2 + DAST_VERSION: 3 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml new file mode 100644 index 00000000000..2207d4ec17a --- /dev/null +++ b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml @@ -0,0 +1,2 @@ +include: + template: Jobs/SAST-IaC.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index a6fd070ec34..b34bfe2a53c 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -18,8 +18,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, - bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, gemnasium, gemnasium-maven, gemnasium-python, license-finder, dast, dast-runner-validation, api-fuzzing @@ -68,6 +67,8 @@ variables: bandit: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -75,6 +76,8 @@ bandit: brakeman: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -91,6 +94,8 @@ gosec: spotbugs: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -98,6 +103,8 @@ spotbugs: flawfinder: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -105,6 +112,8 @@ flawfinder: phpcs-security-audit: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -121,6 +130,8 @@ security-code-scan: nodejs-scan: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -128,6 +139,8 @@ nodejs-scan: eslint: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -140,10 +153,12 @@ secrets: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ variables: - SECURE_BINARIES_ANALYZER_VERSION: "3" + SECURE_BINARIES_ANALYZER_VERSION: "4" semgrep: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -151,6 +166,8 @@ semgrep: sobelow: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -158,6 +175,8 @@ sobelow: pmd-apex: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -165,6 +184,8 @@ pmd-apex: kubesec: extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && @@ -174,20 +195,6 @@ kubesec: # Dependency Scanning jobs # -bundler-audit: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/ - -retire.js: - extends: .download_images - only: - variables: - - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/ - gemnasium: extends: .download_images only: diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml deleted file mode 100644 index 55648437191..00000000000 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ /dev/null @@ -1,35 +0,0 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml - -# GitLab Serverless template - -image: alpine:latest - -stages: - - build - - test - - deploy - -.serverless:build:image: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: build - script: /usr/bin/gitlabktl app build - -.serverless:deploy:image: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: deploy - environment: development - script: /usr/bin/gitlabktl app deploy - -.serverless:build:functions: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: build - script: /usr/bin/gitlabktl serverless build - -.serverless:deploy:functions: - image: registry.gitlab.com/gitlab-org/gitlabktl:latest - stage: deploy - environment: development - script: /usr/bin/gitlabktl serverless deploy diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 1a857ef3eb3..56151a6bcdf 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,27 +1,32 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml include: - - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml stages: - - init - validate + - test - build - deploy -init: - extends: .init +fmt: + extends: .terraform:fmt + needs: [] validate: - extends: .validate + extends: .terraform:validate + needs: [] build: - extends: .build + extends: .terraform:build deploy: - extends: .deploy + extends: .terraform:deploy dependencies: - build + environment: + name: $TF_STATE_NAME diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 12c987a8d37..019b970bc30 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -5,7 +5,7 @@ include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml - - template: Jobs/SAST-IaC.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml stages: - validate diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 84a962e1541..49bdd4b7713 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Terraform/Base.latest +# Terraform/Base # # The purpose of this template is to provide flexibility to the user so # they are able to only include the jobs that they find interesting. @@ -7,10 +7,9 @@ # create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs # # There is a more opinionated template which we suggest the users to abide, -# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml - +# which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.0.3 + name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.1.9 variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project @@ -21,43 +20,46 @@ cache: paths: - ${TF_ROOT}/.terraform/ -.init: &init - stage: init +.terraform:fmt: &terraform_fmt + stage: validate script: - cd "${TF_ROOT}" - - gitlab-terraform init + - gitlab-terraform fmt + allow_failure: true -.validate: &validate +.terraform:validate: &terraform_validate stage: validate script: - cd "${TF_ROOT}" - gitlab-terraform validate -.build: &build +.terraform:build: &terraform_build stage: build script: - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json + resource_group: ${TF_STATE_NAME} artifacts: paths: - ${TF_ROOT}/plan.cache reports: terraform: ${TF_ROOT}/plan.json -.deploy: &deploy +.terraform:deploy: &terraform_deploy stage: deploy script: - cd "${TF_ROOT}" - gitlab-terraform apply - when: manual - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + resource_group: ${TF_STATE_NAME} + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual -.destroy: &destroy +.terraform:destroy: &terraform_destroy stage: cleanup script: - cd "${TF_ROOT}" - gitlab-terraform destroy + resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index a0ec07e61e1..9ba009a5bca 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -24,20 +24,20 @@ cache: .terraform:fmt: &terraform_fmt stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform fmt allow_failure: true .terraform:validate: &terraform_validate stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform validate .terraform:build: &terraform_build stage: build script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} @@ -50,7 +50,7 @@ cache: .terraform:deploy: &terraform_deploy stage: deploy script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: @@ -60,7 +60,7 @@ cache: .terraform:destroy: &terraform_destroy stage: cleanup script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform destroy resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 5ea2bc07ffa..2b5e86f4066 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: a11y: stage: accessibility - image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 + image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.2.3 script: - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index 09fce67db2d..b8d284532bd 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml # This is a simple example illustrating how to build and test .NET Core project # with GitLab Continuous Integration / Continuous Delivery. diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 7d08f0230fc..e93bd75a9fa 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -74,11 +74,11 @@ module Gitlab end def exist? - archived_trace_exist? || live_trace_exist? + archived? || live_trace_exist? end - def archived_trace_exist? - archived? + def archived? + trace_artifact&.stored? end def live_trace_exist? @@ -218,12 +218,6 @@ module Gitlab end end - def archived? - # TODO check checksum to ensure archive completed successfully - # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 - trace_artifact&.archived_trace_exists? - end - def destroy_any_orphan_trace_data! return unless trace_artifact @@ -312,7 +306,7 @@ module Gitlab end def consistent_archived_trace?(build) - ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false) + ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project) end def being_watched_cache_key diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index bcb1fe83ea2..a452cb197ae 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -52,7 +52,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(job).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.token).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index f14279dca2d..576fb509d47 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -103,10 +103,6 @@ module Gitlab }.compact }.compact end - def merged_yaml - @ci_config&.to_hash&.deep_stringify_keys&.to_yaml - end - def variables_with_data @ci_config.variables_with_data end @@ -127,6 +123,10 @@ module Gitlab jobs.dig(job_name, :stage) end + def config_metadata + @ci_config&.metadata || {} + end + private def variables diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index e0caabb0ec6..01c534c15a0 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -170,6 +170,11 @@ module Gitlab Constants::COLOR_NAME_TO_HEX[color.downcase] || new(color) end + # Generate a hex color based on hex-encoded value + def self.color_for(value) + Color.new("##{Digest::SHA256.hexdigest(value.to_s)[0..5]}") + end + def to_s @value.to_s end diff --git a/lib/gitlab/config/entry/validator.rb b/lib/gitlab/config/entry/validator.rb index e5efd4a7b0a..297645a65c1 100644 --- a/lib/gitlab/config/entry/validator.rb +++ b/lib/gitlab/config/entry/validator.rb @@ -7,10 +7,6 @@ module Gitlab include ActiveModel::Validations include Entry::Validators - def initialize(entry) - super(entry) - end - def messages errors.full_messages.map do |error| "#{location} #{error}".downcase diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index f3a3818f010..0559c85647d 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -41,7 +41,7 @@ module Gitlab end def too_big? - return false unless Feature.enabled?(:ci_yaml_limit_size, default_enabled: true) + return false unless Feature.enabled?(:ci_yaml_limit_size) !deep_size.valid? end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 22a4ba8ac7a..521dec110a8 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -61,7 +61,9 @@ module Gitlab end def initialize(csp_directives) - @csp_directives = HashWithIndifferentAccess.new(csp_directives) + @merged_csp_directives = + HashWithIndifferentAccess.new(csp_directives) + .reverse_merge(::Gitlab::ContentSecurityPolicy::ConfigLoader.default_directives) end def load(policy) @@ -77,8 +79,9 @@ module Gitlab private def arguments_for(directive) - arguments = @csp_directives[directive.to_s] - + # In order to disable a directive, the user can explicitly + # set a falsy value like nil, false or empty string + arguments = @merged_csp_directives[directive] return unless arguments.present? && arguments.is_a?(String) arguments.strip.split(' ').map(&:strip) diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb index 2b1529bdc1a..83ff61bbef2 100644 --- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb +++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb @@ -21,7 +21,7 @@ module Gitlab end def unit - _('per day') + _('/day') end def links diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/data_builder/issuable.rb index add9e880475..9a0b964915c 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/data_builder/issuable.rb @@ -1,11 +1,15 @@ # frozen_string_literal: true module Gitlab - module HookData - class IssuableBuilder < BaseBuilder + module DataBuilder + class Issuable CHANGES_KEYS = %i[previous current].freeze - alias_method :issuable, :object + attr_reader :issuable + + def initialize(issuable) + @issuable = issuable + end def build(user: nil, changes: {}) hook_data = { diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 1895f0fab32..677b4485288 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -49,7 +49,7 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze - PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym + PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym # rubocop:disable Database/MultipleDatabases def self.database_base_models @database_base_models ||= { @@ -94,21 +94,6 @@ module Gitlab Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) end - def self.main_database?(name) - # The database is `main` if it is a first entry in `database.yml` - # Rails internally names them `primary` to avoid confusion - # with broad `primary` usage we use `main` instead - # - # TODO: The explicit `== 'main'` is needed in a transition period till - # the `database.yml` is not migrated into `main:` syntax - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65243 - ActiveRecord::Base.configurations.primary?(name.to_s) || name.to_s == 'main' - end - - def self.ci_database?(name) - name.to_s == CI_DATABASE_NAME - end - class PgUser < ApplicationRecord self.table_name = 'pg_user' self.primary_key = :usename diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb index 58c4a214077..c8fdf8281cd 100644 --- a/lib/gitlab/database/background_migration/batch_optimizer.rb +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -41,7 +41,7 @@ module Gitlab end def optimize! - return unless Feature.enabled?(:optimize_batched_migrations, type: :ops, default_enabled: :yaml) + return unless Feature.enabled?(:optimize_batched_migrations, type: :ops) if multiplier = batch_size_multiplier max_batch = migration.max_batch_size || MAX_BATCH_SIZE diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d94bf060d05..a90cae7aea2 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -28,6 +28,8 @@ module Gitlab # on_hold_until is a temporary runtime status which puts execution "on hold" scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') } + scope :created_after, ->(time) { where('created_at > ?', time) } + scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index ec68f401ca2..5f4b2be3da8 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -39,7 +39,40 @@ module Gitlab end def execute_batch(tracking_record) - job_instance = migration_instance_for(tracking_record.migration_job_class) + job_instance = execute_job(tracking_record) + + if job_instance.respond_to?(:batch_metrics) + tracking_record.metrics = job_instance.batch_metrics + end + end + + def execute_job(tracking_record) + job_class = tracking_record.migration_job_class + + if job_class < Gitlab::BackgroundMigration::BatchedMigrationJob + execute_batched_migration_job(job_class, tracking_record) + else + execute_legacy_job(job_class, tracking_record) + end + end + + def execute_batched_migration_job(job_class, tracking_record) + job_instance = job_class.new( + start_id: tracking_record.min_value, + end_id: tracking_record.max_value, + batch_table: tracking_record.migration_table_name, + batch_column: tracking_record.migration_column_name, + sub_batch_size: tracking_record.sub_batch_size, + pause_ms: tracking_record.pause_ms, + connection: connection) + + job_instance.perform(*tracking_record.migration_job_arguments) + + job_instance + end + + def execute_legacy_job(job_class, tracking_record) + job_instance = job_class.new job_instance.perform( tracking_record.min_value, @@ -50,17 +83,7 @@ module Gitlab tracking_record.pause_ms, *tracking_record.migration_job_arguments) - if job_instance.respond_to?(:batch_metrics) - tracking_record.metrics = job_instance.batch_metrics - end - end - - def migration_instance_for(job_class) - if job_class < Gitlab::BackgroundMigration::BaseJob - job_class.new(connection: connection) - else - job_class.new - end + job_instance end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index ae0ea919b62..036ce7d7631 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -217,7 +217,6 @@ geo_event_log: :gitlab_main geo_events: :gitlab_main geo_hashed_storage_attachments_events: :gitlab_main geo_hashed_storage_migrated_events: :gitlab_main -geo_job_artifact_deleted_events: :gitlab_main geo_lfs_object_deleted_events: :gitlab_main geo_node_namespace_links: :gitlab_main geo_nodes: :gitlab_main @@ -327,6 +326,7 @@ namespace_aggregation_schedules: :gitlab_main namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main +namespace_ci_cd_settings: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main namespaces_sync_events: :gitlab_main @@ -348,6 +348,7 @@ operations_strategies: :gitlab_main operations_strategies_user_lists: :gitlab_main operations_user_lists: :gitlab_main packages_build_infos: :gitlab_main +packages_cleanup_policies: :gitlab_main packages_composer_cache_files: :gitlab_main packages_composer_metadata: :gitlab_main packages_conan_file_metadata: :gitlab_main @@ -388,6 +389,7 @@ plan_limits: :gitlab_main plans: :gitlab_main pool_repositories: :gitlab_main postgres_async_indexes: :gitlab_shared +postgres_autovacuum_activity: :gitlab_shared postgres_foreign_keys: :gitlab_shared postgres_index_bloat_estimates: :gitlab_shared postgres_indexes: :gitlab_shared diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 3f03d9e2c12..0ddc745ebae 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -90,7 +90,7 @@ module Gitlab return false unless ::Gitlab::SafeRequestStore.active? ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do - ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model, default_enabled: :yaml) + ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model) end end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 1e27bcfc55d..191ebe18b8a 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -255,6 +255,7 @@ module Gitlab # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching, # and caching for connections pools for each "connection", so we # leverage that. + # rubocop:disable Database/MultipleDatabases def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( @configuration.primary_connection_specification_name, @@ -262,6 +263,7 @@ module Gitlab shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) end + # rubocop:enable Database/MultipleDatabases def wal_diff(location1, location2) read_write do |connection| diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index dc695a74a4b..038af570dbc 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -37,18 +37,19 @@ module Gitlab class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase include LockRetriesConcern include Gitlab::Database::MigrationHelpers::V2 - end - - class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase - include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema # When running migrations, the `db:migrate` switches connection of # ActiveRecord::Base depending where the migration runs. # This helper class is provided to avoid confusion using `ActiveRecord::Base` class MigrationRecord < ActiveRecord::Base + self.abstract_class = true # Prevent STI behavior end end + class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d016dea224b..0453b81d67d 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -3,6 +3,7 @@ module Gitlab module Database module MigrationHelpers + include Migrations::ReestablishedConnectionStack include Migrations::BackgroundMigrationHelpers include Migrations::BatchedBackgroundMigrationHelpers include DynamicModelHelpers @@ -943,7 +944,7 @@ module Gitlab execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end - def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:) + def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) migration = Gitlab::Database::BackgroundMigration::BatchedMigration .for_configuration(job_class_name, table_name, column_name, job_arguments).first @@ -954,14 +955,18 @@ module Gitlab job_arguments: job_arguments } - if migration.nil? - Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" - elsif !migration.finished? + return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? + + return if migration.finished? + + finalize_batched_background_migration(job_class_name: job_class_name, table_name: table_name, column_name: column_name, job_arguments: job_arguments) if finalize + + unless migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ "but it is '#{migration.status_name}':" \ "\t#{configuration}" \ "\n\n" \ - "Finalize it manualy by running" \ + "Finalize it manually by running" \ "\n\n" \ "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ "\n\n" \ diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb index 5a25128f3a9..d8d07fcaf2d 100644 --- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -27,7 +27,7 @@ module Gitlab return end - Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do + Gitlab::Database::QueryAnalyzer.instance.within([validator_class, connection_validator_class]) do validator_class.allowed_gitlab_schemas = self.allowed_gitlab_schemas super @@ -45,6 +45,10 @@ module Gitlab Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas end + def connection_validator_class + Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection + end + def unmatched_schemas (self.allowed_gitlab_schemas || []) - allowed_schemas_for_connection end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 7e5c002d072..9bffed43077 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -41,6 +41,25 @@ module Gitlab # end # end def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + if transaction_open? + raise 'The `#queue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + # Background Migrations do not work well for in cases requiring to update `gitlab_shared` + # Once the decomposition is done, enqueued jobs for `gitlab_shared` tables (on CI database) + # will not be executed since the queue (which is stored in Redis) is tied to main database, not to schema. + # The batched background migrations do not have those limitations since the tracking tables + # are properly database-only. + if background_migration_restrict_gitlab_migration_schemas&.include?(:gitlab_shared) + raise 'The `#queue_background_migration_jobs_by_range_at_intervals` cannot " \ + "use `restrict_gitlab_migration:` " with `:gitlab_shared`. ' \ + 'Background migrations do encode migration worker which is tied to a given database. ' \ + 'After split this worker will not be properly duplicated into decomposed database. ' \ + 'Use batched background migrations instead that do support well working across all databases.' + end + raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer or string column" unless [:integer, :string].include?(model_class.columns_hash[primary_column_name.to_s].type) @@ -90,6 +109,18 @@ module Gitlab # delay_interval - The duration between each job's scheduled time # batch_size - The maximum number of jobs to fetch to memory from the database. def requeue_background_migration_jobs_by_range_at_intervals(job_class_name, delay_interval, batch_size: BATCH_SIZE, initial_delay: 0) + if transaction_open? + raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if background_migration_restrict_gitlab_migration_schemas&.any? + raise 'The `#requeue_background_migration_jobs_by_range_at_intervals` cannot use `restrict_gitlab_migration:`. ' \ + 'The `#requeue_background_migration_jobs_by_range_at_intervals` needs to be executed on all databases since ' \ + 'each database has its own queue of background migrations.' + end + job_coordinator = coordinator_for_tracking_database # To not overload the worker too much we enforce a minimum interval both @@ -133,23 +164,40 @@ module Gitlab # This method does not garauntee that all jobs completed successfully. # It can only be used if the previous background migration used the queue_background_migration_jobs_by_range_at_intervals helper. def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) + if transaction_open? + raise 'The `#finalize_background_migration` can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + + if background_migration_restrict_gitlab_migration_schemas&.any? + raise 'The `#finalize_background_migration` cannot use `restrict_gitlab_migration:`. ' \ + 'The `#finalize_background_migration` needs to be executed on all databases since ' \ + 'each database has its own queue of background migrations.' + end + job_coordinator = coordinator_for_tracking_database - # Empty the sidekiq queue. - job_coordinator.steal(class_name) + with_restored_connection_stack do + # Since we are running trusted code (background migration class) allow to execute any type of finalize + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + # Empty the sidekiq queue. + job_coordinator.steal(class_name) - # Process pending tracked jobs. - jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) + # Process pending tracked jobs. + jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) - jobs.find_each do |job| - job_coordinator.perform(job.class_name, job.arguments) - end + jobs.find_each do |job| + job_coordinator.perform(job.class_name, job.arguments) + end - # Empty the sidekiq queue. - job_coordinator.steal(class_name) + # Empty the sidekiq queue. + job_coordinator.steal(class_name) - # Delete job tracking rows. - delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + # Delete job tracking rows. + delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + end + end end def migrate_in(*args, coordinator: coordinator_for_tracking_database) @@ -174,6 +222,10 @@ module Gitlab private + def background_migration_restrict_gitlab_migration_schemas + self.allowed_gitlab_schemas if self.respond_to?(:allowed_gitlab_schemas) + end + def with_migration_context(&block) Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) end @@ -183,11 +235,9 @@ module Gitlab end def coordinator_for_tracking_database - Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) - end + tracking_database = Gitlab::Database.db_config_name(connection) - def tracking_database - Gitlab::BackgroundMigration::DEFAULT_TRACKING_DATABASE + Gitlab::BackgroundMigration.coordinator_for_database(tracking_database) end end end diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb new file mode 100644 index 00000000000..2772502140e --- /dev/null +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class BaseBackgroundRunner + attr_reader :result_dir + + def initialize(result_dir:) + @result_dir = result_dir + end + + def jobs_by_migration_name + raise NotImplementedError, 'subclass must implement' + end + + def run_job(job) + raise NotImplementedError, 'subclass must implement' + end + + def run_jobs(for_duration:) + jobs_to_run = jobs_by_migration_name + return if jobs_to_run.empty? + + # without .to_f, we do integer division + # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) + duration_per_migration_type = for_duration / jobs_to_run.count.to_f + jobs_to_run.each do |migration_name, jobs| + run_until = duration_per_migration_type.from_now + + run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) + end + end + + private + + def run_jobs_for_migration(migration_name:, jobs:, run_until:) + per_background_migration_result_dir = File.join(@result_dir, migration_name) + + instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) + batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} + + jobs.shuffle.each do |j| + break if run_until <= Time.current + + instrumentation.observe(version: nil, + name: batch_names.next, + connection: ActiveRecord::Migration.connection) do + run_job(j) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 0261ade0fe7..7113c3686f1 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -122,6 +122,22 @@ module Gitlab migration.save! migration end + + def finalize_batched_background_migration(job_class_name:, table_name:, column_name:, job_arguments:) + database_name = Gitlab::Database.db_config_name(connection) + + unless ActiveRecord::Base.configurations.primary?(database_name) + raise 'The `#finalize_background_migration` is currently not supported when running in decomposed database, ' \ + 'and this database is not `main:`. For more information visit: ' \ + 'https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html' + end + + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + + raise 'Could not find batched background migration' if migration.nil? + + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(job_class_name, table_name, column_name, job_arguments, connection: connection) + end end end end diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb index 8ca57bb7df9..543e6b8e302 100644 --- a/lib/gitlab/database/migrations/observers/query_log.rb +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class QueryLog < MigrationObserver def before - @logger_was = ActiveRecord::Base.logger + @logger_was = ActiveRecord::Base.logger # rubocop:disable Database/MultipleDatabases file_path = File.join(output_dir, "migration.log") @logger = Logger.new(file_path) ActiveRecord::Base.logger = @logger diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb index 54504646a79..2d026f0c8d2 100644 --- a/lib/gitlab/database/migrations/observers/query_statistics.rb +++ b/lib/gitlab/database/migrations/observers/query_statistics.rb @@ -22,6 +22,7 @@ module Gitlab observation.query_statistics = connection.execute(<<~SQL) SELECT query, calls, total_time, max_time, mean_time, rows FROM pg_stat_statements + WHERE pg_get_userbyid(userid) = current_user ORDER BY total_time DESC SQL end diff --git a/lib/gitlab/database/migrations/reestablished_connection_stack.rb b/lib/gitlab/database/migrations/reestablished_connection_stack.rb new file mode 100644 index 00000000000..d7cf482c32a --- /dev/null +++ b/lib/gitlab/database/migrations/reestablished_connection_stack.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ReestablishedConnectionStack + # This is workaround for `db:migrate` that switches `ActiveRecord::Base.connection` + # depending on execution. This is subject to be removed once proper fix is implemented: + # https://gitlab.com/gitlab-org/gitlab/-/issues/362341 + # + # In some cases when we run application code we need to restore application connection stack: + # - ApplicationRecord (in fact ActiveRecord::Base): points to main + # - Ci::ApplicationRecord: points to ci + # + # rubocop:disable Database/MultipleDatabases + def with_restored_connection_stack(&block) + original_handler = ActiveRecord::Base.connection_handler + + original_db_config = ActiveRecord::Base.connection_db_config + return yield if ActiveRecord::Base.configurations.primary?(original_db_config.name) + + # If the `ActiveRecord::Base` connection is different than `:main` + # re-establish and configure `SharedModel` context accordingly + # to previously established `ActiveRecord::Base` to allow the application + # code to use `ApplicationRecord` and `Ci::ApplicationRecord` usual way. + # We swap a connection handler as migration context does hold an actual + # connection which we cannot close. + base_model = Gitlab::Database.database_base_models.fetch(original_db_config.name.to_sym) + + # copy connections over to new connection handler + db_configs = original_handler.connection_pool_names.map do |connection_pool_name| + [connection_pool_name.constantize, connection_pool_name.constantize.connection_db_config] + end + + new_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new + ActiveRecord::Base.connection_handler = new_handler + + db_configs.each do |klass, db_config| + new_handler.establish_connection(db_config, owner_name: klass) + end + + # re-establish ActiveRecord::Base to main + ActiveRecord::Base.establish_connection :main # rubocop:disable Database/EstablishConnection + + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + yield + end + ensure + ActiveRecord::Base.connection_handler = original_handler + new_handler&.clear_all_connections! + end + # rubocop:enable Database/MultipleDatabases + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 3b6f52b43a8..4404b5bf961 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -21,6 +21,18 @@ module Gitlab TestBackgroundRunner.new(result_dir: BASE_RESULT_DIR.join('background_migrations')) end + def batched_background_migrations(for_database:) + runner = nil + + # Only one loop iteration since we pass `only:` here + Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner + .new(result_dir: BASE_RESULT_DIR.join('background_migrations'), connection: connection) + end + + runner + end + def migration_context @migration_context ||= ApplicationRecord.connection.migration_context end diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index 74e54d62e05..f7713237b38 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -3,11 +3,9 @@ module Gitlab module Database module Migrations - class TestBackgroundRunner - attr_reader :result_dir - + class TestBackgroundRunner < BaseBackgroundRunner def initialize(result_dir:) - @result_dir = result_dir + super(result_dir: result_dir) @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) end @@ -15,37 +13,12 @@ module Gitlab @job_coordinator.pending_jobs end - def run_jobs(for_duration:) - jobs_to_run = traditional_background_migrations.group_by { |j| class_name_for_job(j) } - return if jobs_to_run.empty? - - # without .to_f, we do integer division - # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) - duration_per_migration_type = for_duration / jobs_to_run.count.to_f - jobs_to_run.each do |migration_name, jobs| - run_until = duration_per_migration_type.from_now - - run_jobs_for_migration(migration_name: migration_name, jobs: jobs, run_until: run_until) - end + def jobs_by_migration_name + traditional_background_migrations.group_by { |j| class_name_for_job(j) } end private - def run_jobs_for_migration(migration_name:, jobs:, run_until:) - per_background_migration_result_dir = File.join(@result_dir, migration_name) - - instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) - batch_names = (1..).each.lazy.map { |i| "batch_#{i}"} - - jobs.shuffle.each do |j| - break if run_until <= Time.current - - instrumentation.observe(version: nil, name: batch_names.next, connection: ActiveRecord::Migration.connection) do - run_job(j) - end - end - end - def run_job(job) Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb new file mode 100644 index 00000000000..0c6a8d3d856 --- /dev/null +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class TestBatchedBackgroundRunner < BaseBackgroundRunner + attr_reader :connection + + def initialize(result_dir:, connection:) + super(result_dir: result_dir) + @connection = connection + end + + def jobs_by_migration_name + Gitlab::Database::BackgroundMigration::BatchedMigration + .executable + .created_after(2.days.ago) # Simple way to exclude migrations already running before migration testing + .to_h do |migration| + batching_strategy = migration.batch_class.new(connection: connection) + + all_migration_jobs = [] + + min_value = migration.next_min_value + + while (next_bounds = batching_strategy.next_batch( + migration.table_name, + migration.column_name, + batch_min_value: min_value, + batch_size: migration.batch_size, + job_arguments: migration.job_arguments + )) + + batch_min, batch_max = next_bounds + + all_migration_jobs << migration.create_batched_job!(batch_min, batch_max) + min_value = batch_max + 1 + end + + [migration.job_class_name, all_migration_jobs] + end + end + + def run_job(job) + Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 034e18ec9f4..a541ecf5316 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -6,7 +6,6 @@ module Gitlab module TableManagementHelpers include ::Gitlab::Database::SchemaHelpers include ::Gitlab::Database::MigrationHelpers - include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze ERROR_SCOPE = 'table partitioning' diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 0c78dda734c..6f64d04270f 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -30,52 +30,25 @@ module Gitlab end end - def within(user_analyzers = nil) - # Due to singleton nature of analyzers - # only an outer invocation of the `.within` - # is allowed to initialize them - if already_within? - raise 'Query analyzers are already defined, cannot re-define them.' if user_analyzers - - return yield - end - - begin!(user_analyzers || all_analyzers) + def within(analyzers = all_analyzers) + newly_enabled_analyzers = begin!(analyzers) begin yield ensure - end! + end!(newly_enabled_analyzers) end end - def already_within? - # If analyzers are set they are already configured - !enabled_analyzers.nil? - end + # Enable query analyzers (only the ones that were not yet enabled) + # Returns a list of newly enabled analyzers + def begin!(analyzers) + analyzers.select do |analyzer| + next if enabled_analyzers.include?(analyzer) - def process_sql(sql, connection) - analyzers = enabled_analyzers - return unless analyzers&.any? - - parsed = parse(sql, connection) - return unless parsed - - analyzers.each do |analyzer| - next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) - - analyzer.analyze(parsed) - rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e - # We catch all standard errors to prevent validation errors to introduce fatal errors in production - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) - end - end - - # Enable query analyzers - def begin!(analyzers = all_analyzers) - analyzers = analyzers.select do |analyzer| if analyzer.enabled? analyzer.begin! + enabled_analyzers.append(analyzer) true end @@ -84,25 +57,40 @@ module Gitlab false end - - Thread.current[:query_analyzer_enabled_analyzers] = analyzers end - # Disable enabled query analyzers - def end! - enabled_analyzers.select do |analyzer| + # Disable enabled query analyzers (only the ones that were enabled previously) + def end!(analyzers) + analyzers.each do |analyzer| + next unless enabled_analyzers.delete(analyzer) + analyzer.end! rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end - - Thread.current[:query_analyzer_enabled_analyzers] = nil end private def enabled_analyzers - Thread.current[:query_analyzer_enabled_analyzers] + Thread.current[:query_analyzer_enabled_analyzers] ||= [] + end + + def process_sql(sql, connection) + analyzers = enabled_analyzers + return unless analyzers&.any? + + parsed = parse(sql, connection) + return unless parsed + + analyzers.each do |analyzer| + next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) + + analyzer.analyze(parsed) + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e + # We catch all standard errors to prevent validation errors to introduce fatal errors in production + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end end def parse(sql, connection) diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb new file mode 100644 index 00000000000..3de9e8011fb --- /dev/null +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + # The purpose of this analyzer is to validate if tables observed + # are properly used according to schema used by current connection + class GitlabSchemasValidateConnection < Base + CrossSchemaAccessError = Class.new(QueryAnalyzerError) + + class << self + def enabled? + true + end + + def analyze(parsed) + tables = parsed.pg.select_tables + parsed.pg.dml_tables + table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) + return if table_schemas.empty? + + allowed_schemas = ::Gitlab::Database.gitlab_schemas_for_connection(parsed.connection) + return unless allowed_schemas + + invalid_schemas = table_schemas - allowed_schemas + if invalid_schemas.any? + message = "The query tried to access #{tables} (of #{table_schemas.to_a}) " + message += "which is outside of allowed schemas (#{allowed_schemas}) " + message += "for the current connection '#{Gitlab::Database.db_config_name(parsed.connection)}'" + + raise CrossSchemaAccessError, message + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index a53da514df2..e0cb803b872 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -33,7 +33,7 @@ module Gitlab def self.enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml) + Feature.enabled?(:detect_cross_database_modification) end def self.requires_tracking?(parsed) diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 91c3fcc7d72..e13dd3b2058 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -16,7 +16,7 @@ module Gitlab REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 def self.enabled? - Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + Feature.enabled?(:database_reindexing, type: :ops) end def self.invoke(database = nil) diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 563fab692ef..f4c8fca8fa2 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -15,14 +15,16 @@ module Gitlab previous_connection = self.overriding_connection unless previous_connection.nil? || previous_connection.equal?(connection) - raise 'cannot nest connection overrides for shared models with different connections' + raise "Cannot change connection for Gitlab::Database::SharedModel "\ + "from '#{Gitlab::Database.db_config_name(previous_connection)}' "\ + "to '#{Gitlab::Database.db_config_name(connection)}'" end self.overriding_connection = connection yield ensure - self.overriding_connection = nil unless previous_connection.equal?(self.overriding_connection) + self.overriding_connection = previous_connection end def connection diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 2d9700cb2bc..1e29ae7761b 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -4,10 +4,18 @@ module Gitlab module DatabaseImporters module WorkItems module BaseTypeImporter - def self.import - ::WorkItems::Type::BASE_TYPES.each do |type, attributes| - ::WorkItems::Type.create!(base_type: type, **attributes.slice(:name, :icon_name)) + def self.upsert_types + current_time = Time.current + + base_types = ::WorkItems::Type::BASE_TYPES.map do |type, attributes| + attributes.slice(:name, :icon_name) + .merge(created_at: current_time, updated_at: current_time, base_type: type) end + + ::WorkItems::Type.upsert_all( + base_types, + unique_by: :idx_work_item_types_on_namespace_id_and_name_null_namespace + ) end end end diff --git a/lib/gitlab/default_branch.rb b/lib/gitlab/default_branch.rb index 6bd9a5675c4..bb540b93a58 100644 --- a/lib/gitlab/default_branch.rb +++ b/lib/gitlab/default_branch.rb @@ -4,7 +4,7 @@ module Gitlab module DefaultBranch def self.value(object: nil) - Feature.enabled?(:main_branch_over_master, object, default_enabled: :yaml) ? 'main' : 'master' + Feature.enabled?(:main_branch_over_master, object) ? 'main' : 'master' end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 61bb0c797b4..d6ee21b93b6 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -50,11 +50,11 @@ module Gitlab end def use_semantic_ipynb_diff? - strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project, default_enabled: :yaml) } + strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project) } end def use_renderable_diff? - strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project, default_enabled: :yaml) } + strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project) } end def has_renderable? @@ -386,6 +386,10 @@ module Gitlab strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end + def ipynb? + file_path.ends_with?('.ipynb') + end + private def diffable_by_attribute? @@ -415,10 +419,6 @@ module Gitlab new_file? || deleted_file? || content_changed? end - def ipynb? - file_path.ends_with?('.ipynb') - end - # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 47f3324752d..225b4f7cf86 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,7 +24,7 @@ module Gitlab end def highlight - populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project) @diff_lines.map.with_index do |diff_line, index| diff_line = diff_line.dup @@ -61,7 +61,7 @@ module Gitlab end def apply_marker_ranges_highlight(diff_line, rich_line, index) - marker_ranges = if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + marker_ranges = if Feature.enabled?(:use_marker_ranges, project) diff_line.marker_ranges else inline_diffs[index] @@ -83,7 +83,7 @@ module Gitlab return unless diff_file && diff_file.diff_refs return diff_line_highlighting(diff_line, plain: true) if blobs_too_large? - if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml) + if Feature.enabled?(:diff_line_syntax_highlighting, project) diff_line_highlighting(diff_line) else blob_highlighting(diff_line) diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 12ed11b0140..f950d01fdf0 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -74,8 +74,8 @@ module Gitlab diffable.cache_key, VERSION, diff_options, - Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), - Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) + Feature.enabled?(:use_marker_ranges, diffable.project), + Feature.enabled?(:diff_line_syntax_highlighting, diffable.project) ].join(":") end end diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index cf97569ca31..1f064d8af50 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -13,6 +13,7 @@ module Gitlab LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' + LOG_IPYNBDIFF_TRUNCATED = 'IPYNB_DIFF_TRUNCATED' attr_reader :source_diff @@ -60,9 +61,16 @@ module Gitlab def notebook_diff strong_memoize(:notebook_diff) do + if source_diff.old_blob&.truncated? || source_diff.new_blob&.truncated? + log_event(LOG_IPYNBDIFF_TRUNCATED) + next + end + Timeout.timeout(timeout_time) do IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, - raise_if_invalid_nb: true, diffy_opts: { include_diff_info: true })&.tap do + raise_if_invalid_nb: true, + hide_images: true, + diffy_opts: { include_diff_info: true })&.tap do log_event(LOG_IPYNBDIFF_GENERATED) end end @@ -141,7 +149,7 @@ module Gitlab def log_event(message, error = nil) Gitlab::AppLogger.info({ message: message }) - Gitlab::ErrorTracking.track_exception(error) if error + Gitlab::ErrorTracking.log_exception(error) if error nil end end diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index 44f5c97c70c..cd075569d10 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -30,14 +30,35 @@ module Gitlab private + # Skipping initializers may be needed if those attempt to access + # encrypted data on initialization and could fail because of it. + # + # format example: + # { + # model_class => { + # [ + # { action: :create, filters: [:before, :filter_name1] }, + # { action: :update, filters: [:after, :filter_name2] } + # ] + # } + # } + MODEL_INITIALIZERS_TO_SKIP = { + Integration => [ + { action: :initialize, filters: [:after, :initialize_properties] } + ] + }.freeze + def check_model_attributes(models_with_attributes) running_failures = 0 models_with_attributes.each do |model, attributes| failures_per_row = Hash.new { |h, k| h[k] = [] } - model.find_each do |data| - attributes.each do |att| - failures_per_row[data.id] << att unless valid_attribute?(data, att) + + with_skipped_callbacks_for(model) do + model.find_each do |data| + attributes.each do |att| + failures_per_row[data.id] << att unless valid_attribute?(data, att) + end end end @@ -82,6 +103,32 @@ module Gitlab false end + + # WARNING: using this logic in other places than a Rake task will need a + # different approach, as simply setting the callback again is not thread-safe + def with_skipped_callbacks_for(model) + raise StandardError, 'can only be used in a Rake environment' unless Gitlab::Runtime.rake? + + skip_callbacks_for_model(model) + + yield + + skip_callbacks_for_model(model, reset: true) + end + + def skip_callbacks_for_model(model, reset: false) + MODEL_INITIALIZERS_TO_SKIP.each do |klass, initializers| + next unless model <= klass + + initializers.each do |initializer| + if reset + model.set_callback(initializer[:action], *initializer[:filters]) + else + model.skip_callback(initializer[:action], *initializer[:filters]) + end + end + end + end end end end diff --git a/lib/gitlab/email/message/build_ios_app_guide.rb b/lib/gitlab/email/message/build_ios_app_guide.rb new file mode 100644 index 00000000000..4acf558a6a2 --- /dev/null +++ b/lib/gitlab/email/message/build_ios_app_guide.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + class BuildIosAppGuide + include Gitlab::Email::Message::InProductMarketing::Helper + include Gitlab::Routing + + attr_accessor :format + + def initialize(format: :html) + @format = format + end + + def subject_line + s_('InProductMarketing|Get set up to build for iOS') + end + + def title + s_("InProductMarketing|Building for iOS? We've got you covered.") + end + + def body_line1 + s_( + 'InProductMarketing|Want to get your iOS app up and running, including publishing all the way to ' \ + 'TestFlight? Follow our guide to set up GitLab and fastlane to publish iOS apps to the App Store.' + ) + end + + def cta_text + s_('InProductMarketing|Learn how to build for iOS') + end + + def cta_link + action_link(cta_text, 'https://about.gitlab.com/blog/2019/03/06/ios-publishing-with-gitlab-and-fastlane/') + end + + def cta2_text + s_('InProductMarketing|Watch iOS building in action.') + end + + def cta2_link + action_link(cta2_text, 'https://www.youtube.com/watch?v=325FyJt7ZG8') + end + + def logo_path + 'mailers/in_product_marketing/create-0.png' + end + + def unsubscribe + unsubscribe_message + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 9b50d86de58..bd20b7e5fc7 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -70,14 +70,8 @@ module Gitlab end def unsubscribe - parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(track, series) - - case format - when :html - parts.join(' ') - else - parts.join("\n" + ' ' * 16) - end + self_managed_preferences_link = marketing_preference_link(track, series) + unsubscribe_message(self_managed_preferences_link) end def progress(current: series + 1, total: total_series, track_name: track.to_s.humanize) @@ -110,26 +104,6 @@ module Gitlab Namespaces::InProductMarketingEmailsService::TRACKS[track][:interval_days].size end - def unsubscribe_com - [ - s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), - s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } - ] - end - - def unsubscribe_self_managed(track, series) - [ - s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, - s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: marketing_preference_link(track, series) } - ] - end - - def unsubscribe_link - unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url - - link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) - end - def marketing_preference_link(track, series) params = { utm_source: 'SM', diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb index 329cace9e9d..0a0e55c2999 100644 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -31,8 +31,39 @@ module Gitlab s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options end + def unsubscribe_message(self_managed_preferences_link = nil) + parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(self_managed_preferences_link) + + case format + when :html + parts.join(' ') + else + parts.join("\n" + ' ' * 16) + end + end + private + def unsubscribe_link + unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url + + link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) + end + + def unsubscribe_com + [ + s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), + s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } + ] + end + + def unsubscribe_self_managed(preferences_link) + [ + s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, + s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: preferences_link } + ] + end + def list(array) case format when :html diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 58e7b2f1b44..4da112bc5a0 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -148,7 +148,7 @@ module Gitlab end def find_first_key_from_received_headers - return unless ::Feature.enabled?(:use_received_header_for_incoming_emails, default_enabled: :yaml) + return unless ::Feature.enabled?(:use_received_header_for_incoming_emails) recipients_from_received_headers.find do |email| key = email_class.key_from_address(email) diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb index 3675646185e..5e1eabe7ec6 100644 --- a/lib/gitlab/encrypted_ldap_command.rb +++ b/lib/gitlab/encrypted_ldap_command.rb @@ -15,7 +15,7 @@ module Gitlab <<~YAML # main: # password: '123' - # user_dn: 'gitlab-adm' + # bind_dn: 'gitlab-adm' YAML end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index d71f9b5e7cf..f9959d5677b 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -116,13 +116,13 @@ module Gitlab private def before_send_raven(event, hint) - return unless Feature.enabled?(:enable_old_sentry_integration, default_enabled: :yaml) + return unless Feature.enabled?(:enable_old_sentry_integration) before_send(event, hint) end def before_send_sentry(event, hint) - return unless Feature.enabled?(:enable_new_sentry_integration, default_enabled: :yaml) + return unless Feature.enabled?(:enable_new_sentry_integration) before_send(event, hint) end diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb new file mode 100644 index 00000000000..4ec636703d9 --- /dev/null +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + # Data access layer for errors and events related to Error Tracking feature. + class ErrorRepository + Pagination = Struct.new(:next, :prev) + + # Generic database error + DatabaseError = Class.new(StandardError) + # Record was invalid + RecordInvalidError = Class.new(DatabaseError) + + # Builds an instance of error repository backed by a strategy. + # + # @return [self] + def self.build(project) + strategy = ActiveRecordStrategy.new(project) + + new(strategy) + end + + # @private + def initialize(strategy) + @strategy = strategy + end + + # Stores an error and the related error event. + # + # @param name [String] name of the error + # @param description [String] description of the error + # @param actor [String] culprit (class/method/function) which triggered this error + # @param platform [String] platform on which the error occurred + # @param environment [String] environment on which the error occurred + # @param level [String] severity of this error + # @param occurred_at [Time] timestamp when the error occurred + # @param payload [Hash] original error payload + # + # @return [void] nothing + # + # @raise [RecordInvalidError] if passed attributes were invalid to store an error or error event + # @raise [DatabaseError] if generic error occurred + def report_error( + name:, description:, actor:, platform:, + environment:, level:, occurred_at: Time.zone.now, payload: {} + ) + strategy.report_error( + name: name, + description: description, + actor: actor, + platform: platform, + environment: environment, + level: level, + occurred_at: occurred_at, + payload: payload + ) + + nil + end + + # Finds an error by +id+. + # + # @param id [Integer, String] unique error identifier + # + # @return [Gitlab::ErrorTracking::DetailedError] a detail error + def find_error(id) + strategy.find_error(id) + end + + # Lists errors. + # + # @param sort [String] order list by 'first_seen', 'last_seen', or 'frequency' + # @param filters [Hash<Symbol, String>] filter list by + # @option filters [String] :status error status + # @param limit [Integer, String] limit result + # @param cursor [Hash] pagination information + # + # @return [Array<Array<Gitlab::ErrorTracking::Error>, Pagination>] + def list_errors(sort: 'last_seen', filters: {}, limit: 20, cursor: {}) + limit = [limit.to_i, 100].min + + strategy.list_errors(filters: filters, sort: sort, limit: limit, cursor: cursor) + end + + # Fetches last event for error +id+. + # + # @param id [Integer, String] unique error identifier + # + # @return [Gitlab::ErrorTracking::ErrorEvent] + # + # @raise [DatabaseError] if generic error occurred + def last_event_for(id) + strategy.last_event_for(id) + end + + # Updates attributes of an error. + # + # @param id [Integer, String] unique error identifier + # @param status [String] error status + # + # @return [true, false] if update was successful + # + # @raise [DatabaseError] if generic error occurred + def update_error(id, status:) + strategy.update_error(id, status: status) + end + + private + + attr_reader :strategy + end + end +end diff --git a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb new file mode 100644 index 00000000000..e5b532ee0f0 --- /dev/null +++ b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ErrorRepository + class ActiveRecordStrategy + def initialize(project) + @project = project + end + + def report_error( + name:, description:, actor:, platform:, + environment:, level:, occurred_at:, payload: + ) + error = project_errors.report_error( + name: name, # Example: ActionView::MissingTemplate + description: description, # Example: Missing template posts/show in... + actor: actor, # Example: PostsController#show + platform: platform, # Example: ruby + timestamp: occurred_at + ) + + # The payload field contains all the data on error including stacktrace in jsonb. + # Together with occurred_at these are 2 main attributes that we need to save here. + error.events.create!( + environment: environment, + description: description, + level: level, + occurred_at: occurred_at, + payload: payload + ) + rescue ActiveRecord::ActiveRecordError => e + handle_exceptions(e) + end + + def find_error(id) + project_error(id).to_sentry_detailed_error + rescue ActiveRecord::ActiveRecordError => e + handle_exceptions(e) + end + + def list_errors(filters:, sort:, limit:, cursor:) + errors = project_errors + errors = filter_by_status(errors, filters[:status]) + errors = sort(errors, sort) + errors = errors.keyset_paginate(cursor: cursor, per_page: limit) + + pagination = ErrorRepository::Pagination.new(errors.cursor_for_next_page, errors.cursor_for_previous_page) + + [errors.map(&:to_sentry_error), pagination] + end + + def last_event_for(id) + project_error(id).last_event&.to_sentry_error_event + rescue ActiveRecord::ActiveRecordError => e + handle_exceptions(e) + end + + def update_error(id, **attributes) + project_error(id).update(attributes) + end + + private + + attr_reader :project + + def project_errors + ::ErrorTracking::Error.where(project: project) # rubocop:disable CodeReuse/ActiveRecord + end + + def project_error(id) + project_errors.find(id) + end + + def filter_by_status(errors, status) + return errors unless ::ErrorTracking::Error.statuses.key?(status) + + errors.for_status(status) + end + + def sort(errors, sort) + return errors.order_id_desc unless sort + + errors.sort_by_attribute(sort) + end + + def handle_exceptions(exception) + case exception + when ActiveRecord::RecordInvalid + raise RecordInvalidError, exception.message + else + raise DatabaseError, exception.message + end + end + end + end + end +end diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 4bef92f5c23..bf31dfe08a0 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -14,7 +14,7 @@ module Gitlab def enabled? return false unless feature_flag_defined? return false unless Gitlab.com? - return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops, default_enabled: :yaml) + return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops) feature_flag_instance.state != :off end @@ -29,7 +29,7 @@ module Gitlab # which will assign the control. Otherwise we call super, which will # assign a variant evenly, or based on our provided distribution rules. def execute_assignment - super if ::Feature.enabled?(feature_flag_name, self, type: :experiment, default_enabled: :yaml) + super if ::Feature.enabled?(feature_flag_name, self, type: :experiment) end # This is what's provided to the `Feature.enabled?` call that will be diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index a68e2db4dac..b09d67b8d5f 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -146,9 +146,9 @@ module Gitlab return experimentation_subject_id if subject.blank? if subject.respond_to?(:to_global_id) - Digest::MD5.hexdigest(subject.to_global_id.to_s) + Digest::SHA256.hexdigest(subject.to_global_id.to_s) else - Digest::MD5.hexdigest(subject.to_s) + Digest::SHA256.hexdigest(subject.to_s) end end end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index b13f55e7969..0c7091d19e3 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -16,7 +16,7 @@ module Gitlab def active? # TODO: just touch a feature flag # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? - Feature.enabled?(feature_flag_name, type: :experiment, default_enabled: :yaml) + Feature.enabled?(feature_flag_name, type: :experiment) ::Gitlab.com? && experiment_percentage > 0 end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index fbe52db9c0b..9637f8756b1 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -13,10 +13,6 @@ module Gitlab end end - def initialize(repository, name, target, target_commit) - super(repository, name, target, target_commit) - end - def active? self.dereferenced_target.committed_date >= STALE_BRANCH_THRESHOLD.ago end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index a66517b4ca0..c473fe6973d 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -230,12 +230,16 @@ module Gitlab private def encode_diff_to_utf8(replace_invalid_utf8_chars) - return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol, default_enabled: :yaml) - return unless replace_invalid_utf8_chars && !detect_binary?(@diff) + return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol) + return unless replace_invalid_utf8_chars && diff_should_be_converted? @diff = Gitlab::EncodingHelper.encode_utf8_with_replacement_character(@diff) end + def diff_should_be_converted? + !detect_binary?(@diff) || !@diff&.valid_encoding? + end + def init_from_hash(hash) raw_diff = hash.symbolize_keys diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index f98fb66ad21..cba63b3c6c7 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -177,8 +177,10 @@ module Gitlab def check_valid_actor! return unless key? - unless actor.valid? + if !actor.valid? raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}." + elsif actor.expired? + raise ForbiddenError, "Your SSH key has expired." end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4fe5c8df36f..5e1f92ae835 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -276,7 +276,36 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) - consume_commits_response(response) + + quarantined_commits = consume_commits_response(response) + + if Feature.enabled?(:filter_quarantined_commits) + quarantined_commit_ids = quarantined_commits.map(&:id) + + # While in general the quarantine directory would only contain objects + # which are actually new, this is not guaranteed by Git. In fact, + # git-push(1) may sometimes push objects which already exist in the + # target repository. We do not want to return those from this method + # though given that they're not actually new. + # + # To fix this edge-case we thus have to filter commits down to those + # which don't yet exist. To do so, we must check for object existence + # in the main repository, but the object directory of our repository + # points into the object quarantine. This can be fixed by unsetting + # it, which will cause us to use the normal repository as indicated by + # its relative path again. + main_repo = @gitaly_repo.dup + main_repo.git_object_directory = "" + + # Check object existence of all quarantined commits' IDs. + quarantined_commit_existence = object_existence_map(quarantined_commit_ids, gitaly_repo: main_repo) + + # And then we reject all quarantined commits which exist in the main + # repository already. + quarantined_commits.reject! { |c| quarantined_commit_existence[c.id] } + end + + quarantined_commits else list_commits(Array.wrap(revisions) + %w[--not --all]) end @@ -387,6 +416,35 @@ module Gitlab consume_commits_response(response) end + # Check whether the given revisions exist. Returns a hash mapping the revision name to either `true` if the + # revision exists, or `false` otherwise. This function accepts all revisions as specified by + # gitrevisions(1). + def object_existence_map(revisions, gitaly_repo: @gitaly_repo) + enum = Enumerator.new do |y| + # This is a bug in Gitaly: revisions of the initial request are ignored. This will be fixed in v15.0 via + # https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4510, so we can merge initial request and the initial + # set of revisions starting with v15.1. + y.yield Gitaly::CheckObjectsExistRequest.new(repository: gitaly_repo) + + revisions.each_slice(100) do |revisions_subset| + y.yield Gitaly::CheckObjectsExistRequest.new(revisions: revisions_subset) + end + end + + response = GitalyClient.call( + @repository.storage, :commit_service, :check_objects_exist, enum, timeout: GitalyClient.medium_timeout + ) + + existence_by_revision = {} + response.each do |message| + message.revisions.each do |revision| + existence_by_revision[revision.name] = revision.exists + end + end + + existence_by_revision + end + def filter_shas_with_signatures(shas) request = Gitaly::FilterShasWithSignaturesRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 1e199a55b5a..5adb8d946a0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -48,7 +48,7 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) response.size end diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 7ac0d875512..9556a9e98ba 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -36,7 +36,7 @@ module Gitlab end def self.per_page(project) - if project.group.present? && Feature.enabled?(:github_importer_lower_per_page_limit, project.group, type: :ops, default_enabled: :yaml) + if project.group.present? && Feature.enabled?(:github_importer_lower_per_page_limit, project.group, type: :ops) Gitlab::GithubImport::Client::LOWER_PER_PAGE else Gitlab::GithubImport::Client::DEFAULT_PER_PAGE diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index 5298a3d81ea..da205ebd345 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -78,7 +78,7 @@ module Gitlab end def timeout - if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops, default_enabled: :yaml) + if project.group.present? && ::Feature.enabled?(:github_importer_single_endpoint_notes_import, project.group, type: :ops) Gitlab::Cache::Import::Caching::LONGER_TIMEOUT else Gitlab::Cache::Import::Caching::TIMEOUT diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 97de2a49e72..ab20b372d53 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -207,13 +207,8 @@ module Gitlab end # Default batch settings for parallel import (can be redefined in Importer classes) - # Example: { size: 100, delay: 1.minute } def parallel_import_batch - if Feature.enabled?(:distribute_github_parallel_import, default_enabled: :yaml) - { size: 1000, delay: 1.minute } - else - {} - end + { size: 1000, delay: 1.minute } end def abort_on_failure diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3c85d56874f..98570c02e3d 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -51,15 +51,14 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) - push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) - push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) - push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) - push_frontend_feature_flag(:source_editor_toolbar, default_enabled: :yaml) - push_frontend_feature_flag(:gl_avatar_for_all_user_avatars, default_enabled: :yaml) - push_frontend_feature_flag(:mr_attention_requests, default_enabled: :yaml) - push_frontend_feature_flag(:markdown_continue_lists, default_enabled: :yaml) + push_frontend_feature_flag(:usage_data_api, type: :ops) + push_frontend_feature_flag(:security_auto_fix) + push_frontend_feature_flag(:new_header_search) + push_frontend_feature_flag(:bootstrap_confirmation_modals) + push_frontend_feature_flag(:sandboxed_mermaid) + push_frontend_feature_flag(:source_editor_toolbar) + push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) + push_frontend_feature_flag(:mr_attention_requests, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/find_argument_in_parent.rb b/lib/gitlab/graphql/find_argument_in_parent.rb deleted file mode 100644 index 1f83f8fce7a..00000000000 --- a/lib/gitlab/graphql/find_argument_in_parent.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module FindArgumentInParent - # Searches up the GraphQL AST and returns the first matching argument - # passed to a node - def self.find(parent, argument, limit_depth: nil) - argument = argument.to_s.camelize(:lower).to_sym - depth = 0 - - while parent.respond_to?(:parent) - args = node_args(parent) - return args[argument] if args.key?(argument) - - depth += 1 - return if limit_depth && depth >= limit_depth - - parent = parent.parent - end - end - - class << self - private - - def node_args(node) - node.irep_node.arguments - end - end - end - end -end diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb deleted file mode 100644 index a96e4c4b976..00000000000 --- a/lib/gitlab/graphql/global_id_compatibility.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module GlobalIDCompatibility - # TODO: remove this module once the compatibility layer is no longer needed. - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883 - def coerce_global_id_arguments!(args) - global_id_arguments = self.class.arguments.values.select do |arg| - arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType - end - - global_id_arguments.each do |arg| - k = arg.keyword - args[k] &&= arg.type.coerce_isolated_input(args[k]) - end - end - end - end -end diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index bf9b73d918a..9beb40ddd7e 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -68,7 +68,7 @@ module Gitlab def items original_items = super - return original_items if Feature.disabled?(:new_graphql_keyset_pagination, default_enabled: :yaml) || Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) + return original_items if Feature.disabled?(:new_graphql_keyset_pagination) || Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) strong_memoize(:generic_keyset_pagination_items) do rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items) diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index 5d3a9245427..cf06a2729d9 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -240,6 +240,9 @@ module Gitlab end end + # TODO: some queries live under app/graphql/queries - we should look there if/when we add fragments there + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/361079 + # for fragments too. class Fragments def initialize(root, dir = 'app/assets/javascripts') @root = root @@ -278,7 +281,7 @@ module Gitlab def self.all ['.', 'ee'].flat_map do |prefix| - find(Rails.root / prefix / 'app/assets/javascripts') + find(Rails.root / prefix / 'app/assets/javascripts') + find(Rails.root / prefix / 'app/graphql/queries') end end diff --git a/lib/gitlab/health_checks/middleware.rb b/lib/gitlab/health_checks/middleware.rb new file mode 100644 index 00000000000..3fe065147c8 --- /dev/null +++ b/lib/gitlab/health_checks/middleware.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + class Middleware + def initialize(app, readiness_probe, liveness_probe) + @app = app + @readiness_probe = readiness_probe + @liveness_probe = liveness_probe + end + + def call(env) + case env['PATH_INFO'] + when '/readiness' then render_probe(@readiness_probe) + when '/liveness' then render_probe(@liveness_probe) + else @app.call(env) + end + end + + private + + def render_probe(probe) + result = probe.execute + + [ + result.http_status, + { 'Content-Type' => 'application/json; charset=utf-8' }, + [result.json.to_json] + ] + end + end + end +end diff --git a/lib/gitlab/health_checks/server.rb b/lib/gitlab/health_checks/server.rb new file mode 100644 index 00000000000..d747b64a221 --- /dev/null +++ b/lib/gitlab/health_checks/server.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'webrick' + +module Gitlab + module HealthChecks + class Server < Daemon + def initialize(address:, port:, **options) + super(**options) + + @address = address + @port = port + end + + private + + def start_working + @server = ::WEBrick::HTTPServer.new( + Port: @port, BindAddress: @address, AccessLog: [] + ) + @server.mount '/', Rack::Handler::WEBrick, rack_app + + true + end + + def run_thread + @server&.start + rescue IOError + # ignore forcibily closed servers + end + + def stop_working + if @server + # we close sockets if thread is not longer running + # this happens, when the process forks + if thread.alive? + @server.shutdown + else + @server.listeners.each(&:close) + end + end + + @server = nil + end + + def rack_app + readiness = new_probe + liveness = new_probe + + Rack::Builder.app do + use Rack::Deflater + use HealthChecks::Middleware, readiness, liveness + run -> (env) { [404, {}, ['']] } + end + end + + def new_probe + ::Gitlab::HealthChecks::Probes::Collection.new + end + end + end +end diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index 44901297870..dd58f6aca26 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -12,7 +12,7 @@ module Gitlab def intercept_hotlinking?(request) request_accepts = parse_request_accepts(request) - return false unless Feature.enabled?(:repository_archive_hotlinking_interception, default_enabled: true) + return false unless Feature.enabled?(:repository_archive_hotlinking_interception) # Block attempts to embed as JS return true if sec_fetch_invalid?(request) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8b775d567c8..c8239c9e308 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,7 +43,7 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 44, + 'da_DK' => 43, 'de' => 14, 'en' => 100, 'eo' => 0, @@ -53,17 +53,17 @@ module Gitlab 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 34, + 'ja' => 33, 'ko' => 12, 'nb_NO' => 29, 'nl_NL' => 0, 'pl_PL' => 4, 'pt_BR' => 50, - 'ro_RO' => 36, - 'ru' => 31, + 'ro_RO' => 58, + 'ru' => 30, 'tr_TR' => 13, - 'uk' => 46, - 'zh_CN' => 97, + 'uk' => 47, + 'zh_CN' => 96, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 74be56df221..3ad01ef2257 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'securerandom' + module Gitlab module I18n class PoLinter @@ -245,16 +247,24 @@ module Gitlab [] elsif variables.any? { |variable| unnamed_variable?(variable) } variables.map do |variable| - variable == '%d' ? Random.rand(1000) : Gitlab::Utils.random_string + variable == '%d' ? random_number : random_string end else variables.each_with_object({}) do |variable, hash| variable_name = variable[/\w+/] - hash[variable_name] = Gitlab::Utils.random_string + hash[variable_name] = random_string end end end + def random_number + Random.rand(1000) + end + + def random_string + SecureRandom.alphanumeric(64) + end + def validate_unnamed_variables(errors, variables) unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) } diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 829b3771518..1878b5b1a30 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -28,7 +28,7 @@ module Gitlab copy_archive wait_for_archived_file do - validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: :yaml) + validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) decompress_archive end rescue StandardError => e diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index f7ab1677001..8df5d52bf77 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -16,6 +16,7 @@ tree: - :board - members: - :user + - :namespace_settings included_attributes: user: @@ -24,6 +25,8 @@ included_attributes: - :username author: - :name + namespace_settings: + - :prevent_sharing_groups_outside_hierarchy excluded_attributes: group: @@ -38,6 +41,7 @@ excluded_attributes: - :shared_runners_minute_limit - :extra_shared_runners_minutes_limit - :repository_size_limit + - :max_pages_size epics: - :state_id diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index adbbd37ce10..258078d595b 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -30,6 +30,10 @@ module Gitlab update_group_references end + def invalid_relation? + @relation_name == :namespace_settings + end + def update_group_references return unless self.class.existing_object_relations.include?(@relation_name) return unless @relation_hash['group_id'] diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index b44874f598c..4b28dd831fc 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -89,7 +89,7 @@ module Gitlab end def save_relation_object(relation_object, relation_key, relation_definition, relation_index) - if Feature.enabled?(:import_relation_object_persistence, default_enabled: :yaml) && relation_object.new_record? + if Feature.enabled?(:import_relation_object_persistence) && relation_object.new_record? Gitlab::ImportExport::Base::RelationObjectSaver.new( relation_object: relation_object, relation_key: relation_key, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 7dcf26ca89a..1625c39595c 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -48,6 +48,8 @@ tree: - :award_emoji - releases: - :links + - milestone_releases: + - :milestone - project_members: - :user - merge_requests: @@ -752,6 +754,7 @@ excluded_attributes: - :compliance_framework_setting - :show_default_award_emojis - :warn_about_potentially_unwanted_characters + - :enforce_auth_checks_on_uploads - :services - :exported_protected_branches - :repository_size_limit @@ -957,6 +960,9 @@ excluded_attributes: system_note_metadata: - :description_version_id - :note_id + milestone_releases: + - :milestone_id + - :release_id methods: notes: - :type diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index d8992061524..47f82a901b7 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -54,7 +54,7 @@ module Gitlab end def ndjson_relation_reader - return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true) + return unless Feature.enabled?(:project_import_ndjson, project.namespace) ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 63c5afa9595..05dcfa5282c 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -80,7 +80,7 @@ module Gitlab def json_writer @json_writer ||= begin - if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true) + if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) full_path = File.join(@shared.export_path, 'tree') Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) else diff --git a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb new file mode 100644 index 00000000000..f3f8e774b4b --- /dev/null +++ b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + class InactiveProjectsDeletionWarningTracker + attr_reader :project_id + + DELETION_TRACKING_REDIS_KEY = 'inactive_projects_deletion_warning_email_notified' + + # Redis key 'inactive_projects_deletion_warning_email_notified' is a hash. It stores the date when the + # deletion warning notification email was sent for an inactive project. The fields and values look like: + # {"project:1"=>"2022-04-22", "project:5"=>"2022-04-22", "project:7"=>"2022-04-25"} + # @return [Hash] + def self.notified_projects + Gitlab::Redis::SharedState.with do |redis| + redis.hgetall(DELETION_TRACKING_REDIS_KEY) + end + end + + def self.reset_all + Gitlab::Redis::SharedState.with do |redis| + redis.del(DELETION_TRACKING_REDIS_KEY) + end + end + + def initialize(project_id) + @project_id = project_id + end + + def notified? + Gitlab::Redis::SharedState.with do |redis| + redis.hexists(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + + def mark_notified + Gitlab::Redis::SharedState.with do |redis| + redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current) + end + end + + def reset + Gitlab::Redis::SharedState.with do |redis| + redis.hdel(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + end +end diff --git a/lib/gitlab/instrumentation/rate_limiting_gates.rb b/lib/gitlab/instrumentation/rate_limiting_gates.rb new file mode 100644 index 00000000000..960b6995030 --- /dev/null +++ b/lib/gitlab/instrumentation/rate_limiting_gates.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + class RateLimitingGates + GATES = :rate_limiting_gates + + class << self + def track(key) + if ::Gitlab::SafeRequestStore.active? + gates_set << key + end + end + + def gates + gates_set.to_a + end + + def payload + { + GATES => gates + } + end + + private + + def gates_set + ::Gitlab::SafeRequestStore[GATES] ||= Set.new + end + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 155e365d04c..379c27caeb7 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -32,6 +32,7 @@ module Gitlab instrument_load_balancing(payload) instrument_pid(payload) instrument_uploads(payload) + instrument_rate_limiting_gates(payload) end def instrument_gitaly(payload) @@ -121,6 +122,10 @@ module Gitlab payload.merge! ::Gitlab::Instrumentation::Uploads.payload end + def instrument_rate_limiting_gates(payload) + payload.merge!(::Gitlab::Instrumentation::RateLimitingGates.payload) + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb deleted file mode 100644 index f347db7bc8c..00000000000 --- a/lib/gitlab/integrations/sti_type.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Integrations - class StiType < ActiveRecord::Type::String - NAMESPACED_INTEGRATIONS = %w[ - Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost - MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker - Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao - ].to_set.freeze - - def self.namespaced_integrations - NAMESPACED_INTEGRATIONS - end - - def cast(value) - new_cast(value) || super - end - - def serialize(value) - new_serialize(value) || super - end - - def deserialize(value) - value - end - - def changed?(original_value, value, _new_value_before_type_cast) - original_value != serialize(value) - end - - def changed_in_place?(original_value_for_database, value) - original_value_for_database != serialize(value) - end - - private - - def new_cast(value) - value = prepare_value(value) - return unless value - - stripped_name = value.delete_suffix('Service') - return unless self.class.namespaced_integrations.include?(stripped_name) - - "Integrations::#{stripped_name}" - end - - def new_serialize(value) - value = prepare_value(value) - return unless value&.starts_with?('Integrations::') - - "#{value.delete_prefix('Integrations::')}Service" - end - - # Returns value cast to a `String`, or `nil` if value is `nil`. - def prepare_value(value) - return value if value.nil? || value.is_a?(String) - - value.to_s - end - end - end -end - -Gitlab::Integrations::StiType.prepend_mod diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/integrations_logger.rb index 9b0357d3161..c62a5f6d321 100644 --- a/lib/gitlab/project_service_logger.rb +++ b/lib/gitlab/integrations_logger.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - class ProjectServiceLogger < Gitlab::JsonLogger + class IntegrationsLogger < Gitlab::JsonLogger def self.file_name_noext 'integrations_json' end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 7abfe8e38e8..02b0c902a70 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -35,6 +35,12 @@ module Gitlab request_params[:base_uri] = uri.to_s request_params.merge!(auth_params) + if Feature.enabled?(:jira_raise_timeouts, type: :ops) + request_params[:open_timeout] = 2.minutes + request_params[:read_timeout] = 2.minutes + request_params[:write_timeout] = 2.minutes + end + result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend @authenticated = result.response.is_a?(Net::HTTPOK) store_cookies(result) if options[:use_cookies] diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 9824b46554f..512936bb4f4 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -160,7 +160,7 @@ module Gitlab # @raise [JSON::ParserError] def handle_legacy_mode!(data) return data unless feature_table_exists? - return data unless Feature.enabled?(:json_wrapper_legacy_mode, default_enabled: true) + return data unless Feature.enabled?(:json_wrapper_legacy_mode) raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) } end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index ed7787ffc49..bf7b7f2d089 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -33,6 +33,10 @@ module Gitlab @_version ||= Rails.root.join(VERSION_FILE).read.chomp end + def version_info + Gitlab::VersionInfo.parse(version) + end + # Return GitLab KAS external_url # # @return [String] external_url diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb deleted file mode 100644 index 8a31e068c30..00000000000 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - class CiliumNetworkPolicy - include NetworkPolicyCommon - extend ::Gitlab::Utils::Override - - API_VERSION = "cilium.io/v2" - KIND = 'CiliumNetworkPolicy' - - PREDEFINED_POLICIES = { - 'allow-inbound-http' => <<~YAML.rstrip, - apiVersion: cilium.io/v2 - kind: CiliumNetworkPolicy - metadata: - name: allow-inbound-http - spec: - endpointSelector: - matchLabels: - network-policy.gitlab.com/disabled_by: gitlab - ingress: - - toPorts: - - ports: - - port: '80' - - port: '443' - YAML - 'drop-outbound' => <<~YAML.rstrip - apiVersion: cilium.io/v2 - kind: CiliumNetworkPolicy - metadata: - name: drop-outbound - spec: - endpointSelector: - matchLabels: - network-policy.gitlab.com/disabled_by: gitlab - egress: - - {} - YAML - }.freeze - - # We are modeling existing kubernetes resource and don't have - # control over amount of parameters. - # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil, environment_ids: []) - @name = name - @description = description - @namespace = namespace - @labels = labels - @creation_timestamp = creation_timestamp - @selector = selector - @resource_version = resource_version - @ingress = ingress - @egress = egress - @annotations = annotations - @environment_ids = environment_ids - end - # rubocop:enable Metrics/ParameterLists - - def self.from_yaml(manifest) - return unless manifest - - policy = YAML.safe_load(manifest, symbolize_names: true) - return if !policy[:metadata] || !policy[:spec] - - metadata = policy[:metadata] - spec = policy[:spec] - self.new( - name: metadata[:name], - description: policy[:description], - namespace: metadata[:namespace], - annotations: metadata[:annotations], - resource_version: metadata[:resourceVersion], - labels: metadata[:labels], - selector: spec[:endpointSelector], - ingress: spec[:ingress], - egress: spec[:egress] - ) - rescue Psych::SyntaxError, Psych::DisallowedClass - nil - end - - def self.from_resource(resource, environment_ids = []) - return unless resource - return if !resource[:metadata] || !resource[:spec] - - metadata = resource[:metadata] - spec = resource[:spec].to_h - self.new( - name: metadata[:name], - description: resource[:description], - namespace: metadata[:namespace], - annotations: metadata[:annotations]&.to_h, - resource_version: metadata[:resourceVersion], - labels: metadata[:labels]&.to_h, - creation_timestamp: metadata[:creationTimestamp], - selector: spec[:endpointSelector], - ingress: spec[:ingress], - egress: spec[:egress], - environment_ids: environment_ids - ) - end - - override :resource - def resource - resource = { - apiVersion: API_VERSION, - kind: KIND, - metadata: metadata, - spec: spec - } - resource[:description] = description if description - resource - end - - private - - attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations, :environment_ids - - def selector - @selector ||= {} - end - - def metadata - meta = { name: name, namespace: namespace } - meta[:labels] = labels if labels - meta[:resourceVersion] = resource_version if resource_version - meta[:annotations] = annotations if annotations - meta - end - - def spec - { - endpointSelector: selector, - ingress: ingress, - egress: egress - }.compact - end - end - end -end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 6caebf445e5..cd03e332175 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -81,24 +81,6 @@ module Gitlab :update_gateway, to: :istio_client - # NetworkPolicy methods delegate to the apis/networking.k8s.io api - # group client - delegate :create_network_policy, - :get_network_policies, - :get_network_policy, - :update_network_policy, - :delete_network_policy, - to: :networking_client - - # CiliumNetworkPolicy methods delegate to the apis/cilium.io api - # group client - delegate :create_cilium_network_policy, - :get_cilium_network_policies, - :get_cilium_network_policy, - :update_cilium_network_policy, - :delete_cilium_network_policy, - to: :cilium_networking_client - attr_reader :api_prefix, :kubeclient_options DEFAULT_KUBECLIENT_OPTIONS = { diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb deleted file mode 100644 index e6111db5b17..00000000000 --- a/lib/gitlab/kubernetes/network_policy.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - class NetworkPolicy - include NetworkPolicyCommon - extend ::Gitlab::Utils::Override - - KIND = 'NetworkPolicy' - - # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil, environment_ids: []) - @name = name - @namespace = namespace - @labels = labels - @creation_timestamp = creation_timestamp - @selector = selector - @policy_types = policy_types - @ingress = ingress - @egress = egress - @environment_ids = environment_ids - end - # rubocop:enable Metrics/ParameterLists - - def self.from_yaml(manifest) - return unless manifest - - policy = YAML.safe_load(manifest, symbolize_names: true) - return if !policy[:metadata] || !policy[:spec] - - metadata = policy[:metadata] - spec = policy[:spec] - self.new( - name: metadata[:name], - namespace: metadata[:namespace], - labels: metadata[:labels], - selector: spec[:podSelector], - policy_types: spec[:policyTypes], - ingress: spec[:ingress], - egress: spec[:egress] - ) - rescue Psych::SyntaxError, Psych::DisallowedClass - nil - end - - def self.from_resource(resource, environment_ids = []) - return unless resource - return if !resource[:metadata] || !resource[:spec] - - metadata = resource[:metadata] - spec = resource[:spec].to_h - self.new( - name: metadata[:name], - namespace: metadata[:namespace], - labels: metadata[:labels]&.to_h, - creation_timestamp: metadata[:creationTimestamp], - selector: spec[:podSelector], - policy_types: spec[:policyTypes], - ingress: spec[:ingress], - egress: spec[:egress], - environment_ids: environment_ids - ) - end - - override :resource - def resource - { - kind: KIND, - metadata: metadata, - spec: spec - } - end - - private - - attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress, :environment_ids - - def selector - @selector ||= {} - end - - def metadata - meta = { name: name, namespace: namespace } - meta[:labels] = labels if labels - meta - end - - def spec - { - podSelector: selector, - policyTypes: policy_types, - ingress: ingress, - egress: egress - } - end - end - end -end diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb deleted file mode 100644 index de91833b734..00000000000 --- a/lib/gitlab/kubernetes/network_policy_common.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module NetworkPolicyCommon - DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by' - - def generate - ::Kubeclient::Resource.new(resource) - end - - def as_json(opts = nil) - { - name: name, - namespace: namespace, - creation_timestamp: creation_timestamp, - manifest: manifest, - is_autodevops: autodevops?, - is_enabled: enabled?, - environment_ids: environment_ids - } - end - - def autodevops? - return false unless labels - - !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-') - end - - # selector selects pods that should be targeted by this - # policy. It can represent podSelector, nodeSelector or - # endpointSelector We can narrow selection by requiring - # this policy to match our custom labels. Since DISABLED_BY - # label will not be on any pod a policy will be effectively disabled. - def enabled? - return true unless selector&.key?(:matchLabels) - - !selector[:matchLabels]&.key?(DISABLED_BY_LABEL) - end - - def enable - return if enabled? - - selector[:matchLabels].delete(DISABLED_BY_LABEL) - end - - def disable - selector[:matchLabels] ||= {} - selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab') - end - - private - - def resource - raise NotImplementedError - end - - def manifest - YAML.dump(resource.deep_stringify_keys) - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index 8a176be30a2..e2b43798b22 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -33,7 +33,6 @@ module Gitlab def import delete_stale_metrics create_or_update_metrics - update_prometheus_environments end # rubocop: disable CodeReuse/ActiveRecord @@ -47,8 +46,6 @@ module Gitlab affected_metric_ids << prometheus_metric.id end - - @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id end # rubocop: enable CodeReuse/ActiveRecord @@ -62,24 +59,9 @@ module Gitlab return unless stale_metrics.exists? - delete_stale_alerts(stale_metrics) stale_metrics.each_batch { |batch| batch.delete_all } end - def delete_stale_alerts(stale_metrics) - stale_alerts = find_alerts(stale_metrics) - - affected_environment_ids = stale_alerts.get_environment_id - return unless affected_environment_ids.present? - - @affected_environment_ids += affected_environment_ids - stale_alerts.each_batch { |batch| batch.delete_all } - end - - def find_alerts(metrics) - Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute - end - def prometheus_metrics_attributes @prometheus_metrics_attributes ||= begin Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( @@ -89,19 +71,6 @@ module Gitlab ).execute end end - - def update_prometheus_environments - affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project) - - return unless affected_environments.exists? - - affected_environments.each do |affected_environment| - ::Clusters::Applications::ScheduleUpdateService.new( - affected_environment.cluster_prometheus_adapter, - project - ).execute - end - end end end end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 2aea8d655fa..ba2eb729d7b 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -71,28 +71,17 @@ module Gitlab end def rack_app - readiness = readiness_probe - liveness = liveness_probe pid = thread_name gc_requests = @gc_requests Rack::Builder.app do use Rack::Deflater use Gitlab::Metrics::Exporter::MetricsMiddleware, pid - use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness use Gitlab::Metrics::Exporter::GcRequestMiddleware if gc_requests use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present? run -> (env) { [404, {}, ['']] } end end - - def readiness_probe - ::Gitlab::HealthChecks::Probes::Collection.new - end - - def liveness_probe - ::Gitlab::HealthChecks::Probes::Collection.new - end end end end diff --git a/lib/gitlab/metrics/exporter/health_checks_middleware.rb b/lib/gitlab/metrics/exporter/health_checks_middleware.rb deleted file mode 100644 index c43b8004b72..00000000000 --- a/lib/gitlab/metrics/exporter/health_checks_middleware.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Exporter - class HealthChecksMiddleware - def initialize(app, readiness_probe, liveness_probe) - @app = app - @readiness_probe = readiness_probe - @liveness_probe = liveness_probe - end - - def call(env) - case env['PATH_INFO'] - when '/readiness' then render_probe(@readiness_probe) - when '/liveness' then render_probe(@liveness_probe) - else @app.call(env) - end - end - - private - - def render_probe(probe) - result = probe.execute - - [ - result.http_status, - { 'Content-Type' => 'application/json; charset=utf-8' }, - [result.json.to_json] - ] - end - end - end - end -end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index dc9a7ed1312..0aad865085b 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -56,7 +56,8 @@ module Gitlab end def disabled_by_feature(options) - options.with_feature && !::Feature.enabled?(options.with_feature, type: :ops) + options.with_feature && !::Feature.enabled?(options.with_feature, + type: :undefined, default_enabled_if_undefined: false) end def build_metric!(type, name, options) diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb index c4f305dbdc4..71da0085c8c 100644 --- a/lib/gitlab/metrics/rails_slis.rb +++ b/lib/gitlab/metrics/rails_slis.rb @@ -5,16 +5,16 @@ module Gitlab module RailsSlis class << self def initialize_request_slis! - Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels) unless Gitlab::Metrics::Sli.initialized?(:rails_request_apdex) - Gitlab::Metrics::Sli.initialize_sli(:graphql_query_apdex, possible_graphql_query_labels) unless Gitlab::Metrics::Sli.initialized?(:graphql_query_apdex) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:rails_request, possible_request_labels) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:graphql_query, possible_graphql_query_labels) end def request_apdex - Gitlab::Metrics::Sli[:rails_request_apdex] + Gitlab::Metrics::Sli::Apdex[:rails_request] end def graphql_query_apdex - Gitlab::Metrics::Sli[:graphql_query_apdex] + Gitlab::Metrics::Sli::Apdex[:graphql_query] end private diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb index de73db0755d..fcd893b675f 100644 --- a/lib/gitlab/metrics/sli.rb +++ b/lib/gitlab/metrics/sli.rb @@ -2,12 +2,10 @@ module Gitlab module Metrics - class Sli - SliNotInitializedError = Class.new(StandardError) - + module Sli COUNTER_PREFIX = 'gitlab_sli' - class << self + module ClassMethods INITIALIZATION_MUTEX = Mutex.new def [](name) @@ -16,6 +14,8 @@ module Gitlab def initialize_sli(name, possible_label_combinations) INITIALIZATION_MUTEX.synchronize do + next known_slis[name] if initialized?(name) + sli = new(name) sli.initialize_counters(possible_label_combinations) known_slis[name] = sli @@ -33,6 +33,10 @@ module Gitlab end end + def self.included(mod) + mod.extend(ClassMethods) + end + attr_reader :name def initialize(name) @@ -41,16 +45,17 @@ module Gitlab end def initialize_counters(possible_label_combinations) - @initialized_with_combinations = possible_label_combinations.any? + # This module is effectively an abstract class + @initialized_with_combinations = possible_label_combinations.any? # rubocop:disable Gitlab/ModuleWithInstanceVariables possible_label_combinations.each do |label_combination| total_counter.get(label_combination) - success_counter.get(label_combination) + numerator_counter.get(label_combination) end end - def increment(labels:, success:) + def increment(labels:, increment_numerator:) total_counter.increment(labels) - success_counter.increment(labels) if success + numerator_counter.increment(labels) if increment_numerator end def initialized? @@ -60,23 +65,43 @@ module Gitlab private def total_counter - prometheus.counter(total_counter_name.to_sym, "Total number of measurements for #{name}") + prometheus.counter(counter_name('total'), "Total number of measurements for #{name}") end - def success_counter - prometheus.counter(success_counter_name.to_sym, "Number of successful measurements for #{name}") + def counter_name(suffix) + :"#{COUNTER_PREFIX}:#{name}_#{self.class.name.demodulize.underscore}:#{suffix}" end - def total_counter_name - "#{COUNTER_PREFIX}:#{name}:total" + def prometheus + Gitlab::Metrics end - def success_counter_name - "#{COUNTER_PREFIX}:#{name}:success_total" + class Apdex + include Sli + + def increment(labels:, success:) + super(labels: labels, increment_numerator: success) + end + + private + + def numerator_counter + prometheus.counter(counter_name('success_total'), "Number of successful measurements for #{name}") + end end - def prometheus - Gitlab::Metrics + class ErrorRate + include Sli + + def increment(labels:, error:) + super(labels: labels, increment_numerator: error) + end + + private + + def numerator_counter + prometheus.counter(counter_name('error_total'), "Number of error measurements for #{name}") + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 12576cabb19..7c22ce64ea2 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -185,15 +185,17 @@ module Gitlab counters << compose_metric_key(metric, role) end - ::Gitlab::Database.db_config_names.each do |config_name| - counters << compose_metric_key(metric, nil, config_name) # main - counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica + ::Gitlab::Database.database_base_models.keys.each do |config_name| + counters << compose_metric_key(metric, nil, config_name) # main / ci + counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica / ci_replica end end counters end + private_class_method :load_balancing_metric_keys + def compose_metric_key(metric, db_role = nil, db_config_name = nil) self.class.compose_metric_key(metric, db_role, db_config_name) end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 70dcc6fad90..e6cf14a6c8c 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -15,22 +15,6 @@ module Gitlab INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation - THROTTLES_WITH_USER_INFORMATION = [ - :throttle_authenticated_api, - :throttle_authenticated_web, - :throttle_authenticated_protected_paths_api, - :throttle_authenticated_protected_paths_web, - :throttle_authenticated_packages_api, - :throttle_authenticated_git_lfs, - :throttle_authenticated_files_api, - :throttle_authenticated_deprecated_api - ].freeze - - PAYLOAD_KEYS = [ - :rack_attack_redis_count, - :rack_attack_redis_duration_s - ].freeze - def self.payload Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { rack_attack_redis_count: 0, @@ -49,20 +33,20 @@ module Gitlab end def throttle(event) - log_into_auth_logger(event) + log_into_auth_logger(event, status: 429) end def blocklist(event) - log_into_auth_logger(event) + log_into_auth_logger(event, status: 403) end def track(event) - log_into_auth_logger(event) + log_into_auth_logger(event, status: nil) end private - def log_into_auth_logger(event) + def log_into_auth_logger(event, status:) req = event.payload[:request] rack_attack_info = { message: 'Rack_Attack', @@ -73,6 +57,10 @@ module Gitlab matched: req.env['rack.attack.matched'] } + if status + rack_attack_info[:status] = status + end + discriminator = req.env['rack.attack.match_discriminator'].to_s discriminator_id = discriminator.split(':').last diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index bfa4e4cf5f8..dcbb4557377 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -21,6 +21,7 @@ module Gitlab rescue Gitlab::Auth::IpBlacklisted Gitlab::AuthLogger.error( message: 'Rack_Attack', + status: 403, env: :blocklist, remote_ip: request.ip, request_method: request.request_method, diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 51277497c99..b78cd2a6b95 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -136,8 +136,6 @@ module Gitlab def setup_provider(provider) case provider - when :kerberos - require 'omniauth-kerberos' when *omniauth_customized_providers require_dependency "omni_auth/strategies/#{provider}" end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index e76cab688cc..8bbc9a93610 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -30,11 +30,11 @@ module Gitlab return false unless params[:pagination] == "keyset" if finder.is_a?(BranchesFinder) - Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:tag_list_keyset_pagination, project) elsif finder.is_a?(::Repositories::TreeFinder) - Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml) + Feature.enabled?(:repository_tree_gitaly_pagination, project) else false end @@ -44,11 +44,11 @@ module Gitlab return false unless params[:page].blank? || params[:page].to_i == 1 if finder.is_a?(BranchesFinder) - Feature.enabled?(:branch_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project, default_enabled: :yaml) + Feature.enabled?(:tag_list_keyset_pagination, project) elsif finder.is_a?(::Repositories::TreeFinder) - Feature.enabled?(:repository_tree_gitaly_pagination, project, default_enabled: :yaml) + Feature.enabled?(:repository_tree_gitaly_pagination, project) else false end diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb index 702e8d404b1..c5c73d50518 100644 --- a/lib/gitlab/patch/database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -31,10 +31,6 @@ module Gitlab module DatabaseConfig extend ActiveSupport::Concern - prepended do - attr_reader :uses_legacy_database_config - end - def load_database_yaml return super unless Gitlab.ee? @@ -70,24 +66,7 @@ module Gitlab end def database_configuration - @uses_legacy_database_config = false # rubocop:disable Gitlab/ModuleWithInstanceVariables - super.to_h do |env, configs| - # TODO: To be removed in 15.0. See https://gitlab.com/gitlab-org/gitlab/-/issues/338182 - # This preload is needed to convert legacy `database.yml` - # from `production: adapter: postgresql` - # into a `production: main: adapter: postgresql` - unless Gitlab::Utils.to_boolean(ENV['SKIP_DATABASE_CONFIG_VALIDATION'], default: false) - # This check is taken from Rails where the transformation - # of a flat database.yml is done into `primary:` - # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 - if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } - configs = { "main" => configs } - - @uses_legacy_database_config = true # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - end - if Gitlab.ee? if !configs.key?("geo") && File.exist?(Rails.root.join("config/database_geo.yml")) configs["geo"] = Rails.application.config_for(:database_geo).stringify_keys diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 6f497c6376d..b05d7160a4b 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -23,7 +23,6 @@ module Gitlab 503.html admin api - apple-touch-icon-precomposed.png apple-touch-icon.png assets dashboard diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb index 4c9d54a93ce..3885a9934d5 100644 --- a/lib/gitlab/phabricator_import.rb +++ b/lib/gitlab/phabricator_import.rb @@ -5,7 +5,7 @@ module Gitlab BaseError = Class.new(StandardError) def self.available? - Feature.enabled?(:phabricator_import, default_enabled: :yaml) && + Feature.enabled?(:phabricator_import) && Gitlab::CurrentSettings.import_sources.include?('phabricator') end end diff --git a/lib/gitlab/process_supervisor.rb b/lib/gitlab/process_supervisor.rb index 18fd24aa582..714034f043d 100644 --- a/lib/gitlab/process_supervisor.rb +++ b/lib/gitlab/process_supervisor.rb @@ -20,7 +20,7 @@ module Gitlab health_check_interval_seconds: DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS, check_terminate_interval_seconds: DEFAULT_TERMINATE_INTERVAL_SECONDS, terminate_timeout_seconds: DEFAULT_TERMINATE_TIMEOUT_SECONDS, - term_signals: %i(INT TERM), + term_signals: [], forwarded_signals: [], **options) super(**options) @@ -31,7 +31,7 @@ module Gitlab @check_terminate_interval_seconds = check_terminate_interval_seconds @terminate_timeout_seconds = terminate_timeout_seconds - @pids = [] + @pids = Set.new @alive = false end @@ -43,7 +43,7 @@ module Gitlab # If the block returns a non-empty list of IDs, the supervisor will # start observing those processes instead. Otherwise it will shut down. def supervise(pid_or_pids, &on_process_death) - @pids = Array(pid_or_pids) + @pids = Array(pid_or_pids).to_set @on_process_death = on_process_death trap_signals! @@ -56,7 +56,6 @@ module Gitlab return unless @alive stop_processes(signal) - stop end def supervised_pids @@ -75,26 +74,25 @@ module Gitlab def run_thread while @alive - sleep(@health_check_interval_seconds) - check_process_health + + sleep(@health_check_interval_seconds) end end def check_process_health unless all_alive? - existing_pids = live_pids # Capture this value for the duration of the block. + existing_pids = live_pids.to_set # Capture this value for the duration of the block. dead_pids = @pids - existing_pids - new_pids = Array(@on_process_death.call(dead_pids)) - @pids = existing_pids + new_pids - @alive = @pids.any? + new_pids = Array(@on_process_death.call(dead_pids.to_a)) + @pids = existing_pids + new_pids.to_set end end def stop_processes(signal) # Set this prior to shutting down so that shutdown hooks which read `alive` # know the supervisor is about to shut down. - @alive = false + stop_working # Shut down supervised processes. signal_all(signal) diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 3a5f1a1d480..d15b57eb888 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -16,7 +16,6 @@ module Gitlab lib/gitlab/middleware/ ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ - lib/gitlab/request_profiler/ lib/gitlab/query_limiting/ lib/gitlab/tracing/ lib/gitlab/profiler.rb diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 9d954a74948..8a1dcc083e8 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -8,6 +8,7 @@ module Gitlab :assign, :create, :description, + :draft, :label, :merge_when_pipeline_succeeds, :milestone, diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 4bfd526914b..49f76ce7814 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -8,7 +8,7 @@ module Gitlab def sql(event) return if !::Gitlab::QueryLimiting::Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE') - ::Gitlab::QueryLimiting::Transaction.current.increment + ::Gitlab::QueryLimiting::Transaction.current.increment(event.payload[:sql]) ::Gitlab::QueryLimiting::Transaction.current.executed_sql(event.payload[:sql]) end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 643b2540c37..2e31849caaa 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -57,12 +57,28 @@ module Gitlab raise(error) if raise_error? end - def increment - @count += 1 if enabled? + def increment(sql = nil) + @count += 1 if enabled? && !ignorable?(sql) + end + + GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1' + LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"' + ATTR_INTROSPECTION = %r/SELECT .*\ba.attname\b.* (FROM|JOIN) pg_attribute a/m.freeze + + # queries can be safely ignored if they are amoritized in regular usage + # (i.e. only requested occasionally and otherwise cached). + def ignorable?(sql) + return true if sql&.include?(GEO_NODES_LOAD) + return true if sql&.include?(LICENSES_LOAD) + return true if ATTR_INTROSPECTION =~ sql + + false end def executed_sql(sql) - @sql_executed << sql if @count <= LOG_THRESHOLD + return if @count > LOG_THRESHOLD || ignorable?(sql) + + @sql_executed << sql end def raise_error? diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 4efa29337d1..abf55f56c73 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -285,7 +285,7 @@ module Gitlab end types MergeRequest condition do - Feature.enabled?(:mr_attention_requests, project, default_enabled: :yaml) && + current_user.mr_attention_requests_enabled? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |attention_param| @@ -321,7 +321,7 @@ module Gitlab end types MergeRequest condition do - Feature.enabled?(:mr_attention_requests, project, default_enabled: :yaml) && + current_user.mr_attention_requests_enabled? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |attention_param| diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index e4a92ed5122..7ccbeadfd8a 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -10,10 +10,6 @@ module Gitlab @expires_in = expires_in end - def cache_key(key) - super(key) - end - def clear_cache!(key) with do |redis| keys = read(key).map { |value| "#{cache_namespace}:#{value}" } diff --git a/lib/gitlab/repository_archive_rate_limiter.rb b/lib/gitlab/repository_archive_rate_limiter.rb index d395b1aba7f..31a3dc34bf6 100644 --- a/lib/gitlab/repository_archive_rate_limiter.rb +++ b/lib/gitlab/repository_archive_rate_limiter.rb @@ -3,7 +3,7 @@ module Gitlab module RepositoryArchiveRateLimiter def check_archive_rate_limit!(current_user, project, &block) - return unless Feature.enabled?(:archive_rate_limit, default_enabled: :yaml) + return unless Feature.enabled?(:archive_rate_limit) threshold = current_user ? nil : 100 diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb deleted file mode 100644 index 541d505e735..00000000000 --- a/lib/gitlab/request_profiler.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require 'fileutils' - -module Gitlab - module RequestProfiler - PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles" - - def all - Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| - Profile.new(File.basename(path)) - end.select(&:valid?) - end - module_function :all # rubocop: disable Style/AccessModifierDeclarations - - def find(name) - file_path = File.join(PROFILES_DIR, name) - return unless File.exist?(file_path) - - Profile.new(name) - end - module_function :find # rubocop: disable Style/AccessModifierDeclarations - - def profile_token - Rails.cache.fetch('profile-token') do - Devise.friendly_token - end - end - module_function :profile_token # rubocop: disable Style/AccessModifierDeclarations - - def remove_all_profiles - FileUtils.rm_rf(PROFILES_DIR) - end - module_function :remove_all_profiles # rubocop: disable Style/AccessModifierDeclarations - end -end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb deleted file mode 100644 index acdf8d4541f..00000000000 --- a/lib/gitlab/request_profiler/middleware.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -require 'ruby-prof' -require 'memory_profiler' - -module Gitlab - module RequestProfiler - class Middleware - def initialize(app) - @app = app - end - - def call(env) - if profile?(env) - call_with_profiling(env) - else - @app.call(env) - end - end - - def profile?(env) - header_token = env['HTTP_X_PROFILE_TOKEN'] - return unless header_token.present? - - profile_token = Gitlab::RequestProfiler.profile_token - return unless profile_token.present? - - header_token == profile_token - end - - def call_with_profiling(env) - case env['HTTP_X_PROFILE_MODE'] - when 'execution', nil - call_with_call_stack_profiling(env) - when 'memory' - call_with_memory_profiling(env) - else - raise ActionController::BadRequest, invalid_profile_mode(env) - end - end - - def invalid_profile_mode(env) - <<~HEREDOC - Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}. - Supported profile mode request header: - - X-Profile-Mode: execution - - X-Profile-Mode: memory - HEREDOC - end - - def call_with_call_stack_profiling(env) - ret = nil - report = RubyProf::Profile.profile do - ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow - @app.call(env) - end - end - - generate_report(env, 'execution', 'html') do |file| - printer = RubyProf::CallStackPrinter.new(report) - printer.print(file) - end - - handle_request_ret(ret) - end - - def call_with_memory_profiling(env) - ret = nil - report = MemoryProfiler.report do - ret = catch(:warden) do # rubocop:disable Cop/BanCatchThrow - @app.call(env) - end - end - - generate_report(env, 'memory', 'txt') do |file| - report.pretty_print(to_file: file) - end - - handle_request_ret(ret) - end - - def generate_report(env, report_type, extension) - file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\ - "_#{report_type}.#{extension}" - file_path = "#{PROFILES_DIR}/#{file_name}" - - FileUtils.mkdir_p(PROFILES_DIR) - - begin - File.open(file_path, 'wb') do |file| - yield(file) - end - rescue StandardError - FileUtils.rm(file_path) - end - end - - def handle_request_ret(ret) - if ret.is_a?(Array) - ret - else - throw(:warden, ret) # rubocop:disable Cop/BanCatchThrow - end - end - end - end -end diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb deleted file mode 100644 index 76c675658b1..00000000000 --- a/lib/gitlab/request_profiler/profile.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RequestProfiler - class Profile - attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type - - alias_method :to_param, :name - - def initialize(name) - @name = name - @file_path = File.join(PROFILES_DIR, name) - - set_attributes - end - - def valid? - @request_path.present? - end - - def content_type - case type - when 'html' - 'text/html' - when 'txt' - 'text/plain' - end - end - - private - - def set_attributes - matches = name.match(/^(?<path>.*)_(?<timestamp>\d+)(_(?<profile_mode>\w+))?\.(?<type>html|txt)$/) - return unless matches - - @request_path = matches[:path].tr('|', '/') - @time = Time.at(matches[:timestamp].to_i).utc - @profile_mode = matches[:profile_mode] || 'unknown' - @type = matches[:type] - end - end - end -end diff --git a/lib/gitlab/safe_request_purger.rb b/lib/gitlab/safe_request_purger.rb new file mode 100644 index 00000000000..b8795f1cc88 --- /dev/null +++ b/lib/gitlab/safe_request_purger.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + class SafeRequestPurger + def self.execute(args) + new(**args).execute + end + + def initialize(resource_key:, resource_ids:) + @resource_key = resource_key + @resource_ids = resource_ids.uniq + @resource_data = {} + end + + def execute + load_resource_data + purge_resource_ids + write_resource_data_to_store + end + + private + + attr_reader :resource_key, :resource_ids, :resource_data + + def load_resource_data + @resource_data = Gitlab::SafeRequestStore.fetch(resource_key) { resource_data } + end + + def purge_resource_ids + @resource_data.delete_if { |id| resource_ids.include?(id) } + end + + def write_resource_data_to_store + Gitlab::SafeRequestStore.write(resource_key, resource_data) + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index a498e329c3f..1e42003b203 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -149,28 +149,47 @@ module Gitlab module Praefect extend Gitlab::SetupHelper class << self - def configuration_toml(gitaly_dir, _, _) + def configuration_toml(gitaly_dir, _storage_paths, options) + raise 'This configuration is only intended for test' unless Rails.env.test? + nodes = [{ storage: 'default', address: "unix:#{gitaly_dir}/gitaly.socket", primary: true, token: 'secret' }] second_storage_nodes = [{ storage: 'test_second_storage', address: "unix:#{gitaly_dir}/gitaly2.socket", primary: true, token: 'secret' }] storages = [{ name: 'default', node: nodes }, { name: 'test_second_storage', node: second_storage_nodes }] - failover = { enabled: false, election_strategy: 'local' } + config = { - i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning: true, socket_path: "#{gitaly_dir}/praefect.socket", - memory_queue_enabled: true, virtual_storage: storages, - failover: failover + token: 'secret' } - config[:token] = 'secret' if Rails.env.test? + + if options[:per_repository] + failover = { enabled: true, election_strategy: 'per_repository' } + database = { host: options.fetch(:pghost), + port: options.fetch(:pgport).to_i, + user: options.fetch(:pguser), + dbname: options.fetch(:dbname, 'praefect_test') } + + config.merge!(database: database, + failover: failover) + else + failover = { enabled: false, election_strategy: 'local' } + + config.merge!( + i_understand_my_election_strategy_is_unsupported_and_will_be_removed_without_warning: true, + memory_queue_enabled: true, + failover: failover + ) + end TomlRB.dump(config) end private - def get_config_path(dir, _) - File.join(dir, 'praefect.config.toml') + def get_config_path(dir, options) + config_filename = options[:config_filename] || 'praefect.config.toml' + File.join(dir, config_filename) end end end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 3eef41a2ca2..ac9a7d25fc2 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -141,6 +141,20 @@ module Gitlab .to_h end + # Get the list of queues from all available workers following queue + # routing rules. Sidekiq::Queue.all fetches the list of queues from Redis. + # It may contain some redundant, obsolete queues from previous iterations + # of GitLab. + def routing_queues + @routing_queues ||= workers.map do |worker| + if worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) + worker.queue + else + ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass) + end + end.uniq.sort + end + private def find_workers(root, ee:, jh:) diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index 8a2ea1acaab..ba1f2b8d2ab 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -22,6 +22,10 @@ module Gitlab @attributes[:queue] end + def queue + @attributes[:queue] + end + def queue_namespace nil end diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 1e3fb675ca7..1abdcde6a15 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -9,7 +9,7 @@ module Gitlab delegate :feature_category_not_owned?, :generated_queue_name, :get_feature_category, :get_sidekiq_options, :get_tags, :get_urgency, :get_weight, - :get_worker_resource_boundary, :idempotent?, :queue_namespace, + :get_worker_resource_boundary, :idempotent?, :queue_namespace, :queue, :worker_has_external_dependencies?, to: :klass diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index f3e1d0af2aa..dc5481289da 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -43,7 +43,7 @@ module Gitlab metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) - return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize, default_enabled: :yaml) + return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize) ::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue| worker_class = worker.safe_constantize diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 581d6b738f3..b77f48d1a2c 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -4,10 +4,6 @@ module Gitlab class SnippetSearchResults < SearchResults include SnippetsHelper - def initialize(current_user, query) - super(current_user, query) - end - def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) paginated_objects(snippet_titles, page, per_page) end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index 892c4468107..8369e6fbe9b 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -15,7 +15,7 @@ module Gitlab def feature_enabled?(actor = nil) # Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around. - Feature.enabled?(:sourcegraph, actor, default_enabled: :yaml) + Feature.enabled?(:sourcegraph, actor) end private diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index b8d124541f9..7ef1be6ff44 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -18,6 +18,10 @@ module Gitlab "#{self.subscriptions_url}/payment_forms/cc_validation" end + def self.payment_validation_form_id + "payment_method_validation" + end + def self.registration_validation_form_url "#{self.subscriptions_url}/payment_forms/cc_registration_validation" end @@ -75,7 +79,7 @@ module Gitlab end def self.renewal_service_email - 'renewals-support@gitlab.com' + 'renewals-service@customers.gitlab.com' end end end @@ -83,5 +87,6 @@ end Gitlab::SubscriptionPortal.prepend_mod Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze +Gitlab::SubscriptionPortal::PAYMENT_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.payment_validation_form_id.freeze Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 323f59d3373..5496bd5f682 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -4,8 +4,7 @@ module Gitlab module Template class GitlabCiYmlTemplate < BaseTemplate BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze - - TEMPLATES_WITH_LATEST_VERSION = {}.freeze + BASE_DIR = 'lib/gitlab/ci/templates' def description "# This file is a template, and might need editing before it works on your project." @@ -45,7 +44,7 @@ module Gitlab end def base_dir - Rails.root.join('lib/gitlab/ci/templates') + Rails.root.join(BASE_DIR) end def finder(project = nil) @@ -57,31 +56,6 @@ module Gitlab excluded_patterns: self.excluded_patterns ) end - - override :find - def find(key, project = nil) - if try_redirect_to_latest?(key, project) - key += '.latest' - end - - super(key, project) - end - - private - - # To gauge the impact of the latest template, - # you can redirect the stable template to the latest template by enabling the feature flag. - # See https://docs.gitlab.com/ee/development/cicd/templates.html#versioning for more information. - def try_redirect_to_latest?(key, project) - return false unless templates_with_latest_version[key] - - flag_name = "redirect_to_latest_template_#{key.underscore.tr('/', '_')}" - ::Feature.enabled?(flag_name, project, default_enabled: :yaml) - end - - def templates_with_latest_version - TEMPLATES_WITH_LATEST_VERSION - end end end end diff --git a/lib/gitlab/testing/clear_process_memory_cache_middleware.rb b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb index 1e69e5e142d..39bcad271be 100644 --- a/lib/gitlab/testing/clear_process_memory_cache_middleware.rb +++ b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb @@ -11,6 +11,8 @@ module Gitlab Gitlab::ProcessMemoryCache.cache_backend.clear @app.call(env) + ensure + Gitlab::ProcessMemoryCache.cache_backend.clear end end end diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb index 8f70c8ecab7..928eb6338f6 100644 --- a/lib/gitlab/tracking/event_definition.rb +++ b/lib/gitlab/tracking/event_definition.rb @@ -6,7 +6,6 @@ module Gitlab class EventDefinition EVENT_SCHEMA_PATH = Rails.root.join('config', 'events', 'schema.json') - BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SCHEMA = ::JSONSchemer.schema(Pathname.new(EVENT_SCHEMA_PATH)) attr_reader :path diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index c0730e7bd59..96e74f00c78 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -61,9 +61,9 @@ module Gitlab def self.with_fallback(pattern, multiline: false) UntrustedRegexp.new(pattern, multiline: multiline) rescue RegexpError - raise if Feature.enabled?(:disable_unsafe_regexp, default_enabled: :yaml) + raise if Feature.enabled?(:disable_unsafe_regexp) - if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) + if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops) Gitlab::AppJsonLogger.info( class: self.name, regexp: pattern.to_s, diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index c00a0e1bcb4..a6d6cffec17 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -28,6 +28,8 @@ module Gitlab compare_url(object, **options) when Group instance.group_canonical_url(object, **options) + when WorkItem + instance.work_item_url(object, **options) when Issue instance.issue_url(object, **options) when MergeRequest diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index 24e044c5740..cf48aa49938 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -18,19 +18,25 @@ module Gitlab end def with_value - unflatten_key_path(intrumentation_object.value) + with_availability(proc { instrumentation_object.value }) end def with_instrumentation - unflatten_key_path(intrumentation_object.instrumentation) + with_availability(proc { instrumentation_object.instrumentation }) end def with_suggested_name - unflatten_key_path(intrumentation_object.suggested_name) + with_availability(proc { instrumentation_object.suggested_name }) end private + def with_availability(value_proc) + return {} unless instrumentation_object.available? + + unflatten_key_path(value_proc.call) + end + def unflatten_key_path(value) ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value) end @@ -39,8 +45,8 @@ module Gitlab "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" end - def intrumentation_object - instrumentation_class.constantize.new( + def instrumentation_object + @instrumentation_object ||= instrumentation_class.constantize.new( time_frame: definition.time_frame, options: definition.attributes[:options] ) diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 1031f38792b..2c50678c6bf 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,9 +4,9 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') - BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated].freeze + AVAILABLE_STATUSES = %w[active data_available implemented deprecated].to_set.freeze + VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze InvalidError = Class.new(RuntimeError) @@ -26,20 +26,22 @@ module Gitlab attributes end + def json_schema + return unless has_json_schema? + + @json_schema ||= Gitlab::Json.parse(File.read(json_schema_path)) + end + def json_schema_path return '' unless has_json_schema? - "#{BASE_REPO_PATH}/#{attributes[:value_json_schema]}" + Rails.root.join(attributes[:value_json_schema]) end def has_json_schema? attributes[:value_type] == 'object' && attributes[:value_json_schema].present? end - def yaml_path - "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" - end - def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| @@ -64,6 +66,10 @@ module Gitlab AVAILABLE_STATUSES.include?(attributes[:status]) end + def valid_service_ping_status? + VALID_SERVICE_PING_STATUSES.include?(attributes[:status]) + end + alias_method :to_dictionary, :to_h class << self diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 2545a505984..11e2fd22638 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -30,7 +30,7 @@ module Gitlab def aggregated_metrics_data(start_date:, end_date:, time_frame:) aggregated_metrics.each_with_object({}) do |aggregation, data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) next unless aggregation[:time_frame].include?(time_frame) case aggregation[:source] diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index a264f9484f3..f76ed1753b2 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -11,6 +11,18 @@ module Gitlab attr_reader :time_frame attr_reader :options + class << self + def available?(&block) + return @metric_available = block if block_given? + + return @metric_available.call if instance_variable_defined?('@metric_available') + + true + end + + attr_reader :metric_available + end + def initialize(time_frame:, options: {}) @time_frame = time_frame @options = options @@ -19,6 +31,10 @@ module Gitlab def instrumentation value end + + def available? + self.class.available? + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb index 6df6fef5d3a..d42250c9297 100644 --- a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class CertBasedClustersFfMetric < GenericMetric value do - Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + Feature.enabled?(:certificate_based_clusters, type: :ops) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb index ee51180973c..51be4bf3ccf 100644 --- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class CollectedDataCategoriesMetric < GenericMetric value do - ::ServicePing::PermitDataCategoriesService.new.execute.to_a + ::ServicePing::PermitDataCategories.new.execute.to_a end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb new file mode 100644 index 00000000000..c0d53b1b21a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountBulkImportsEntitiesMetric < DatabaseMetric + operation :count + + def initialize(time_frame:, options: {}) + super + + if source_type.present? && !source_type.in?(allowed_source_types) + raise ArgumentError, "source_type '#{source_type}' must be one of: #{allowed_source_types.join(', ')}" + end + end + + relation { ::BulkImports::Entity } + + private + + def relation + return super.where(source_type: source_type) if source_type.present? # rubocop: disable CodeReuse/ActiveRecord + + super + end + + def source_type + options[:source_type].to_s + end + + def allowed_source_types + BulkImports::Entity.source_types.keys.map(&:to_s) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb new file mode 100644 index 00000000000..c5498ce530f --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountImportedProjectsMetric < DatabaseMetric + operation :count + + def initialize(time_frame:, options: {}) + super + + raise ArgumentError, "import_type options attribute is required" unless import_type.present? + end + + relation { ::Project } + + start do |time_constraints| + unless time_constraints.nil? + start = time_constraints[:created_at]&.first + + unless start.nil? + ::Project + .select(:id) + .where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable + .order(created_at: :asc).limit(1).first&.id + end + end + end + + finish do |time_constraints| + unless time_constraints.nil? + finish = time_constraints[:created_at]&.last + + unless finish.nil? + ::Project + .select(:id) + .where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable + .order(created_at: :desc).limit(1).first&.id + end + end + end + + private + + def relation + super.imported_from(import_type) # rubocop: disable CodeReuse/ActiveRecord + end + + def import_type + options[:import_type] + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 34a8bfd08b5..a000b4509c6 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -14,7 +14,14 @@ module Gitlab # ::Issue.where(database_time_constraints) # end # end + + UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass + class << self + IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count).freeze + + private_constant :IMPLEMENTED_OPERATIONS + def start(&block) return @metric_start&.call unless block_given? @@ -40,6 +47,8 @@ module Gitlab end def operation(symbol, column: nil, &block) + raise UnimplementedOperationError unless symbol.in?(IMPLEMENTED_OPERATIONS) + @metric_operation = symbol @column = column @metric_operation_block = block if block_given? @@ -82,6 +91,14 @@ module Gitlab private + def start + self.class.metric_start&.call(time_constraints) + end + + def finish + self.class.metric_finish&.call(time_constraints) + end + def relation self.class.metric_relation.call.where(time_constraints) end @@ -100,19 +117,19 @@ module Gitlab end def get_or_cache_batch_ids - return [self.class.start, self.class.finish] unless self.class.cache_key.present? + return [start, finish] unless self.class.cache_key.present? key_name = "metric_instrumentation/#{self.class.cache_key}" - start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do - self.class.start + cached_start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do + start end - finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do - self.class.finish + cached_finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do + finish end - [start, finish] + [cached_start, cached_finish] end end end diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb index f6947c4c8ff..851aa7a50e8 100644 --- a/lib/gitlab/usage/metrics/query.rb +++ b/lib/gitlab/usage/metrics/query.rb @@ -61,9 +61,31 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def raw_sql(relation, column, distinct = false) column ||= relation.primary_key - relation.select(relation.all.table[column].count(distinct)).to_sql + node = node_to_count(relation, column) + + relation.unscope(:order).select(node.count(distinct)).to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + def node_to_count(relation, column) + if join_relation?(relation) && joined_column?(column) + table_name, column_name = column.split(".") + Arel::Table.new(table_name)[column_name] + else + relation.all.table[column] + end + end + + def join_relation?(relation) + relation.is_a?(ActiveRecord::Relation) && relation.joins_values.present? + end + + # checks if the passed column is of format "table.column" + def joined_column?(column) + column.is_a?(String) && column.include?(".") end end end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb new file mode 100644 index 00000000000..e32dcd3777b --- /dev/null +++ b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module ServicePing + class LegacyMetricTimingDecorator < SimpleDelegator + attr_reader :duration + + delegate :class, :is_a?, :kind_of?, to: :__getobj__ + + def initialize(value, duration) + @duration = duration + super(value) + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index 3e653b186a0..e73200cbd4a 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -3,6 +3,8 @@ module Gitlab module Usage class ServicePingReport + CACHE_KEY = 'usage_data' + class << self def for(output:, cached: false) case output.to_sym @@ -26,7 +28,7 @@ module Gitlab end def all_metrics_values(cached) - Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do + Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do Gitlab::UsageData.data end end diff --git a/lib/gitlab/usage_counters/common.rb b/lib/gitlab/usage_counters/common.rb deleted file mode 100644 index a5bdac430f4..00000000000 --- a/lib/gitlab/usage_counters/common.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageCounters - class Common - class << self - def increment(project_id) - Gitlab::Redis::SharedState.with { |redis| redis.hincrby(base_key, project_id, 1) } - end - - def usage_totals - Gitlab::Redis::SharedState.with do |redis| - total_sum = 0 - - totals = redis.hgetall(base_key).each_with_object({}) do |(project_id, count), result| - total_sum += result[project_id.to_i] = count.to_i - end - - totals[:total] = total_sum - totals - end - end - - def base_key - raise NotImplementedError - end - end - end - end -end diff --git a/lib/gitlab/usage_counters/pod_logs.rb b/lib/gitlab/usage_counters/pod_logs.rb deleted file mode 100644 index 94e29d2fad7..00000000000 --- a/lib/gitlab/usage_counters/pod_logs.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageCounters - class PodLogs < Common - def self.base_key - 'POD_LOGS_USAGE_COUNTS' - end - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b465d4bcc9b..7a17288e5e5 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -9,7 +9,7 @@ # active_user_count: count(User.active) # alt_usage_data { Gitlab::VERSION } # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } +# redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_expanding_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } # NOTE: # Implementing metrics direct in `usage_data.rb` is deprecated, @@ -308,7 +308,7 @@ module Gitlab Settings[component]['object_store'] end - if config + if config.present? { enabled: alt_usage_data { Settings[component]['enabled'] }, object_store: { @@ -684,6 +684,17 @@ module Gitlab .merge!(ide_monthly_active_users(date_range)) end + def with_duration + return yield unless Feature.enabled?(:measure_service_ping_metric_collection) + + result = nil + duration = Benchmark.realtime do + result = yield + end + + ::Gitlab::Usage::ServicePing::LegacyMetricTimingDecorator.new(result, duration) + end + private def stage_manage_events(time_period) @@ -855,16 +866,17 @@ module Gitlab end def project_imports(time_period) + time_frame = metric_time_period(time_period) counters = { - gitlab_project: projects_imported_count('gitlab_project', time_period), - gitlab: projects_imported_count('gitlab', time_period), - github: projects_imported_count('github', time_period), - bitbucket: projects_imported_count('bitbucket', time_period), - bitbucket_server: projects_imported_count('bitbucket_server', time_period), - gitea: projects_imported_count('gitea', time_period), - git: projects_imported_count('git', time_period), - manifest: projects_imported_count('manifest', time_period), - gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord + gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }), + gitlab: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab' }), + github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }), + bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }), + bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }), + gitea: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitea' }), + git: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'git' }), + manifest: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'manifest' }), + gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :project_entity }) } counters[:total] = add(*counters.values) @@ -872,46 +884,21 @@ module Gitlab counters end - def projects_imported_count(from, time_period) - # rubocop:disable CodeReuse/ActiveRecord - relation = ::Project.imported_from(from).where.not(import_type: nil) # rubocop:disable UsageData/LargeTable - if time_period.empty? - count(relation) - else - @project_import_id ||= {} - start = time_period[:created_at].first - finish = time_period[:created_at].last - - # can be nil values here if no records are in our range and it is possible the same instance - # is called with different time periods since it is passed in as a variable - unless @project_import_id.key?(start) - @project_import_id[start] = ::Project.select(:id).where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable - .order(created_at: :asc).limit(1).first&.id - end - - unless @project_import_id.key?(finish) - @project_import_id[finish] = ::Project.select(:id).where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable - .order(created_at: :desc).limit(1).first&.id - end - - count(relation, start: @project_import_id[start], finish: @project_import_id[finish]) - end - # rubocop:enable CodeReuse/ActiveRecord - end - def issue_imports(time_period) + time_frame = metric_time_period(time_period) { jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord - fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period), + fogbugz: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'fogbugz' }), + phabricator: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'phabricator' }), csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord } end def group_imports(time_period) + time_frame = metric_time_period(time_period) { group_import: count(::GroupImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord - gitlab_migration: count(::BulkImports::Entity.where(time_period).group_entity) # rubocop: disable CodeReuse/ActiveRecord + gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :group_entity }) } end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index cf3caf3f0c7..61c071c8738 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -15,7 +15,7 @@ module Gitlab::UsageDataCounters ) namespace = project.namespace - if Feature.enabled?(:route_hll_to_snowplow, namespace, default_enabled: :yaml) + if Feature.enabled?(:route_hll_to_snowplow, namespace) Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, user: user, project: project) end end diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index bc0126cd893..f97ebdccecf 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -8,6 +8,7 @@ module Gitlab EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' EDIT_BY_SSE = 'g_edit_by_sse' EDIT_CATEGORY = 'ide_edit' + EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' class << self def track_web_ide_edit_action(author:, time: Time.zone.now) @@ -47,6 +48,10 @@ module Gitlab count_unique(EDIT_BY_SSE, date_from, date_to) end + def track_live_preview_edit_action(author:, time: Time.zone.now) + track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time) + end + private def track_unique_action(action, author, time) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 3b34cd77cf5..0ace6e99c59 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -185,7 +185,7 @@ module Gitlab def feature_enabled?(event) return true if event[:feature_flag].blank? - Feature.enabled?(event[:feature_flag], default_enabled: :yaml) && Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) + Feature.enabled?(event[:feature_flag]) && Feature.enabled?(:redis_hll_tracking, type: :ops) end # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level diff --git a/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb b/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb new file mode 100644 index 00000000000..a34ae909c82 --- /dev/null +++ b/lib/gitlab/usage_data_counters/ipynb_diff_activity_counter.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# noinspection RubyConstantNamingConvention +module Gitlab + module UsageDataCounters + module IpynbDiffActivityCounter + NOTE_CREATED_IN_IPYNB_DIFF_ACTION = 'i_code_review_create_note_in_ipynb_diff' + USER_CREATED_NOTE_IN_IPYNB_DIFF_ACTION = 'i_code_review_user_create_note_in_ipynb_diff' + NOTE_CREATED_IN_IPYNB_DIFF_MR_ACTION = 'i_code_review_create_note_in_ipynb_diff_mr' + USER_CREATED_NOTE_IN_IPYNB_DIFF_MR_ACTION = 'i_code_review_user_create_note_in_ipynb_diff_mr' + NOTE_CREATED_IN_IPYNB_DIFF_COMMIT_ACTION = 'i_code_review_create_note_in_ipynb_diff_commit' + USER_CREATED_NOTE_IN_IPYNB_DIFF_COMMIT_ACTION = 'i_code_review_user_create_note_in_ipynb_diff_commit' + + class << self + def note_created(note) + return unless note.for_merge_request? || note.for_commit? + + if note.for_merge_request? + track(NOTE_CREATED_IN_IPYNB_DIFF_MR_ACTION, USER_CREATED_NOTE_IN_IPYNB_DIFF_MR_ACTION, note) + else + track(NOTE_CREATED_IN_IPYNB_DIFF_COMMIT_ACTION, USER_CREATED_NOTE_IN_IPYNB_DIFF_COMMIT_ACTION, note) + end + + track(NOTE_CREATED_IN_IPYNB_DIFF_ACTION, USER_CREATED_NOTE_IN_IPYNB_DIFF_ACTION, note) + end + + private + + def track(action, per_user_action, note) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, note.id) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(per_user_action, note.author_id) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index f179f6d679d..3b883e505f8 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -127,6 +127,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -147,10 +151,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_security_cluster_image_scanning - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_qualys_iac_security category: ci_templates redis_slot: ci_templates @@ -187,6 +187,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_liquibase + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_flutter category: ci_templates redis_slot: ci_templates @@ -207,10 +211,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_managed_cluster_applications - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_php category: ci_templates redis_slot: ci_templates @@ -231,10 +231,6 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_serverless - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - name: p_ci_templates_go category: ci_templates redis_slot: ci_templates @@ -255,6 +251,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_matlab + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_deploy_ecs category: ci_templates redis_slot: ci_templates @@ -331,6 +331,18 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_sast_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_jobs_secret_detection_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -523,6 +535,18 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_sast_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_implicit_jobs_secret_detection_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -595,6 +619,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_sast_iac + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_dependency_scanning category: ci_templates redis_slot: ci_templates @@ -615,15 +643,3 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_security_cluster_image_scanning - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_liquibase - category: ci_templates - redis_slot: ci_templates - aggregation: weekly -- name: p_ci_templates_matlab - category: ci_templates - redis_slot: ci_templates - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index 63498a35858..5159dcf62ab 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -2,4 +2,4 @@ category: ci_users redis_slot: ci_users aggregation: weekly - feature_flag: job_deployment_count + feature_flag: diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index df2864bba89..e3bb3f6fef3 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -173,6 +173,30 @@ redis_slot: code_review category: code_review aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_commit + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_commit + redis_slot: code_review + category: code_review + aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting redis_slot: code_review @@ -234,6 +258,7 @@ redis_slot: code_review category: code_review aggregation: weekly + feature_flag: usage_data_diff_searches - name: i_code_review_total_suggestions_applied redis_slot: code_review category: code_review diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 0d89a5181ec..448ed4c66e1 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -40,6 +40,11 @@ redis_slot: edit expiry: 29 aggregation: daily +- name: g_edit_by_live_preview + category: ide_edit + redis_slot: edit + expiry: 29 + aggregation: daily - name: i_search_total category: search redis_slot: search diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 82787b7bf29..dd6625a9cc9 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -218,3 +218,10 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_epic_blocked_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 1661a1b6987..79d4b45a1ce 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -31,6 +31,10 @@ module Gitlab SQL_METRIC_DEFAULT end + def add(*args) + SQL_METRIC_DEFAULT + end + def maximum_id(model, column = nil) end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 977cc3549d8..b2d74b1f0dd 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,6 +5,10 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self + def with_duration + yield + end + def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index a4a1cccf9d5..c2f61741cc5 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -103,9 +103,7 @@ module Gitlab def branch_allows_collaboration_for?(ref) return false if skip_collaboration_check - # Checking for an internal project or group to prevent an infinite loop: - # https://gitlab.com/gitlab-org/gitlab/issues/36805 - (!project.internal? && project.branch_allows_collaboration?(user, ref)) + project.branch_allows_collaboration?(user, ref) end def permission_cache diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 816ede4136a..a67a0758257 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -128,10 +128,6 @@ module Gitlab end end - def random_string - Random.rand(Float::MAX.to_i).to_s(36) - end - # Behaves like `which` on Linux machines: given PATH, try to resolve the given # executable name to an absolute path, or return nil. # diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 6c182f98dd0..633f4683b6b 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -31,7 +31,7 @@ # # Examples: # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } +# redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'users_expanding_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } module Gitlab module Utils @@ -44,57 +44,64 @@ module Gitlab DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 + def with_duration + yield + end + def add_metric(metric, time_frame: 'none', options: {}) metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize metric_class.new(time_frame: time_frame, options: options).value end - def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - if batch - Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) - else - relation.count + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil, start_at: Time.current) + with_duration do + if batch + Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) + else + relation.count + end + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK end - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - if batch - Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) - else - relation.distinct_count_by(column) + with_duration do + if batch + Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) + else + relation.distinct_count_by(column) + end + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK end - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter - .new(relation, column) - .execute(batch_size: batch_size, start: start, finish: finish) + with_duration do + buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter + .new(relation, column) + .execute(batch_size: batch_size, start: start, finish: finish) - yield buckets if block_given? + yield buckets if block_given? - buckets.estimated_distinct_count - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK - # catch all rescue should be removed as a part of feature flag rollout issue - # https://gitlab.com/gitlab-org/gitlab/-/issues/285485 - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - DISTRIBUTED_HLL_FALLBACK + buckets.estimated_distinct_count + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end end def sum(relation, column, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) - rescue ActiveRecord::StatementInvalid => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK + with_duration do + Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end end # We don't support batching with histograms. @@ -103,103 +110,113 @@ module Gitlab # # rubocop: disable CodeReuse/ActiveRecord def histogram(relation, column, buckets:, bucket_size: buckets.size) - # Using lambda to avoid exposing histogram specific methods - parameters_valid = lambda do - error_message = - if buckets.first == buckets.last - 'Lower bucket bound cannot equal to upper bucket bound' - elsif bucket_size == 0 - 'Bucket size cannot be zero' - elsif bucket_size > MAX_BUCKET_SIZE - "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}" - end - - return true unless error_message - - exception = ArgumentError.new(error_message) - exception.set_backtrace(caller) - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + with_duration do + # Using lambda to avoid exposing histogram specific methods + parameters_valid = lambda do + error_message = + if buckets.first == buckets.last + 'Lower bucket bound cannot equal to upper bucket bound' + elsif bucket_size == 0 + 'Bucket size cannot be zero' + elsif bucket_size > MAX_BUCKET_SIZE + "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}" + end + + break true unless error_message + + exception = ArgumentError.new(error_message) + exception.set_backtrace(caller) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + + false + end - false + break HISTOGRAM_FALLBACK unless parameters_valid.call + + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + # For example, 9 segments gives 10 buckets + bucket_segments = bucket_size - 1 + + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + # Return the histogram as a Hash because buckets are unique. + relation + .connection + .exec_query(query.to_sql) + .rows + .to_h + # Keys are converted to strings in Usage Ping JSON + .stringify_keys + rescue ActiveRecord::StatementInvalid => e + Gitlab::AppJsonLogger.error( + event: 'histogram', + relation: relation.table_name, + operation: 'histogram', + operation_args: [column, buckets.first, buckets.last, bucket_segments], + query: query.to_sql, + message: e.message + ) + # Raises error for dev env + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + HISTOGRAM_FALLBACK end - - return HISTOGRAM_FALLBACK unless parameters_valid.call - - count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) - cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) - - # For example, 9 segments gives 10 buckets - bucket_segments = bucket_size - 1 - - width_bucket = Arel::Nodes::NamedFunction - .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) - .as('buckets') - - query = cte - .table - .project(width_bucket, cte.table[:count]) - .group('buckets') - .order('buckets') - .with(cte.to_arel) - - # Return the histogram as a Hash because buckets are unique. - relation - .connection - .exec_query(query.to_sql) - .rows - .to_h - # Keys are converted to strings in Usage Ping JSON - .stringify_keys - rescue ActiveRecord::StatementInvalid => e - Gitlab::AppJsonLogger.error( - event: 'histogram', - relation: relation.table_name, - operation: 'histogram', - operation_args: [column, buckets.first, buckets.last, bucket_segments], - query: query.to_sql, - message: e.message - ) - # Raises error for dev env - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) - HISTOGRAM_FALLBACK end # rubocop: enable CodeReuse/ActiveRecord def add(*args) - return -1 if args.any?(&:negative?) + with_duration do + break -1 if args.any?(&:negative?) - args.sum - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - FALLBACK + args.sum + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end end def alt_usage_data(value = nil, fallback: FALLBACK, &block) - if block_given? - yield - else - value + with_duration do + if block_given? + yield + else + value + end + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + fallback end - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - fallback end def redis_usage_data(counter = nil, &block) - if block_given? - redis_usage_counter(&block) - elsif counter.present? - redis_usage_data_totals(counter) + with_duration do + if block_given? + redis_usage_counter(&block) + elsif counter.present? + redis_usage_data_totals(counter) + end end end def with_prometheus_client(fallback: {}, verify: true) - client = prometheus_client(verify: verify) - return fallback unless client + with_duration do + client = prometheus_client(verify: verify) + break fallback unless client - yield client - rescue StandardError - fallback + yield client + rescue StandardError + fallback + end end def measure_duration @@ -231,25 +248,28 @@ module Gitlab # rubocop: disable UsageData/LargeTable: def jira_integration_data - data = { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - - # rubocop: disable CodeReuse/ActiveRecord - ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| - counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - service_url = service.data_fields&.url || (service.properties && service.properties['url']) - service_url&.include?('.atlassian.net') ? :cloud : :server + with_duration do + data = { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } + + # rubocop: disable CodeReuse/ActiveRecord + ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| + counts = services.group_by do |service| + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + service_url = service.data_fields&.url || (service.properties && service.properties['url']) + service_url&.include?('.atlassian.net') ? :cloud : :server + end + + data[:projects_jira_server_active] += counts[:server].size if counts[:server] + data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] end - data[:projects_jira_server_active] += counts[:server].size if counts[:server] - data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] + data end - - data end + # rubocop: enable CodeReuse/ActiveRecord # rubocop: enable UsageData/LargeTable: @@ -263,9 +283,11 @@ module Gitlab end def epics_deepest_relationship_level - # rubocop: disable UsageData/LargeTable - { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } - # rubocop: enable UsageData/LargeTable + with_duration do + # rubocop: disable UsageData/LargeTable + { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } + # rubocop: enable UsageData/LargeTable + end end private diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 19d30daa577..e81670ce89a 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,8 +33,7 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags(repository.project), - sidechannel: Feature.enabled?(:workhorse_use_sidechannel, repository.project, default_enabled: :yaml) + features: Feature::Gitaly.server_feature_flags(repository.project) } } @@ -226,6 +225,13 @@ module Gitlab end end + def detect_content_type + [ + Gitlab::Workhorse::DETECT_HEADER, + 'true' + ] + end + protected # This is the outermost encoding of a senddata: header. It is safe for diff --git a/lib/gitlab/zentao/client.rb b/lib/gitlab/zentao/client.rb index 8acfb4913f3..4da4631eecf 100644 --- a/lib/gitlab/zentao/client.rb +++ b/lib/gitlab/zentao/client.rb @@ -58,7 +58,7 @@ module Gitlab def url(path) host = integration.api_url.presence || integration.url - URI.join(host, '/api.php/v1/', path) + URI.parse(Gitlab::Utils.append_path(host, "api.php/v1/#{path}")) end def headers diff --git a/lib/product_analytics/collector_app.rb b/lib/product_analytics/collector_app.rb index cf971eef4b6..1008d2f264c 100644 --- a/lib/product_analytics/collector_app.rb +++ b/lib/product_analytics/collector_app.rb @@ -14,7 +14,7 @@ module ProductAnalytics # for project without the feature enabled. During increase of feature adoption, this # check will be removed for better performance. project = Project.find(params['aid'].to_i) - return not_found unless Feature.enabled?(:product_analytics, project, default_enabled: false) + return not_found unless Feature.enabled?(:product_analytics, project) # Snowplow tracker has own format of events. # We need to convert them to match the schema of our database. diff --git a/lib/service_ping/build_payload.rb b/lib/service_ping/build_payload.rb new file mode 100644 index 00000000000..4d3b32a1fc0 --- /dev/null +++ b/lib/service_ping/build_payload.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module ServicePing + class BuildPayload + def execute + return {} unless ServicePingSettings.product_intelligence_enabled? + + filtered_usage_data + end + + private + + def raw_payload + @raw_payload ||= ::Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values) + end + + def filtered_usage_data(payload = raw_payload, parents = []) + return unless payload.is_a?(Hash) + + payload.keep_if do |label, node| + key_path = parents.dup.append(label).join('.') + + if has_metric_definition?(key_path) + include_metric?(key_path) + else + filtered_usage_data(node, parents.dup << label) if node.is_a?(Hash) + end + end + end + + def include_metric?(key_path) + valid_metric_status?(key_path) && permitted_metric?(key_path) + end + + def valid_metric_status?(key_path) + metric_definitions[key_path]&.valid_service_ping_status? + end + + def permitted_categories + @permitted_categories ||= ::ServicePing::PermitDataCategories.new.execute + end + + def permitted_metric?(key_path) + permitted_categories.include?(metric_category(key_path)) + end + + def has_metric_definition?(key_path) + metric_definitions[key_path].present? + end + + def metric_category(key_path) + metric_definitions[key_path] + &.attributes + &.fetch(:data_category, ::ServicePing::PermitDataCategories::OPTIONAL_CATEGORY) + end + + def metric_definitions + @metric_definitions ||= ::Gitlab::Usage::MetricDefinition.definitions + end + end +end + +ServicePing::BuildPayload.prepend_mod_with('ServicePing::BuildPayload') diff --git a/lib/service_ping/devops_report.rb b/lib/service_ping/devops_report.rb new file mode 100644 index 00000000000..2444dfa1d21 --- /dev/null +++ b/lib/service_ping/devops_report.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module ServicePing + class DevopsReport + def initialize(data) + @data = data + end + + def execute + # `conv_index` was previously named `dev_ops_score` in + # version-gitlab-com, so we check both for backwards compatibility. + metrics = @data['conv_index'] || @data['dev_ops_score'] + + # Do not attempt to save a report for the first Service Ping + # response for a given GitLab instance, which comes without + # metrics. + return if metrics.keys == ['usage_data_id'] + + report = DevOpsReport::Metric.create( + metrics.slice(*DevOpsReport::Metric::METRICS) + ) + + unless report.persisted? + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + ActiveRecord::RecordInvalid.new(report) + ) + end + end + end +end diff --git a/lib/service_ping/permit_data_categories.rb b/lib/service_ping/permit_data_categories.rb new file mode 100644 index 00000000000..51eec0808cb --- /dev/null +++ b/lib/service_ping/permit_data_categories.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module ServicePing + class PermitDataCategories + STANDARD_CATEGORY = 'standard' + SUBSCRIPTION_CATEGORY = 'subscription' + OPERATIONAL_CATEGORY = 'operational' + OPTIONAL_CATEGORY = 'optional' + CATEGORIES = [ + STANDARD_CATEGORY, + SUBSCRIPTION_CATEGORY, + OPERATIONAL_CATEGORY, + OPTIONAL_CATEGORY + ].to_set.freeze + + def execute + return [] unless ServicePingSettings.product_intelligence_enabled? + + CATEGORIES + end + end +end + +ServicePing::PermitDataCategories.prepend_mod_with('ServicePing::PermitDataCategories') diff --git a/lib/service_ping/service_ping_settings.rb b/lib/service_ping/service_ping_settings.rb new file mode 100644 index 00000000000..6964210b1db --- /dev/null +++ b/lib/service_ping/service_ping_settings.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ServicePing + module ServicePingSettings + extend self + + def product_intelligence_enabled? + enabled? && !User.single_user&.requires_usage_stats_consent? + end + + def enabled? + ::Gitlab::CurrentSettings.usage_ping_enabled? + end + end +end + +ServicePing::ServicePingSettings.extend_mod_with('ServicePing::ServicePingSettings') diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb index c1d80458f49..0c2995f95e6 100644 --- a/lib/sidebars/groups/menus/ci_cd_menu.rb +++ b/lib/sidebars/groups/menus/ci_cd_menu.rb @@ -35,8 +35,7 @@ module Sidebars end def show_runners? - can?(context.current_user, :read_group_runners, context.group) && - Feature.enabled?(:runner_list_group_view_vue_ui, context.group, default_enabled: :yaml) + can?(context.current_user, :read_group_runners, context.group) end end end diff --git a/lib/sidebars/groups/menus/kubernetes_menu.rb b/lib/sidebars/groups/menus/kubernetes_menu.rb index 98ca7865995..0d845978a93 100644 --- a/lib/sidebars/groups/menus/kubernetes_menu.rb +++ b/lib/sidebars/groups/menus/kubernetes_menu.rb @@ -23,7 +23,7 @@ module Sidebars def render? clusterable = context.group - Feature.enabled?(:certificate_based_clusters, clusterable, default_enabled: :yaml, type: :ops) && + clusterable.certificate_based_clusters_enabled? && can?(context.current_user, :read_cluster, clusterable) end diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index 09226256476..18ff3ebc714 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -89,16 +89,10 @@ module Sidebars end def ci_cd_menu_item - active_routes_path = if Feature.enabled?(:runner_list_group_view_vue_ui, context.group, default_enabled: :yaml) - 'ci_cd#show' - else - %w[ci_cd#show groups/runners#show groups/runners#edit] - end - ::Sidebars::MenuItem.new( title: _('CI/CD'), link: group_settings_ci_cd_path(context.group), - active_routes: { path: active_routes_path }, + active_routes: { path: 'ci_cd#show' }, item_id: :ci_cd ) end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 7bd9ac91efa..a98cc20d51a 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -9,7 +9,6 @@ module Sidebars return false unless context.project.feature_available?(:operations, context.current_user) add_item(kubernetes_menu_item) - add_item(serverless_menu_item) add_item(terraform_menu_item) add_item(google_cloud_menu_item) @@ -63,19 +62,6 @@ module Sidebars auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } end - def serverless_menu_item - unless Feature.enabled?(:deprecated_serverless, context.project, default_enabled: :yaml, type: :ops) && can?(context.current_user, :read_cluster, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :serverless) - end - - ::Sidebars::MenuItem.new( - title: _('Serverless platform'), - link: project_serverless_functions_path(context.project), - active_routes: { controller: :functions }, - item_id: :serverless - ) - end - def terraform_menu_item unless can?(context.current_user, :read_terraform_state, context.project) return ::Sidebars::NilMenuItem.new(item_id: :terraform) diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index 59554726263..c35bc1f5481 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -58,7 +58,8 @@ module Sidebars end def logs_menu_item - if !can?(context.current_user, :read_environment, context.project) || + if !Feature.enabled?(:monitor_logging, context.project) || + !can?(context.current_user, :read_environment, context.project) || !can?(context.current_user, :read_pod_logs, context.project) return ::Sidebars::NilMenuItem.new(item_id: :logs) end @@ -72,7 +73,8 @@ module Sidebars end def tracing_menu_item - if !can?(context.current_user, :read_environment, context.project) || + if !Feature.enabled?(:monitor_tracing, context.project) || + !can?(context.current_user, :read_environment, context.project) || !can?(context.current_user, :admin_project, context.project) return ::Sidebars::NilMenuItem.new(item_id: :tracing) end diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 2411ca8263a..2b5b3cdbb22 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -14,9 +14,9 @@ module Sidebars add_item(access_tokens_menu_item) add_item(repository_menu_item) add_item(ci_cd_menu_item) - add_item(monitor_menu_item) - add_item(pages_menu_item) add_item(packages_and_registries_menu_item) + add_item(pages_menu_item) + add_item(monitor_menu_item) add_item(usage_quotas_menu_item) true @@ -103,16 +103,17 @@ module Sidebars ) end - def monitor_menu_item - if context.project.archived? || !can?(context.current_user, :admin_operations, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :monitor) + def packages_and_registries_menu_item + if !Gitlab.config.registry.enabled || + !can?(context.current_user, :destroy_container_image, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) end ::Sidebars::MenuItem.new( - title: _('Monitor'), - link: project_settings_operations_path(context.project), - active_routes: { path: 'operations#show' }, - item_id: :monitor + title: _('Packages & Registries'), + link: project_settings_packages_and_registries_path(context.project), + active_routes: { path: 'packages_and_registries#index' }, + item_id: :packages_and_registries ) end @@ -129,17 +130,16 @@ module Sidebars ) end - def packages_and_registries_menu_item - if !Gitlab.config.registry.enabled || - !can?(context.current_user, :destroy_container_image, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) + def monitor_menu_item + if context.project.archived? || !can?(context.current_user, :admin_operations, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :monitor) end ::Sidebars::MenuItem.new( - title: _('Packages & Registries'), - link: project_settings_packages_and_registries_path(context.project), - active_routes: { path: 'packages_and_registries#index' }, - item_id: :packages_and_registries + title: _('Monitor'), + link: project_settings_operations_path(context.project), + active_routes: { path: 'operations#show' }, + item_id: :monitor ) end diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb index 6bb4fb52e2a..1af8e14f034 100644 --- a/lib/sidebars/projects/panel.rb +++ b/lib/sidebars/projects/panel.rb @@ -28,9 +28,9 @@ module Sidebars add_menu(Sidebars::Projects::Menus::CiCdMenu.new(context)) add_menu(Sidebars::Projects::Menus::SecurityComplianceMenu.new(context)) add_menu(Sidebars::Projects::Menus::DeploymentsMenu.new(context)) - add_menu(Sidebars::Projects::Menus::MonitorMenu.new(context)) - add_menu(Sidebars::Projects::Menus::InfrastructureMenu.new(context)) add_menu(Sidebars::Projects::Menus::PackagesRegistriesMenu.new(context)) + add_menu(Sidebars::Projects::Menus::InfrastructureMenu.new(context)) + add_menu(Sidebars::Projects::Menus::MonitorMenu.new(context)) add_menu(Sidebars::Projects::Menus::AnalyticsMenu.new(context)) add_wiki_menus add_menu(Sidebars::Projects::Menus::SnippetsMenu.new(context)) diff --git a/lib/support/systemd/gitlab-sidekiq.service b/lib/support/systemd/gitlab-sidekiq.service index 81046f5348a..7d09944c862 100644 --- a/lib/support/systemd/gitlab-sidekiq.service +++ b/lib/support/systemd/gitlab-sidekiq.service @@ -6,7 +6,7 @@ After=network.target JoinsNamespaceOf=gitlab-puma.service [Service] -Type=simple +Type=notify User=git WorkingDirectory=/home/git/gitlab Environment=RAILS_ENV=production @@ -17,6 +17,7 @@ Restart=on-failure RestartSec=1 SyslogIdentifier=gitlab-sidekiq Slice=gitlab.slice +WatchdogSec=10 [Install] WantedBy=gitlab.target diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 42b12cd0ae3..08a11100431 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -58,9 +58,13 @@ namespace :dev do namespace :copy_db do ALLOWED_DATABASES = %w[ci].freeze + defined_copy_db_tasks = [] + ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |name| next unless ALLOWED_DATABASES.include?(name) + defined_copy_db_tasks << name + desc "Copies the #{name} database from the main database" task name => :environment do Rake::Task["dev:terminate_all_connections"].invoke @@ -72,5 +76,16 @@ namespace :dev do warn "Database '#{db_config.database}' already exists" end end + + ALLOWED_DATABASES.each do |name| + next if defined_copy_db_tasks.include?(name) + + # :nocov: we cannot mock ActiveRecord::Tasks::DatabaseTasks in time + # Workaround for GDK issue, see + # https://gitlab.com/gitlab-org/gitlab-development-kit/-/issues/1464 + desc "No-op task" + task name + # :nocov: + end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 3a7e53a27e4..068dc463d16 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -282,7 +282,7 @@ namespace :gitlab do puts "There are #{Gitlab::Database::Reindexing::QueuedAction.queued.size} queued actions in total." end - unless Feature.enabled?(:database_reindexing, type: :ops, default_enabled: :yaml) + unless Feature.enabled?(:database_reindexing, type: :ops) puts <<~NOTE.color(:yellow) Note: database_reindexing feature is currently disabled. @@ -328,6 +328,15 @@ namespace :gitlab do Gitlab::Database::Migrations::Runner.background_migrations.run_jobs(for_duration: duration) end + + desc 'Sample batched background migrations with instrumentation' + task :sample_batched_background_migrations, [:database, :duration_s] => [:environment] do |_t, args| + database_name = args[:database] || 'main' + duration = args[:duration_s]&.to_i&.seconds || 30.minutes # Default of 30 minutes + + Gitlab::Database::Migrations::Runner.batched_background_migrations(for_database: database_name) + .run_jobs(for_duration: duration) + end end desc 'Run all pending batched migrations' diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index cc5f6bb6e09..66aa949cc94 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -6,7 +6,7 @@ namespace :gitlab do namespace :db do desc 'Validates `config/database.yml` to ensure a correct behavior is configured' task validate_config: :environment do - original_db_config = ActiveRecord::Base.connection_db_config + original_db_config = ActiveRecord::Base.connection_db_config # rubocop:disable Database/MultipleDatabases # The include_replicas: is a legacy name to fetch all hidden entries (replica: true or database_tasks: false) # Once we upgrade to Rails 7.x this should be changed to `include_hidden: true` @@ -15,6 +15,7 @@ namespace :gitlab do db_configs = db_configs.reject(&:replica?) # Map each database connection into unique identifier of system+database + # rubocop:disable Database/MultipleDatabases all_connections = db_configs.map do |db_config| identifier = begin @@ -32,6 +33,7 @@ namespace :gitlab do identifier: identifier } end.compact + # rubocop:enable Database/MultipleDatabases unique_connections = all_connections.group_by { |connection| connection[:identifier] } primary_connection = all_connections.find { |connection| ActiveRecord::Base.configurations.primary?(connection[:name]) } diff --git a/lib/tasks/gitlab/metrics_exporter.rake b/lib/tasks/gitlab/metrics_exporter.rake new file mode 100644 index 00000000000..d9dd80b8eeb --- /dev/null +++ b/lib/tasks/gitlab/metrics_exporter.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true +require_relative Rails.root.join('metrics_server', 'dependencies') +require_relative Rails.root.join('metrics_server', 'metrics_server') + +namespace :gitlab do + namespace :metrics_exporter do + REPO = 'https://gitlab.com/gitlab-org/gitlab-metrics-exporter.git' + + desc "GitLab | Metrics Exporter | Install or upgrade gitlab-metrics-exporter" + task :install, [:dir] => :gitlab_environment do |t, args| + unless args.dir.present? + abort %(Please specify the directory where you want to install the exporter +Usage: rake "gitlab:metrics_exporter:install[/installation/dir]") + end + + version = ENV['GITLAB_METRICS_EXPORTER_VERSION'] || MetricsServer.version + make = Gitlab::Utils.which('gmake') || Gitlab::Utils.which('make') + + abort "Couldn't find a 'make' binary" unless make + + checkout_or_clone_version(version: version, repo: REPO, target_dir: args.dir, clone_opts: %w[--depth 1]) + + Dir.chdir(args.dir) { run_command!([make]) } + end + end +end diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake index a9a350fb6c3..4f5df7841e2 100644 --- a/lib/tasks/gitlab/seed/group_seed.rake +++ b/lib/tasks/gitlab/seed/group_seed.rake @@ -158,7 +158,7 @@ class GroupSeeder group = Group.find(group_id) label_title = FFaker::Product.brand - Labels::CreateService.new(title: label_title, color: "##{Digest::MD5.hexdigest(label_title)[0..5]}").execute(group: group) + Labels::CreateService.new(title: label_title, color: "#{::Gitlab::Color.color_for(label_title)}").execute(group: group) end end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b3abc48f8e0..8627a326247 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -21,7 +21,6 @@ namespace :gitlab do config = { user: Gitlab.config.gitlab.user, gitlab_url: gitlab_url, - http_settings: { self_signed_cert: false }.stringify_keys, auth_file: File.join(user_home, ".ssh", "authorized_keys"), log_level: "INFO", audit_usernames: false diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 0aed017c84a..6574bfd2549 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -17,16 +17,16 @@ namespace :tw do CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), - CodeOwnerRule.new('Configure', '@marcia'), + CodeOwnerRule.new('Configure', '@sselhorn'), CodeOwnerRule.new('Container Security', '@claytoncornell'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), - CodeOwnerRule.new('Database', '@marcia'), - CodeOwnerRule.new('Development', '@marcia'), + CodeOwnerRule.new('Database', '@aqualls'), + CodeOwnerRule.new('Development', '@sselhorn'), CodeOwnerRule.new('Distribution', '@axil'), CodeOwnerRule.new('Distribution (Charts)', '@axil'), CodeOwnerRule.new('Distribution (Omnibus)', '@axil'), - CodeOwnerRule.new('Documentation Guidelines', '@cnorris'), + CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), CodeOwnerRule.new('Ecosystem', '@kpaizee'), CodeOwnerRule.new('Editor', '@aqualls'), @@ -35,13 +35,12 @@ namespace :tw do CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), CodeOwnerRule.new('Geo', '@axil'), CodeOwnerRule.new('Gitaly', '@eread'), - CodeOwnerRule.new('Global Search', '@marcia'), + CodeOwnerRule.new('Global Search', '@sselhorn'), CodeOwnerRule.new('Import', '@eread'), - CodeOwnerRule.new('Infrastructure', '@marcia'), + CodeOwnerRule.new('Infrastructure', '@sselhorn'), CodeOwnerRule.new('Integrations', '@kpaizee'), CodeOwnerRule.new('Knowledge', '@aqualls'), - CodeOwnerRule.new('License', '@sselhorn'), - CodeOwnerRule.new('Memory', '@marcia'), + CodeOwnerRule.new('Memory', '@sselhorn'), CodeOwnerRule.new('Monitor', '@msedlakjakubowski'), CodeOwnerRule.new('Observability', 'msedlakjakubowski'), CodeOwnerRule.new('Optimize', '@fneill'), @@ -53,20 +52,20 @@ namespace :tw do CodeOwnerRule.new('Product Intelligence', '@claytoncornell'), CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), CodeOwnerRule.new('Project Management', '@msedlakjakubowski'), - CodeOwnerRule.new('Provision', '@sselhorn'), - CodeOwnerRule.new('Purchase', '@sselhorn'), + CodeOwnerRule.new('Provision', '@fneill'), + CodeOwnerRule.new('Purchase', '@fneill'), CodeOwnerRule.new('Redirect', 'Redirect'), CodeOwnerRule.new('Release', '@rdickenson'), CodeOwnerRule.new('Respond', '@msedlakjakubowski'), CodeOwnerRule.new('Runner', '@sselhorn'), - CodeOwnerRule.new('Sharding', '@marcia'), + CodeOwnerRule.new('Sharding', '@sselhorn'), CodeOwnerRule.new('Source Code', '@aqualls'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Static Site Editor', '@aqualls'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Testing', '@eread'), CodeOwnerRule.new('Threat Insights', '@claytoncornell'), - CodeOwnerRule.new('Utilization', '@sselhorn'), + CodeOwnerRule.new('Utilization', '@fneill'), CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'), CodeOwnerRule.new('Workspace', '@fneill') ].freeze diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 247897bed0b..07dd5ebeacb 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -79,7 +79,7 @@ namespace :gitlab do Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz']) extracted_project_basename = Dir['*/'].first Dir.chdir(extracted_project_basename) do - Gitlab::TaskHelpers.run_command!(%w(git init)) + Gitlab::TaskHelpers.run_command!(%w(git init --initial-branch=master)) Gitlab::TaskHelpers.run_command!(%w(git add .)) Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message]) diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index 71e84d3795f..29589571344 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -2,7 +2,6 @@ unless Rails.env.production? require 'haml_lint/rake_task' - require Rails.root.join('haml_lint/inline_javascript') HamlLint::RakeTask.new end |