diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-19 23:18:09 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-19 23:18:09 +0000 |
| commit | 6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde (patch) | |
| tree | dc4d20fe6064752c0bd323187252c77e0a89144b /lib | |
| parent | 9868dae7fc0655bd7ce4a6887d4e6d487690eeed (diff) | |
| download | gitlab-ce-6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-4-stable-eev15.4.0-rc42
Diffstat (limited to 'lib')
430 files changed, 20113 insertions, 2120 deletions
diff --git a/lib/api/admin/batched_background_migrations.rb b/lib/api/admin/batched_background_migrations.rb new file mode 100644 index 00000000000..675f3365bd3 --- /dev/null +++ b/lib/api/admin/batched_background_migrations.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module API + module Admin + class BatchedBackgroundMigrations < ::API::Base + feature_category :database + urgency :low + + before do + authenticated_as_admin! + end + + namespace 'admin' do + resources 'batched_background_migrations/:id' do + desc 'Retrieve a batched background migration' + params do + optional :database, + type: String, + values: Gitlab::Database.all_database_names, + desc: 'The name of the database', + default: 'main' + requires :id, + type: Integer, + desc: 'The batched background migration id' + end + get do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + present_entity(batched_background_migration) + end + end + end + + resources 'batched_background_migrations' do + desc 'Get the list of the batched background migrations' + params do + optional :database, + type: String, + values: Gitlab::Database.all_database_names, + desc: 'The name of the database, the default `main`', + default: 'main' + end + get do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + migrations = Database::BatchedBackgroundMigrationsFinder.new(connection: base_model.connection).execute + present_entity(migrations) + end + end + end + + resources 'batched_background_migrations/:id/resume' do + desc 'Resume a batched background migration' + params do + optional :database, + type: String, + values: Gitlab::Database.all_database_names, + desc: 'The name of the database', + default: 'main' + requires :id, + type: Integer, + desc: 'The batched background migration id' + end + put do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + batched_background_migration.execute! + present_entity(batched_background_migration) + end + end + end + + resources 'batched_background_migrations/:id/pause' do + desc 'Pause a batched background migration' + params do + optional :database, + type: String, + values: Gitlab::Database.all_database_names, + desc: 'The name of the database', + default: 'main' + requires :id, + type: Integer, + desc: 'The batched background migration id' + end + put do + Gitlab::Database::SharedModel.using_connection(base_model.connection) do + batched_background_migration.pause! + present_entity(batched_background_migration) + end + end + end + end + + helpers do + def batched_background_migration + @batched_background_migration ||= Gitlab::Database::BackgroundMigration::BatchedMigration.find(params[:id]) + end + + def base_model + database = params[:database] || Gitlab::Database::MAIN_DATABASE_NAME + @base_model ||= Gitlab::Database.database_base_models[database] + end + + def present_entity(result) + present result, + with: ::API::Entities::BatchedBackgroundMigration + end + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index e4158eee37f..443bf1d649a 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -167,6 +167,7 @@ module API # Keep in alphabetical order mount ::API::AccessRequests + mount ::API::Admin::BatchedBackgroundMigrations mount ::API::Admin::Ci::Variables mount ::API::Admin::InstanceClusters mount ::API::Admin::PlanLimits @@ -237,7 +238,6 @@ module API mount ::API::ImportGithub mount ::API::Integrations mount ::API::Integrations::JiraConnect::Subscriptions - mount ::API::Integrations::Slack::Events mount ::API::Invitations mount ::API::IssueLinks mount ::API::Issues @@ -263,6 +263,7 @@ module API mount ::API::PackageFiles mount ::API::Pages mount ::API::PagesDomains + mount ::API::PersonalAccessTokens::SelfRevocation mount ::API::PersonalAccessTokens mount ::API::ProjectClusters mount ::API::ProjectContainerRepositories @@ -290,6 +291,7 @@ module API mount ::API::ResourceLabelEvents mount ::API::ResourceMilestoneEvents mount ::API::ResourceStateEvents + mount ::API::RpmProjectPackages mount ::API::RubygemPackages mount ::API::Search mount ::API::Settings @@ -316,6 +318,7 @@ module API mount ::API::Users mount ::API::Version mount ::API::Wikis + mount ::API::Ml::Mlflow end mount ::API::Internal::Base diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b8444351029..5588818cbaf 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -52,25 +52,21 @@ module API merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - if Feature.enabled?(:api_caching_branches, user_project, type: :development) - present_cached( - branches, - with: Entities::Branch, - current_user: current_user, - project: user_project, - merged_branch_names: merged_branch_names, - expires_in: 10.minutes, - cache_context: -> (branch) { [current_user&.cache_key, merged_branch_names.include?(branch.name)] } - ) - else - present( - branches, - with: Entities::Branch, - current_user: current_user, - project: user_project, - merged_branch_names: merged_branch_names - ) - end + expiry_time = if Feature.enabled?(:increase_branch_cache_expiry, type: :ops) + 60.minutes + else + 10.minutes + end + + present_cached( + branches, + with: Entities::Branch, + current_user: current_user, + project: user_project, + merged_branch_names: merged_branch_names, + expires_in: expiry_time, + cache_context: -> (branch) { [current_user&.cache_key, merged_branch_names.include?(branch.name)] } + ) end end @@ -146,7 +142,8 @@ module API branch = find_branch!(params[:branch]) protected_branch = user_project.protected_branches.find_by(name: branch.name) - protected_branch&.destroy + + ::ProtectedBranches::DestroyService.new(user_project, current_user).execute(protected_branch) if protected_branch present branch, with: Entities::Branch, current_user: current_user, project: user_project end diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index b843404e9d7..b3a0a9ef54a 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -143,7 +143,7 @@ module API reject_if_build_artifacts_size_refreshing!(build.project) - build.erase_erasable_artifacts! + ::Ci::JobArtifacts::DeleteService.new(build).execute status :no_content end diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index cd5f1f77ced..6049993bf6f 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -142,7 +142,8 @@ module API reject_if_build_artifacts_size_refreshing!(build.project) - build.erase(erased_by: current_user) + ::Ci::BuildEraseService.new(build, current_user).execute + present build, with: Entities::Ci::Job end @@ -209,8 +210,8 @@ module API .select { |_role, role_access_level| role_access_level <= user_access_level } .map(&:first) - environment = if environment_slug = current_authenticated_job.persisted_environment&.slug - { slug: environment_slug } + environment = if persisted_environment = current_authenticated_job.persisted_environment + { tier: persisted_environment.tier, slug: persisted_environment.slug } end # See https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_ci_access.md#apiv4joballowed_agents-api diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index ec9b09a3419..4b578f8b7e5 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -93,7 +93,7 @@ module API params[:active] = !params.delete(:paused) if params.include?(:paused) update_service = ::Ci::Runners::UpdateRunnerService.new(runner) - if update_service.update(declared_params(include_missing: false)) + if update_service.execute(declared_params(include_missing: false)).success? present runner, with: Entities::Ci::RunnerDetails, current_user: current_user else render_validation_error!(runner) @@ -129,8 +129,17 @@ module API authenticate_list_runners_jobs!(runner) jobs = ::Ci::RunnerJobsFinder.new(runner, current_user, params).execute + jobs = jobs.preload( # rubocop: disable CodeReuse/ActiveRecord + [ + :user, + { pipeline: { project: [:route, { namespace: :route }] } }, + { project: [:route, { namespace: :route }] } + ] + ) + jobs = paginate(jobs) + jobs.each(&:commit) # batch loads all commits in the page - present paginate(jobs), with: Entities::Ci::JobBasicWithProject + present jobs, with: Entities::Ci::JobBasicWithProject end desc 'Reset runner authentication token' do @@ -352,7 +361,7 @@ module API def authenticate_list_runners_jobs!(runner) return if current_user.admin? - forbidden!("No access granted") unless can?(current_user, :read_runner, runner) + forbidden!("No access granted") unless can?(current_user, :read_builds, runner) end end end diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index de59cb4a7c3..d9806fa37d1 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -150,17 +150,18 @@ module API get 'archives/*package_name', urgency: :default do authorize_read_package!(authorized_user_project) - metadata = authorized_user_project + package = authorized_user_project .packages .composer .with_name(params[:package_name]) .with_composer_target(params[:sha]) .first - &.composer_metadatum + metadata = package&.composer_metadatum not_found! unless metadata track_package_event('pull_package', :composer, project: authorized_user_project, namespace: authorized_user_project.namespace) + package.touch_last_downloaded_at send_git_archive authorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true end diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index a90269b565c..d8c2eb4ff33 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -135,7 +135,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true get 'packages/:conan_package_reference', urgency: :low do - authorize!(:read_package, project) + authorize_read_package!(project) presenter = ::Packages::Conan::PackagePresenter.new( package, @@ -154,7 +154,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true get urgency: :low do - authorize!(:read_package, project) + authorize_read_package!(project) presenter = ::Packages::Conan::PackagePresenter.new(package, current_user, project) @@ -237,7 +237,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true post 'packages/:conan_package_reference/upload_urls', urgency: :low do - authorize!(:read_package, project) + authorize_read_package!(project) status 200 present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls @@ -250,7 +250,7 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true post 'upload_urls', urgency: :low do - authorize!(:read_package, project) + authorize_read_package!(project) status 200 present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index e8d27448f02..2883944a745 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -35,12 +35,30 @@ module API ::Packages::Debian::DistributionsFinder.new(container, codename_or_suite: params[:distribution]).execute.last! end - def present_package_file! + def present_distribution_package_file! not_found! unless params[:package_name].start_with?(params[:letter]) package_file = distribution_from!(user_project).package_files.with_file_name(params[:file_name]).last! - present_carrierwave_file!(package_file.file) + present_package_file!(package_file) + end + + def present_index_file!(file_type) + relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize + + relation = relation + .preload_distribution + .with_container(project_or_group) + .with_codename_or_suite(params[:distribution]) + .with_component_name(params[:component]) + .with_file_type(file_type) + .with_architecture_name(params[:architecture]) + .with_compression_type(nil) + .order_created_asc + + relation = relation.with_file_sha256(params[:file_sha256]) if params[:file_sha256] + + present_carrierwave_file!(relation.last!.file) end end @@ -66,6 +84,7 @@ module API namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg + # https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files desc 'The Release file signature' do detail 'This feature was introduced in GitLab 13.5' end @@ -76,6 +95,7 @@ module API end # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release + # https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files desc 'The unsigned Release file' do detail 'This feature was introduced in GitLab 13.5' end @@ -86,6 +106,7 @@ module API end # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease + # https://wiki.debian.org/DebianRepository/Format#A.22Release.22_files desc 'The signed Release file' do detail 'This feature was introduced in GitLab 13.5' end @@ -97,31 +118,87 @@ module API params do requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex end - namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages - desc 'The binary files index' do - detail 'This feature was introduced in GitLab 13.5' + namespace ':component', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + params do + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + end + + namespace 'debian-installer/binary-:architecture' do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/Packages + # https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices + desc 'The installer (udeb) binary files index' do + detail 'This feature was introduced in GitLab 15.4' + end + + route_setting :authentication, authenticate_non_public: true + get 'Packages' do + present_index_file!(:di_packages) + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/debian-installer/binary-:architecture/by-hash/SHA256/:file_sha256 + # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 + desc 'The installer (udeb) binary files index by hash' do + detail 'This feature was introduced in GitLab 15.4' + end + + route_setting :authentication, authenticate_non_public: true + get 'by-hash/SHA256/:file_sha256' do + present_index_file!(:di_packages) + end + end + + namespace 'source', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/source/Sources + # https://wiki.debian.org/DebianRepository/Format#A.22Sources.22_Indices + desc 'The source files index' do + detail 'This feature was introduced in GitLab 15.4' + end + + route_setting :authentication, authenticate_non_public: true + get 'Sources' do + present_index_file!(:sources) + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/source/by-hash/SHA256/:file_sha256 + # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 + desc 'The source files index by hash' do + detail 'This feature was introduced in GitLab 15.4' + end + + route_setting :authentication, authenticate_non_public: true + get 'by-hash/SHA256/:file_sha256' do + present_index_file!(:sources) + end + end + + params do + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex end - route_setting :authentication, authenticate_non_public: true - get 'Packages' do - relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize - - component_file = relation - .preload_distribution - .with_container(project_or_group) - .with_codename_or_suite(params[:distribution]) - .with_component_name(params[:component]) - .with_file_type(:packages) - .with_architecture_name(params[:architecture]) - .with_compression_type(nil) - .order_created_asc - .last! - - present_carrierwave_file!(component_file.file) + namespace 'binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + # https://wiki.debian.org/DebianRepository/Format#A.22Packages.22_Indices + desc 'The binary files index' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, authenticate_non_public: true + get 'Packages' do + present_index_file!(:packages) + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/by-hash/SHA256/:file_sha256 + # https://wiki.debian.org/DebianRepository/Format?action=show&redirect=RepositoryFormat#indices_acquisition_via_hashsums_.28by-hash.29 + desc 'The binary files index by hash' do + detail 'This feature was introduced in GitLab 15.4' + end + + route_setting :authentication, authenticate_non_public: true + get 'by-hash/SHA256/:file_sha256' do + present_index_file!(:packages) + end end end end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 8bf4ac22802..0962d749558 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -48,7 +48,7 @@ module API route_setting :authentication, authenticate_non_public: true get 'pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do - present_package_file! + present_distribution_package_file! end end end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 06846d8f36e..9dedc4390f7 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -51,7 +51,7 @@ module API route_setting :authentication, authenticate_non_public: true get 'pool/:distribution/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do - present_package_file! + present_distribution_package_file! end params do diff --git a/lib/api/entities/batched_background_migration.rb b/lib/api/entities/batched_background_migration.rb new file mode 100644 index 00000000000..eba17ff98f4 --- /dev/null +++ b/lib/api/entities/batched_background_migration.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class BatchedBackgroundMigration < Grape::Entity + expose :id + expose :job_class_name + expose :table_name + expose :status, &:status_name + expose :progress + expose :created_at + end + end +end diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb index 0badde4089e..3d9318ec428 100644 --- a/lib/api/entities/ci/job_basic.rb +++ b/lib/api/entities/ci/job_basic.rb @@ -18,6 +18,12 @@ module API expose :web_url do |job, _options| Gitlab::Routing.url_helpers.project_job_url(job.project, job) end + + expose :project do + expose :ci_job_token_scope_enabled do |job| + job.project.ci_job_token_scope_enabled? + end + end end end end diff --git a/lib/api/entities/ci/job_request/image.rb b/lib/api/entities/ci/job_request/image.rb index 83f64da6050..92d68269265 100644 --- a/lib/api/entities/ci/job_request/image.rb +++ b/lib/api/entities/ci/job_request/image.rb @@ -8,7 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port - expose :pull_policy, if: ->(_) { ::Feature.enabled?(:ci_docker_image_pull_policy) } + expose :pull_policy end end end diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb index 7d494c7e516..128591058fe 100644 --- a/lib/api/entities/ci/job_request/service.rb +++ b/lib/api/entities/ci/job_request/service.rb @@ -8,7 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port - expose :pull_policy, if: ->(_) { ::Feature.enabled?(:ci_docker_image_pull_policy) } + expose :pull_policy expose :alias, :command expose :variables end diff --git a/lib/api/entities/merge_request_reviewer.rb b/lib/api/entities/merge_request_reviewer.rb index 3bf2ccc36aa..a47321ef929 100644 --- a/lib/api/entities/merge_request_reviewer.rb +++ b/lib/api/entities/merge_request_reviewer.rb @@ -4,7 +4,6 @@ module API module Entities class MergeRequestReviewer < Grape::Entity expose :reviewer, as: :user, using: Entities::UserBasic - expose :updated_state_by, using: Entities::UserBasic expose :state expose :created_at end diff --git a/lib/api/entities/ml/mlflow/experiment.rb b/lib/api/entities/ml/mlflow/experiment.rb new file mode 100644 index 00000000000..cfe366feaab --- /dev/null +++ b/lib/api/entities/ml/mlflow/experiment.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class Experiment < Grape::Entity + expose :experiment do + expose :experiment_id + expose :name + expose :lifecycle_stage + expose :artifact_location + end + + private + + def lifecycle_stage + object.deleted_on? ? 'deleted' : 'active' + end + + def experiment_id + object.iid.to_s + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/new_experiment.rb b/lib/api/entities/ml/mlflow/new_experiment.rb new file mode 100644 index 00000000000..09791839850 --- /dev/null +++ b/lib/api/entities/ml/mlflow/new_experiment.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class NewExperiment < Grape::Entity + expose :experiment_id + + private + + def experiment_id + object.iid.to_s + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/run.rb b/lib/api/entities/ml/mlflow/run.rb new file mode 100644 index 00000000000..c679330206e --- /dev/null +++ b/lib/api/entities/ml/mlflow/run.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class Run < Grape::Entity + expose :run do + expose(:info) { |candidate| RunInfo.represent(candidate) } + expose(:data) { |candidate| {} } + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/run_info.rb b/lib/api/entities/ml/mlflow/run_info.rb new file mode 100644 index 00000000000..096950e349d --- /dev/null +++ b/lib/api/entities/ml/mlflow/run_info.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class RunInfo < Grape::Entity + expose :run_id + expose :run_id, as: :run_uuid + expose(:experiment_id) { |candidate| candidate.experiment.iid.to_s } + expose(:start_time) { |candidate| candidate.start_time || 0 } + expose :end_time, expose_nil: false + expose(:status) { |candidate| candidate.status.to_s.upcase } + expose(:artifact_uri) { |candidate| 'not_implemented' } + expose(:lifecycle_stage) { |candidate| 'active' } + expose(:user_id) { |candidate| candidate.user_id.to_s } + + private + + def run_id + object.iid.to_s + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/update_run.rb b/lib/api/entities/ml/mlflow/update_run.rb new file mode 100644 index 00000000000..5acdaab0e33 --- /dev/null +++ b/lib/api/entities/ml/mlflow/update_run.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class UpdateRun < Grape::Entity + expose :run_info + + private + + def run_info + ::API::Entities::Ml::Mlflow::RunInfo.represent object + end + end + end + end + end +end diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index 1efd457aa5f..18fc0576dd4 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -39,6 +39,7 @@ module API end expose :created_at + expose :last_downloaded_at expose :project_id, if: ->(_, opts) { opts[:group] } expose :project_path, if: ->(obj, opts) { opts[:group] && Ability.allowed?(opts[:user], :read_project, obj.project) } expose :tags diff --git a/lib/api/entities/personal_access_token_with_details.rb b/lib/api/entities/personal_access_token_with_details.rb deleted file mode 100644 index 5654bd4a1e1..00000000000 --- a/lib/api/entities/personal_access_token_with_details.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - class PersonalAccessTokenWithDetails < Entities::PersonalAccessToken - expose :expired?, as: :expired - expose :expires_soon?, as: :expires_soon - expose :revoke_path do |token| - Gitlab::Routing.url_helpers.revoke_profile_personal_access_token_path(token) - end - end - end -end diff --git a/lib/api/entities/user_safe.rb b/lib/api/entities/user_safe.rb index fb99c2e960d..127a8ef2160 100644 --- a/lib/api/entities/user_safe.rb +++ b/lib/api/entities/user_safe.rb @@ -3,9 +3,13 @@ module API module Entities class UserSafe < Grape::Entity + include RequestAwareEntity + expose :id, :username expose :name do |user| - user.redacted_name(options[:current_user]) + current_user = request.respond_to?(:current_user) ? request.current_user : options.fetch(:current_user, nil) + + user.redacted_name(current_user) end end end diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 0b1c06b3c26..ad5455c5de6 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -102,7 +102,7 @@ module API track_package_event('pull_package', :generic, project: project, user: current_user, namespace: project.namespace) - present_carrierwave_file!(package_file.file) + present_package_file!(package_file) end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 82bbab5d7d4..6b1fc0d4279 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -96,9 +96,9 @@ module API present options[:with].prepare_relation(projects, options), options end - def present_groups(params, groups) + def present_groups(params, groups, serializer: Entities::Group) options = { - with: Entities::Group, + with: serializer, current_user: current_user, statistics: params[:statistics] && current_user&.admin? } @@ -248,6 +248,8 @@ module API authorize! :admin_group, group + group.remove_avatar! if params.key?(:avatar) && params[:avatar].nil? + if update_group(group) present_group_details(params, group, with_projects: true) else @@ -392,6 +394,21 @@ module API end end + desc 'Get the groups to where the current group can be transferred to' + params do + optional :search, type: String, desc: 'Return list of namespaces matching the search criteria' + use :pagination + end + get ':id/transfer_locations', feature_category: :subgroups do + authorize! :admin_group, user_group + args = declared_params(include_missing: false) + + groups = ::Groups::AcceptingGroupTransfersFinder.new(current_user, user_group, args).execute + groups = groups.with_route + + present_groups params, groups, serializer: Entities::PublicGroupDetails + end + desc 'Transfer a group to a new parent group or promote a subgroup to a root group' params do optional :group_id, diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index a1b265bc8f3..f90084a7e57 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -67,7 +67,7 @@ module API track_package_event('pull_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) - present_carrierwave_file!(package_file.file) + present_package_file!(package_file) end desc 'Authorize a chart upload from workhorse' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1d0f0c6e7bb..e29d76a5950 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -7,9 +7,12 @@ module API include Helpers::Pagination include Helpers::PaginationStrategies include Gitlab::Ci::Artifacts::Logger + include Gitlab::Utils::StrongMemoize SUDO_HEADER = "HTTP_SUDO" GITLAB_SHARED_SECRET_HEADER = "Gitlab-Shared-Secret" + GITLAB_SHELL_API_HEADER = "Gitlab-Shell-Api-Request" + GITLAB_SHELL_JWT_ISSUER = "gitlab-shell" SUDO_PARAM = :sudo API_USER_ENV = 'gitlab.api.user' API_TOKEN_ENV = 'gitlab.api.token' @@ -283,12 +286,22 @@ module API end def authenticate_by_gitlab_shell_token! - input = params['secret_token'] - input ||= Base64.decode64(headers[GITLAB_SHARED_SECRET_HEADER]) if headers.key?(GITLAB_SHARED_SECRET_HEADER) + if Feature.enabled?(:gitlab_shell_jwt_token) + begin + payload, _ = JSONWebToken::HMACToken.decode(headers[GITLAB_SHELL_API_HEADER], secret_token) + unauthorized! unless payload['iss'] == GITLAB_SHELL_JWT_ISSUER + rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex + Gitlab::ErrorTracking.track_exception(ex) + unauthorized! + end + else + input = params['secret_token'] + input ||= Base64.decode64(headers[GITLAB_SHARED_SECRET_HEADER]) if headers.key?(GITLAB_SHARED_SECRET_HEADER) - input&.chomp! + input&.chomp! - unauthorized! unless Devise.secure_compare(secret_token, input) + unauthorized! unless Devise.secure_compare(secret_token, input) + end end def authenticated_with_can_read_all_resources! @@ -719,7 +732,13 @@ module API end def secret_token - Gitlab::Shell.secret_token + if Feature.enabled?(:gitlab_shell_jwt_token) + strong_memoize(:secret_token) do + File.read(Gitlab.config.gitlab_shell.secret_file) + end + else + Gitlab::Shell.secret_token + end end def authenticate_non_public? diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index 2b10eebb009..e9af50b80be 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -11,8 +11,7 @@ module API optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' - # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - optional :avatar, type: File, desc: 'Avatar image for the group' # rubocop:disable Scalability/FileUploads + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for the group' optional :share_with_group_lock, type: Boolean, desc: 'Prevent sharing a project with another group within this group' optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users in this group to setup Two-factor authentication' optional :two_factor_grace_period, type: Integer, desc: 'Time before Two-factor authentication is enforced' diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index 994d3c4c473..a9d91895cfe 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -23,7 +23,7 @@ module API end def present_download_urls(entity) - authorize!(:read_package, project) + authorize_read_package!(project) presenter = ::Packages::Conan::PackagePresenter.new( package, @@ -161,7 +161,7 @@ module API end def download_package_file(file_type) - authorize!(:read_package, project) + authorize_read_package!(project) package_file = ::Packages::Conan::PackageFileFinder .new( @@ -173,7 +173,7 @@ module API track_package_event('pull_package', :conan, category: 'API::ConanPackages', user: current_user, project: project, namespace: project.namespace) if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY - present_carrierwave_file!(package_file.file) + present_package_file!(package_file) end def find_or_create_package diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index b8ae1dddd7e..a09499e00d7 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -6,16 +6,18 @@ module API module DependencyProxyHelpers REGISTRY_BASE_URLS = { npm: 'https://registry.npmjs.org/', - pypi: 'https://pypi.org/simple/' + pypi: 'https://pypi.org/simple/', + maven: 'https://repo.maven.apache.org/maven2/' }.freeze APPLICATION_SETTING_NAMES = { npm: 'npm_package_requests_forwarding', - pypi: 'pypi_package_requests_forwarding' + pypi: 'pypi_package_requests_forwarding', + maven: 'maven_package_requests_forwarding' }.freeze def redirect_registry_request(forward_to_registry, package_type, options) - if forward_to_registry && redirect_registry_request_available?(package_type) + if forward_to_registry && redirect_registry_request_available?(package_type) && maven_forwarding_ff_enabled?(package_type, options[:target]) ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward") redirect(registry_url(package_type, options)) else @@ -33,6 +35,8 @@ module API "#{base_url}#{options[:package_name]}" when :pypi "#{base_url}#{options[:package_name]}/" + when :maven + "#{base_url}#{options[:path]}/#{options[:file_name]}" end end @@ -46,6 +50,16 @@ module API .attributes .fetch(application_setting_name, false) end + + private + + def maven_forwarding_ff_enabled?(package_type, target) + return true unless package_type == :maven + return true if Feature.enabled?(:maven_central_request_forwarding) + return false unless target + + Feature.enabled?(:maven_central_request_forwarding, target.root_ancestor) + end end end end diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 2221eec0f82..687c8330cc8 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -14,7 +14,7 @@ module API end def authorize_read_package!(subject = user_project) - authorize!(:read_package, subject) + authorize!(:read_package, subject.try(:packages_policy_subject) || subject) end def authorize_create_package!(subject = user_project) @@ -53,6 +53,11 @@ module API category = args.delete(:category) || self.options[:for].name ::Gitlab::Tracking.event(category, event_name.to_s, **args) end + + def present_package_file!(package_file, supports_direct_download: true) + package_file.package.touch_last_downloaded_at + present_carrierwave_file!(package_file.file, supports_direct_download: supports_direct_download) + end end end end diff --git a/lib/api/helpers/personal_access_tokens_helpers.rb b/lib/api/helpers/personal_access_tokens_helpers.rb new file mode 100644 index 00000000000..db28daa5396 --- /dev/null +++ b/lib/api/helpers/personal_access_tokens_helpers.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module API + module Helpers + module PersonalAccessTokensHelpers + def finder_params(current_user) + if current_user.can_admin_all_resources? + { user: user(params[:user_id]) } + else + { user: current_user, impersonation: false } + end + end + + def user(user_id) + UserFinder.new(user_id).find_by_id + end + + def restrict_non_admins! + return if params[:user_id].blank? + + unauthorized! unless Ability.allowed?(current_user, :read_user_personal_access_tokens, user(params[:user_id])) + end + + def find_token(id) + PersonalAccessToken.find(id) || not_found! + end + + def revoke_token(token) + service = ::PersonalAccessTokens::RevokeService.new(current_user, token: token).execute + + service.success? ? no_content! : bad_request!(nil) + end + end + end +end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 628182ad1ab..7ca3f55b5a2 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -39,6 +39,7 @@ module API optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis' + optional :show_diff_preview_in_email, type: Boolean, desc: 'Include the code diff preview in merge request notification emails' optional :warn_about_potentially_unwanted_characters, type: Boolean, desc: 'Warn about Potentially Unwanted Characters' optional :enforce_auth_checks_on_uploads, type: Boolean, desc: 'Enforce auth check on uploads' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' @@ -57,8 +58,7 @@ module API optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all threads are resolved' optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :topics instead' optional :topics, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The list of topics for a project' - # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - optional :avatar, type: File, desc: 'Avatar image for project' # rubocop:disable Scalability/FileUploads + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for project' optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' optional :suggestion_commit_message, type: String, desc: 'The commit message used to apply merge request suggestions' @@ -160,6 +160,7 @@ module API :request_access_enabled, :resolve_outdated_diff_discussions, :restrict_user_defined_variables, + :show_diff_preview_in_email, :security_and_compliance_access_level, :squash_option, :shared_runners_enabled, diff --git a/lib/api/helpers/resource_events_helpers.rb b/lib/api/helpers/resource_events_helpers.rb new file mode 100644 index 00000000000..c47a58e8fce --- /dev/null +++ b/lib/api/helpers/resource_events_helpers.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Helpers + module ResourceEventsHelpers + def self.eventable_types + # This is a method instead of a constant, allowing EE to more easily extend it. + { + Issue => { feature_category: :team_planning, id_field: 'IID' }, + MergeRequest => { feature_category: :code_review, id_field: 'IID' } + } + end + end + end +end + +API::Helpers::ResourceEventsHelpers.prepend_mod_with('API::Helpers::ResourceEventsHelpers') diff --git a/lib/api/helpers/resource_label_events_helpers.rb b/lib/api/helpers/resource_label_events_helpers.rb deleted file mode 100644 index eeb68362c1d..00000000000 --- a/lib/api/helpers/resource_label_events_helpers.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module ResourceLabelEventsHelpers - def self.feature_category_per_eventable_type - # This is a method instead of a constant, allowing EE to more easily - # extend it. - { - Issue => :team_planning, - MergeRequest => :code_review - } - end - end - end -end - -API::Helpers::ResourceLabelEventsHelpers.prepend_mod_with('API::Helpers::ResourceLabelEventsHelpers') diff --git a/lib/api/integrations/slack/events.rb b/lib/api/integrations/slack/events.rb deleted file mode 100644 index 6227b75a9d7..00000000000 --- a/lib/api/integrations/slack/events.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -# This API endpoint handles all events sent from Slack once a Slack -# workspace has installed the GitLab Slack app. -# -# See https://api.slack.com/apis/connections/events-api. -module API - class Integrations - module Slack - class Events < ::API::Base - feature_category :integrations - - before { verify_slack_request! } - - helpers do - def verify_slack_request! - unauthorized! unless Request.verify!(request) - end - end - - namespace 'integrations/slack' do - post :events do - type = params['type'] - raise ArgumentError, "Unable to handle event type: '#{type}'" unless type == 'url_verification' - - status :ok - UrlVerification.call(params) - rescue ArgumentError => e - # Track the error, but respond with a `2xx` because we don't want to risk - # Slack rate-limiting, or disabling our app, due to error responses. - # See https://api.slack.com/apis/connections/events-api. - Gitlab::ErrorTracking.track_exception(e) - - no_content! - end - end - end - end - end -end diff --git a/lib/api/integrations/slack/events/url_verification.rb b/lib/api/integrations/slack/events/url_verification.rb deleted file mode 100644 index 4628b93665d..00000000000 --- a/lib/api/integrations/slack/events/url_verification.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module API - class Integrations - module Slack - class Events - class UrlVerification - # When the GitLab Slack app is first configured to receive Slack events, - # Slack will issue a special request to the endpoint and expect it to respond - # with the `challenge` param. - # - # This must be done in-request, rather than on a queue. - # - # See https://api.slack.com/apis/connections/events-api. - def self.call(params) - { challenge: params[:challenge] } - end - end - end - end - end -end diff --git a/lib/api/integrations/slack/request.rb b/lib/api/integrations/slack/request.rb deleted file mode 100644 index df0109b07aa..00000000000 --- a/lib/api/integrations/slack/request.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module API - class Integrations - module Slack - module Request - VERIFICATION_VERSION = 'v0' - VERIFICATION_TIMESTAMP_HEADER = 'X-Slack-Request-Timestamp' - VERIFICATION_SIGNATURE_HEADER = 'X-Slack-Signature' - VERIFICATION_DELIMITER = ':' - VERIFICATION_HMAC_ALGORITHM = 'sha256' - VERIFICATION_TIMESTAMP_EXPIRY = 1.minute.to_i - - # Verify the request by comparing the given request signature in the header - # with a signature value that we compute according to the steps in: - # https://api.slack.com/authentication/verifying-requests-from-slack. - def self.verify!(request) - return false unless Gitlab::CurrentSettings.slack_app_signing_secret - - timestamp, signature = request.headers.values_at( - VERIFICATION_TIMESTAMP_HEADER, - VERIFICATION_SIGNATURE_HEADER - ) - - return false if timestamp.nil? || signature.nil? - return false if Time.current.to_i - timestamp.to_i >= VERIFICATION_TIMESTAMP_EXPIRY - - request.body.rewind - - basestring = [ - VERIFICATION_VERSION, - timestamp, - request.body.read - ].join(VERIFICATION_DELIMITER) - - hmac_digest = OpenSSL::HMAC.hexdigest( - VERIFICATION_HMAC_ALGORITHM, - Gitlab::CurrentSettings.slack_app_signing_secret, - basestring - ) - - # Signature will look like: 'v0=a2114d57b48eac39b9ad189dd8316235a7b4a8d21a10bd27519666489c69b503' - ActiveSupport::SecurityUtils.secure_compare( - signature, - "#{VERIFICATION_VERSION}=#{hmac_digest}" - ) - end - end - end - end -end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 6f475fa8d74..c4464666020 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -133,11 +133,6 @@ module API 'Could not find a user for the given key' unless actor.user end - # TODO: backwards compatibility; remove after https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/454 is merged - def two_factor_otp_check - { success: false, message: 'Feature is not available' } - end - def two_factor_manual_otp_check { success: false, message: 'Feature is not available' } end @@ -339,13 +334,6 @@ module API end end - # TODO: backwards compatibility; remove after https://gitlab.com/gitlab-org/gitlab-shell/-/merge_requests/454 is merged - post '/two_factor_otp_check', feature_category: :authentication_and_authorization do - status 200 - - two_factor_manual_otp_check - end - post '/two_factor_push_otp_check', feature_category: :authentication_and_authorization do status 200 diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index fb0221ee907..a3a25ec1696 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -22,6 +22,7 @@ module API end helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::DependencyProxyHelpers helpers do def path_exists?(path) @@ -76,7 +77,10 @@ module API format == 'jar' end - def present_carrierwave_file_with_head_support!(file, supports_direct_download: true) + def present_carrierwave_file_with_head_support!(package_file, supports_direct_download: true) + package_file.package.touch_last_downloaded_at + file = package_file.file + if head_request_on_aws_file?(file, supports_direct_download) && !file.file_storage? return redirect(signed_head_url(file)) end @@ -110,7 +114,31 @@ module API project || group, path: params[:path], order_by_package_file: order_by_package_file - ).execute! + ).execute + end + + def find_and_present_package_file(package, file_name, format, params) + project = package&.project + package_file = nil + + package_file = ::Packages::PackageFileFinder.new(package, file_name).execute if package + + no_package_found = package_file ? false : true + + redirect_registry_request(no_package_found, :maven, path: params[:path], file_name: params[:file_name], target: params[:target]) do + not_found!('Package') if no_package_found + + case format + when 'md5' + package_file.file_md5 + when 'sha1' + package_file.file_sha1 + else + track_package_event('pull_package', :maven, project: project, namespace: project&.namespace) if jar_file?(format) + + present_carrierwave_file_with_head_support!(package_file) + end + end end end @@ -138,6 +166,8 @@ module API package = fetch_package(file_name: file_name, project: project) + not_found!('Package') unless package + package_file = ::Packages::PackageFileFinder .new(package, file_name).execute! @@ -148,7 +178,7 @@ module API package_file.file_sha1 else track_package_event('pull_package', :maven, project: project, namespace: project.namespace) if jar_file?(format) - present_carrierwave_file_with_head_support!(package_file.file) + present_carrierwave_file_with_head_support!(package_file) end end @@ -166,31 +196,20 @@ module API route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to group = find_group(params[:id]) - not_found!('Group') unless path_exists?(params[:path]) - - file_name, format = extract_format(params[:file_name]) - group = find_group(params[:id]) + if Feature.disabled?(:maven_central_request_forwarding, group&.root_ancestor) + not_found!('Group') unless path_exists?(params[:path]) + end + not_found!('Group') unless can?(current_user, :read_group, group) + file_name, format = extract_format(params[:file_name]) package = fetch_package(file_name: file_name, group: group) - authorize_read_package!(package.project) + authorize_read_package!(package.project) if package - package_file = ::Packages::PackageFileFinder - .new(package, file_name).execute! - - case format - when 'md5' - package_file.file_md5 - when 'sha1' - package_file.file_sha1 - else - track_package_event('pull_package', :maven, project: package.project, namespace: package.project.namespace) if jar_file?(format) - - present_carrierwave_file_with_head_support!(package_file.file) - end + find_and_present_package_file(package, file_name, format, params.merge(target: group)) end end @@ -208,7 +227,9 @@ module API route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do # return a similar failure to user_project - not_found!('Project') unless path_exists?(params[:path]) + unless Feature.enabled?(:maven_central_request_forwarding, user_project&.root_ancestor) + not_found!('Project') unless path_exists?(params[:path]) + end authorize_read_package!(user_project) @@ -216,19 +237,7 @@ module API package = fetch_package(file_name: file_name, project: user_project) - package_file = ::Packages::PackageFileFinder - .new(package, file_name).execute! - - case format - when 'md5' - package_file.file_md5 - when 'sha1' - package_file.file_sha1 - else - track_package_event('pull_package', :maven, project: user_project, namespace: user_project.namespace) if jar_file?(format) - - present_carrierwave_file_with_head_support!(package_file.file) - end + find_and_present_package_file(package, file_name, format, params.merge(target: user_project)) end desc 'Workhorse authorize the maven package file upload' do diff --git a/lib/api/members.rb b/lib/api/members.rb index d26fdd09ee7..f4e38207aca 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -24,6 +24,7 @@ module API params do optional :query, type: String, desc: 'A query string to search for members' optional :user_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Array of user ids to look up for membership' + optional :skip_users, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Array of user ids to be skipped for membership' optional :show_seat_info, type: Boolean, desc: 'Show seat information for members' use :optional_filter_params_ee use :pagination diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a8f58e91067..1dc0e1f0d22 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -212,7 +212,17 @@ module API recheck_mergeability_of(merge_requests: merge_requests) unless options[:skip_merge_status_recheck] - present_cached merge_requests, expires_in: 8.hours, cache_context: -> (mr) { "#{current_user&.cache_key}:#{mr.merge_status}" }, **options + present_cached merge_requests, + expires_in: 8.hours, + cache_context: -> (mr) do + [ + current_user&.cache_key, + mr.merge_status, + mr.merge_request_assignees.map(&:cache_key), + mr.merge_request_reviewers.map(&:cache_key) + ].join(":") + end, + **options end desc 'Create a merge request' do @@ -544,6 +554,19 @@ module API render_api_error!(e.message, 409) end + desc 'Remove merge request approvals' do + detail 'This feature was added in GitLab 15.4' + end + put ':id/merge_requests/:merge_request_iid/reset_approvals', feature_category: :code_review, urgency: :low do + merge_request = find_project_merge_request(params[:merge_request_iid]) + + unauthorized! unless current_user.bot? && merge_request.can_be_approved_by?(current_user) + + merge_request.approvals.delete_all + + status :accepted + end + desc 'List issues that will be closed on merge' do success Entities::MRNote end diff --git a/lib/api/ml/mlflow.rb b/lib/api/ml/mlflow.rb new file mode 100644 index 00000000000..4f5bd42f8f9 --- /dev/null +++ b/lib/api/ml/mlflow.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +require 'mime/types' + +module API + # MLFlow integration API, replicating the Rest API https://www.mlflow.org/docs/latest/rest-api.html#rest-api + module Ml + class Mlflow < ::API::Base + include APIGuard + + # The first part of the url is the namespace, the second part of the URL is what the MLFlow client calls + MLFLOW_API_PREFIX = ':id/ml/mflow/api/2.0/mlflow/' + + allow_access_with_scope :api + allow_access_with_scope :read_api, if: -> (request) { request.get? || request.head? } + + before do + authenticate! + not_found! unless Feature.enabled?(:ml_experiment_tracking, user_project) + end + + feature_category :mlops + + content_type :json, 'application/json' + default_format :json + + helpers do + def resource_not_found! + render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404) + end + + def resource_already_exists! + render_structured_api_error!({ error_code: 'RESOURCE_ALREADY_EXISTS' }, 400) + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'API to interface with MLFlow Client, REST API version 1.28.0' do + detail 'This feature is gated by :ml_experiment_tracking.' + end + namespace MLFLOW_API_PREFIX do + resource :experiments do + desc 'Fetch experiment by experiment_id' do + success Entities::Ml::Mlflow::Experiment + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment' + end + params do + optional :experiment_id, type: String, default: '', desc: 'Experiment ID, in reference to the project' + end + get 'get', urgency: :low do + experiment = ::Ml::Experiment.by_project_id_and_iid(user_project.id, params[:experiment_id]) + + resource_not_found! unless experiment + + present experiment, with: Entities::Ml::Mlflow::Experiment + end + + desc 'Fetch experiment by experiment_name' do + success Entities::Ml::Mlflow::Experiment + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-experiment-by-name' + end + params do + optional :experiment_name, type: String, default: '', desc: 'Experiment name' + end + get 'get-by-name', urgency: :low do + experiment = ::Ml::Experiment.by_project_id_and_name(user_project, params[:experiment_name]) + + resource_not_found! unless experiment + + present experiment, with: Entities::Ml::Mlflow::Experiment + end + + desc 'Create experiment' do + success Entities::Ml::Mlflow::NewExperiment + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#create-experiment' + end + params do + requires :name, type: String, desc: 'Experiment name' + optional :artifact_location, type: String, desc: 'This will be ignored' + optional :tags, type: Array, desc: 'This will be ignored' + end + post 'create', urgency: :low do + resource_already_exists! if ::Ml::Experiment.has_record?(user_project.id, params[:name]) + + experiment = ::Ml::Experiment.create!(name: params[:name], + user: current_user, + project: user_project) + + present experiment, with: Entities::Ml::Mlflow::NewExperiment + end + end + + resource :runs do + desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do + success Entities::Ml::Mlflow::Run + detail 'https://www.mlflow.org/docs/1.28.0/rest-api.html#get-run' + end + params do + optional :run_id, type: String, desc: 'UUID of the candidate.' + optional :run_uuid, type: String, desc: 'This parameter is ignored' + end + get 'get', urgency: :low do + candidate = ::Ml::Candidate.with_project_id_and_iid(user_project.id, params[:run_id]) + + resource_not_found! unless candidate + + present candidate, with: Entities::Ml::Mlflow::Run + end + + desc 'Creates a Run.' do + success Entities::Ml::Mlflow::Run + detail ['https://www.mlflow.org/docs/1.28.0/rest-api.html#create-run', + 'MLFlow Runs map to GitLab Candidates'] + end + params do + requires :experiment_id, type: Integer, + desc: 'Id for the experiment, relative to the project' + optional :start_time, type: Integer, + desc: 'Unix timestamp in milliseconds of when the run started.', + default: 0 + optional :user_id, type: String, desc: 'This will be ignored' + optional :tags, type: Array, desc: 'This will be ignored' + end + post 'create', urgency: :low do + experiment = ::Ml::Experiment.by_project_id_and_iid(user_project.id, params[:experiment_id].to_i) + + resource_not_found! unless experiment + + candidate = ::Ml::Candidate.create!( + experiment: experiment, + user: current_user, + start_time: params[:start_time] || 0 + ) + + present candidate, with: Entities::Ml::Mlflow::Run + end + + desc 'Updates a Run.' do + success Entities::Ml::Mlflow::UpdateRun + detail ['https://www.mlflow.org/docs/1.28.0/rest-api.html#update-run', + 'MLFlow Runs map to GitLab Candidates'] + end + params do + optional :run_id, type: String, desc: 'UUID of the candidate.' + optional :status, type: String, + values: ::Ml::Candidate.statuses.keys.map(&:upcase), + desc: "Status of the run. Accepts: " \ + "#{::Ml::Candidate.statuses.keys.map(&:upcase)}." + optional :end_time, type: Integer, desc: 'Ending time of the run' + end + post 'update', urgency: :low do + candidate = ::Ml::Candidate.with_project_id_and_iid(user_project.id, params[:run_id]) + + resource_not_found! unless candidate + + candidate.status = params[:status].downcase if params[:status] + candidate.end_time = params[:end_time] if params[:end_time] + + candidate.save if candidate.valid? + + present candidate, with: Entities::Ml::Mlflow::UpdateRun + end + end + end + end + end + end +end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index a12fbbb9bb6..eeb66c86b3b 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -66,6 +66,8 @@ module API optional :parent_id, type: Integer, desc: "The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered." end get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do + check_rate_limit!(:namespace_exists, scope: current_user) + namespace_path = params[:namespace] existing_namespaces_within_the_parent = Namespace.without_project_namespaces.by_parent(params[:parent_id]) diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 21bb2e69799..166c0b755fe 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -35,7 +35,7 @@ module API track_package_event('pull_package', package, category: 'API::NpmPackages', project: project, namespace: project.namespace) - present_carrierwave_file!(package_file.file) + present_package_file!(package_file) end desc 'Create NPM package' do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 1e630cffea1..3e05ea13311 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -193,7 +193,7 @@ module API ) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false - present_carrierwave_file!(package_file.file, supports_direct_download: false) + present_package_file!(package_file, supports_direct_download: false) end end end diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index 0d7d2dc6a0c..1c00569bba2 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -18,34 +18,10 @@ module API before do authenticate! - restrict_non_admins! unless current_user.admin? + restrict_non_admins! unless current_user.can_admin_all_resources? end - helpers do - def finder_params(current_user) - current_user.admin? ? { user: user(params[:user_id]) } : { user: current_user, impersonation: false } - end - - def user(user_id) - UserFinder.new(user_id).find_by_id - end - - def restrict_non_admins! - return if params[:user_id].blank? - - unauthorized! unless Ability.allowed?(current_user, :read_user_personal_access_tokens, user(params[:user_id])) - end - - def find_token(id) - PersonalAccessToken.find(id) || not_found! - end - - def revoke_token(token) - service = ::PersonalAccessTokens::RevokeService.new(current_user, token: token).execute - - service.success? ? no_content! : bad_request!(nil) - end - end + helpers ::API::Helpers::PersonalAccessTokensHelpers resources :personal_access_tokens do get do @@ -63,14 +39,10 @@ module API present token, with: Entities::PersonalAccessToken else # Only admins should be informed if the token doesn't exist - current_user.admin? ? not_found! : unauthorized! + current_user.can_admin_all_resources? ? not_found! : unauthorized! end end - delete 'self' do - revoke_token(access_token) - end - delete ':id' do token = find_token(params[:id]) diff --git a/lib/api/personal_access_tokens/self_revocation.rb b/lib/api/personal_access_tokens/self_revocation.rb new file mode 100644 index 00000000000..22e07f4cc7b --- /dev/null +++ b/lib/api/personal_access_tokens/self_revocation.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module API + class PersonalAccessTokens + class SelfRevocation < ::API::Base + include APIGuard + + feature_category :authentication_and_authorization + + helpers ::API::Helpers::PersonalAccessTokensHelpers + + # As any token regardless of `scope` should be able to revoke itself + # all availabe scopes are allowed for this API class. + # Please be aware of the permissive scope when adding new endpoints to this class. + allow_access_with_scope(Gitlab::Auth.all_available_scopes) + + before { authenticate! } + + resource :personal_access_tokens do + delete 'self' do + revoke_token(access_token) + end + end + end + end +end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6ed480518ee..8c58cc585d8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -453,6 +453,8 @@ module API filter_attributes_using_license!(attrs) verify_update_project_attrs!(user_project, attrs) + user_project.remove_avatar! if attrs.key?(:avatar) && attrs[:avatar].nil? + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success @@ -743,6 +745,22 @@ module API end end + desc 'Get the namespaces to where the project can be transferred' + params do + optional :search, type: String, desc: 'Return list of namespaces matching the search criteria' + use :pagination + end + get ":id/transfer_locations", feature_category: :projects do + authorize! :change_namespace, user_project + args = declared_params(include_missing: false) + args[:permission_scope] = :transfer_projects + + groups = ::Groups::UserGroupsFinder.new(current_user, current_user, args).execute + groups = groups.with_route + + present_groups(groups) + end + desc 'Show the storage information' do success Entities::ProjectRepositoryStorage end diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index f8a7a3c0ecc..ae583ca968a 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -120,7 +120,7 @@ module API track_package_event('pull_package', :pypi) - present_carrierwave_file!(package_file.file, supports_direct_download: true) + present_package_file!(package_file, supports_direct_download: true) end desc 'The PyPi Simple Group Index Endpoint' do @@ -180,7 +180,7 @@ module API track_package_event('pull_package', :pypi, project: project, namespace: project.namespace) - present_carrierwave_file!(package_file.file, supports_direct_download: true) + present_package_file!(package_file, supports_direct_download: true) end desc 'The PyPi Simple Project Index Endpoint' do diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 10e879ec70b..cdfcce9dddb 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -100,6 +100,62 @@ module API present release, with: Entities::Release, current_user: current_user, include_html_description: params[:include_html_description] end + desc 'Download a project release asset file' do + detail 'This feature was introduced in GitLab 15.4.' + named 'download_release_asset_file' + end + params do + requires :tag_name, type: String, + desc: 'The name of the tag.', as: :tag + requires :file_path, type: String, + file_path: true, + desc: 'The path to the file to download, as specified when creating the release asset.' + end + route_setting :authentication, job_token_allowed: true + get ':id/releases/:tag_name/downloads/*file_path', format: false, requirements: RELEASE_ENDPOINT_REQUIREMENTS do + authorize_download_code! + + not_found! unless release + + link = release.links.find_by_filepath!("/#{params[:file_path]}") + + not_found! unless link + + redirect link.url + end + + desc 'Get the latest project release' do + detail 'This feature was introduced in GitLab 15.4.' + named 'get_latest_release' + end + params do + requires :suffix_path, type: String, file_path: true, desc: 'The path to be suffixed to the latest release' + end + route_setting :authentication, job_token_allowed: true + get ':id/releases/permalink/latest(/)(*suffix_path)', format: false, requirements: RELEASE_ENDPOINT_REQUIREMENTS do + authorize_download_code! + + # Try to find the latest release + latest_release = find_latest_release + not_found! unless latest_release + + # Build the full API URL with the tag of the latest release + redirect_url = api_v4_projects_releases_path(id: user_project.id, tag_name: latest_release.tag) + + # Include the additional suffix_path if present + redirect_url += "/#{params[:suffix_path]}" if params[:suffix_path].present? + + # Include any query parameter except `order_by` since we have plans to extend it in the future. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/352945 for reference. + query_parameters_except_order_by = get_query_params.except('order_by') + + if query_parameters_except_order_by.present? + redirect_url += "?#{query_parameters_except_order_by.compact.to_param}" + end + + redirect redirect_url + end + desc 'Create a new release' do detail 'This feature was introduced in GitLab 11.7.' named 'create_release' @@ -232,6 +288,16 @@ module API @release ||= user_project.releases.find_by_tag(params[:tag]) end + def find_latest_release + ReleasesFinder.new(user_project, current_user, { order_by: 'released_at', sort: 'desc' }).execute.first + end + + def get_query_params + return {} unless @request.query_string.present? + + Rack::Utils.parse_nested_query(@request.query_string) + end + def log_release_created_audit_event(release) # extended in EE end diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb index cd56809f45a..e74b6509a17 100644 --- a/lib/api/resource_label_events.rb +++ b/lib/api/resource_label_events.rb @@ -7,20 +7,22 @@ module API before { authenticate! } - Helpers::ResourceLabelEventsHelpers.feature_category_per_eventable_type.each do |eventable_type, feature_category| + Helpers::ResourceEventsHelpers.eventable_types.each do |eventable_type, details| parent_type = eventable_type.parent_class.to_s.underscore eventables_str = eventable_type.to_s.underscore.pluralize + human_eventable_str = eventable_type.to_s.underscore.humanize.downcase + feature_category = details[:feature_category] params do requires :id, type: String, desc: "The ID of a #{parent_type}" end resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "Get a list of #{eventable_type.to_s.downcase} resource label events" do + desc "Get a list of #{human_eventable_str} resource label events" do success Entities::ResourceLabelEvent detail 'This feature was introduced in 11.3' end params do - requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + requires :eventable_id, types: [Integer, String], desc: "The #{details[:id_field]} of the #{human_eventable_str}" use :pagination end @@ -32,13 +34,13 @@ module API present ResourceLabelEvent.visible_to_user?(current_user, paginate(events)), with: Entities::ResourceLabelEvent end - desc "Get a single #{eventable_type.to_s.downcase} resource label event" do + desc "Get a single #{human_eventable_str} resource label event" do success Entities::ResourceLabelEvent detail 'This feature was introduced in 11.3' end params do requires :event_id, type: String, desc: 'The ID of a resource label event' - requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + requires :eventable_id, types: [Integer, String], desc: "The #{details[:id_field]} of the #{human_eventable_str}" end get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id", feature_category: feature_category do eventable = find_noteable(eventable_type, params[:eventable_id]) diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb index 4b92f320d6f..f817d55c505 100644 --- a/lib/api/resource_state_events.rb +++ b/lib/api/resource_state_events.rb @@ -7,41 +7,41 @@ module API before { authenticate! } - { - Issue => :team_planning, - MergeRequest => :code_review - }.each do |eventable_class, feature_category| - eventable_name = eventable_class.to_s.underscore + Helpers::ResourceEventsHelpers.eventable_types.each do |eventable_type, details| + parent_type = eventable_type.parent_class.to_s.underscore + eventables_str = eventable_type.to_s.underscore.pluralize + human_eventable_str = eventable_type.to_s.underscore.humanize.downcase + feature_category = details[:feature_category] params do - requires :id, type: String, desc: "The ID of a project" + requires :id, type: String, desc: "The ID of a #{parent_type}" end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "Get a list of #{eventable_class.to_s.downcase} resource state events" do + resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc "Get a list of #{human_eventable_str} resource state events" do success Entities::ResourceStateEvent end params do - requires :eventable_iid, types: Integer, desc: "The IID of the #{eventable_name}" + requires :eventable_id, types: Integer, desc: "The #{details[:id_field]} of the #{human_eventable_str}" use :pagination end - get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events", feature_category: feature_category, urgency: :low do - eventable = find_noteable(eventable_class, params[:eventable_iid]) + get ":id/#{eventables_str}/:eventable_id/resource_state_events", feature_category: feature_category, urgency: :low do + eventable = find_noteable(eventable_type, params[:eventable_id]) events = ResourceStateEventFinder.new(current_user, eventable).execute present paginate(events), with: Entities::ResourceStateEvent end - desc "Get a single #{eventable_class.to_s.downcase} resource state event" do + desc "Get a single #{human_eventable_str} resource state event" do success Entities::ResourceStateEvent end params do - requires :eventable_iid, types: Integer, desc: "The IID of the #{eventable_name}" + requires :eventable_id, types: Integer, desc: "The #{details[:id_field]} of the #{human_eventable_str}" requires :event_id, type: Integer, desc: 'The ID of a resource state event' end - get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events/:event_id", feature_category: feature_category do - eventable = find_noteable(eventable_class, params[:eventable_iid]) + get ":id/#{eventables_str}/:eventable_id/resource_state_events/:event_id", feature_category: feature_category do + eventable = find_noteable(eventable_type, params[:eventable_id]) event = ResourceStateEventFinder.new(current_user, eventable).find(params[:event_id]) diff --git a/lib/api/rpm_project_packages.rb b/lib/api/rpm_project_packages.rb new file mode 100644 index 00000000000..d17470ae92d --- /dev/null +++ b/lib/api/rpm_project_packages.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +module API + class RpmProjectPackages < ::API::Base + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + include ::API::Helpers::Authentication + + feature_category :package_registry + + before do + require_packages_enabled! + + not_found! unless ::Feature.enabled?(:rpm_packages, authorized_user_project) + + authorize_read_package!(authorized_user_project) + end + + authenticate_with do |accept| + accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) + .sent_through(:http_basic_auth) + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/rpm' do + desc 'Download repository metadata files' + params do + requires :file_name, type: String, desc: 'Repository metadata file name' + end + get 'repodata/*file_name', requirements: { file_name: API::NO_SLASH_URL_PART_REGEX } do + not_found! + end + + desc 'Download RPM package files' + params do + requires :package_file_id, type: Integer, desc: 'RPM package file id' + requires :file_name, type: String, desc: 'RPM package file name' + end + get '*package_file_id/*file_name', requirements: { file_name: API::NO_SLASH_URL_PART_REGEX } do + not_found! + end + + desc 'Upload a RPM package' + post do + authorize_create_package!(authorized_user_project) + + if authorized_user_project.actual_limits.exceeded?(:rpm_max_file_size, params[:file].size) + bad_request!('File is too large') + end + + not_found! + end + + desc 'Authorize package upload from workhorse' + post 'authorize' do + not_found! + end + end + end + end +end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 85bbd0879b7..b4d02613e4c 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -65,7 +65,7 @@ module API requires :file_name, type: String, desc: 'Package file name' end get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do - authorize!(:read_package, user_project) + authorize_read_package!(user_project) package_files = ::Packages::PackageFile .for_rubygem_with_file_name(user_project, params[:file_name]) @@ -74,7 +74,7 @@ module API track_package_event('pull_package', :rubygems, project: user_project, namespace: user_project.namespace) - present_carrierwave_file!(package_file.file) + present_package_file!(package_file) end namespace 'api/v1' do diff --git a/lib/api/search.rb b/lib/api/search.rb index 7aa3cf8a5cb..44bb4228786 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -65,9 +65,26 @@ module API set_global_search_log_information + Gitlab::Metrics::GlobalSearchSlis.record_apdex( + elapsed: @search_duration_s, + search_type: search_type, + search_level: search_service.level, + search_scope: search_scope + ) + Gitlab::UsageDataCounters::SearchCounter.count(:all_searches) paginate(@results) + + ensure + # If we raise an error somewhere in the @search_duration_s benchmark block, we will end up here + # with a 200 status code, but an empty @search_duration_s. + Gitlab::Metrics::GlobalSearchSlis.record_error_rate( + error: @search_duration_s.nil? || (status < 200 || status >= 400), + search_type: search_type, + search_level: search_service(additional_params).level, + search_scope: search_scope + ) end def snippets? diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c25a56d5f08..f393f862f55 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -98,6 +98,7 @@ module API optional :max_export_size, type: Integer, desc: 'Maximum export size in MB' optional :max_import_size, type: Integer, desc: 'Maximum import size in MB' optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' + optional :max_pages_custom_domains_per_project, type: Integer, desc: 'Maximum number of GitLab Pages custom domains per project' optional :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5 optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 97a2aebf53b..c8ac68189f5 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -22,8 +22,8 @@ module API params do optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return tags sorted in updated by `asc` or `desc` order.' - optional :order_by, type: String, values: %w[name updated], default: 'updated', - desc: 'Return tags ordered by `name` or `updated` fields.' + optional :order_by, type: String, values: %w[name updated version], default: 'updated', + desc: 'Return tags ordered by `name`, `updated`, `version` fields.' optional :search, type: String, desc: 'Return list of tags matching the search criteria' optional :page_token, type: String, desc: 'Name of tag to start the paginaition from' use :pagination diff --git a/lib/api/topics.rb b/lib/api/topics.rb index a08b4c6c107..38cfdc44021 100644 --- a/lib/api/topics.rb +++ b/lib/api/topics.rb @@ -94,5 +94,25 @@ module API destroy_conditionally!(topic) end + + desc 'Merge topics' do + detail 'This feature was introduced in GitLab 15.4.' + success Entities::Projects::Topic + end + params do + requires :source_topic_id, type: Integer, desc: 'ID of source project topic' + requires :target_topic_id, type: Integer, desc: 'ID of target project topic' + end + post 'topics/merge' do + authenticated_as_admin! + + source_topic = ::Projects::Topic.find(params[:source_topic_id]) + target_topic = ::Projects::Topic.find(params[:target_topic_id]) + + response = ::Topics::MergeService.new(source_topic, target_topic).execute + render_api_error!(response.message, :bad_request) if response.error? + + present target_topic, with: Entities::Projects::Topic + end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index c93c0f601a0..1d1c633824e 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -54,8 +54,7 @@ module API optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' - # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - optional :avatar, type: File, desc: 'Avatar image for user' # rubocop:disable Scalability/FileUploads + optional :avatar, type: ::API::Validations::Types::WorkhorseFile, desc: 'Avatar image for user' optional :theme_id, type: Integer, desc: 'The GitLab theme for the user' optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' @@ -733,7 +732,7 @@ module API unless user.can_be_deactivated? forbidden!('A blocked user cannot be deactivated by the API') if user.blocked? forbidden!('An internal user cannot be deactivated by the API') if user.internal? - forbidden!("The user you are trying to deactivate has been active in the past #{::User::MINIMUM_INACTIVE_DAYS} days and cannot be deactivated") + forbidden!("The user you are trying to deactivate has been active in the past #{Gitlab::CurrentSettings.deactivate_dormant_users_period} days and cannot be deactivated") end if user.deactivate diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index e07cbfe8d85..e5cf20d00df 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -6,13 +6,13 @@ module Banzai REGEX = %r{ #{::Gitlab::Regex.markdown_code_or_html_blocks} | - (?=^>>>\ *\n.*\n>>>\ *$)(?: + (?=(?<=^\n|\A)>>>\ *\n.*\n>>>\ *(?=\n$|\z))(?: # Blockquote: # >>> # Anything, including code and HTML blocks # >>> - ^>>>\ *\n + (?<=^\n|\A)>>>\ *\n (?<quote> (?: # Any character that doesn't introduce a code or HTML block @@ -30,7 +30,7 @@ module Banzai \g<html> )+? ) - \n>>>\ *$ + \n>>>\ *(?=\n$|\z) ) }mx.freeze diff --git a/lib/banzai/filter/kroki_filter.rb b/lib/banzai/filter/kroki_filter.rb index 845c7f2bc0a..713ff2439fc 100644 --- a/lib/banzai/filter/kroki_filter.rb +++ b/lib/banzai/filter/kroki_filter.rb @@ -14,7 +14,10 @@ module Banzai return doc unless settings.kroki_enabled diagram_selectors = ::Gitlab::Kroki.formats(settings) - .map { |diagram_type| %(pre[lang="#{diagram_type}"] > code) } + .map do |diagram_type| + %(pre[lang="#{diagram_type}"] > code, + pre > code[lang="#{diagram_type}"]) + end .join(', ') xpath = Gitlab::Utils::Nokogiri.css_to_xpath(diagram_selectors) @@ -22,7 +25,7 @@ module Banzai diagram_format = "svg" doc.xpath(xpath).each do |node| - diagram_type = node.parent['lang'] + diagram_type = node.parent['lang'] || node['lang'] diagram_src = node.content image_src = create_image_src(diagram_type, diagram_format, diagram_src) img_tag = Nokogiri::HTML::DocumentFragment.parse(%(<img src="#{image_src}" />)) @@ -33,8 +36,8 @@ module Banzai img_tag.set_attribute('hidden', '') if lazy_load img_tag.set_attribute('class', 'js-render-kroki') - img_tag.set_attribute('data-diagram', node.parent['lang']) - img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(node.content)}") + img_tag.set_attribute('data-diagram', diagram_type) + img_tag.set_attribute('data-diagram-src', "data:text/plain;base64,#{Base64.strict_encode64(diagram_src)}") node.parent.replace(img_tag) end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index 0ac506776be..1ca4b2c89db 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -7,7 +7,7 @@ require 'uri' # - app/assets/javascripts/behaviors/markdown/nodes/code_block.js module Banzai module Filter - # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # HTML filter that implements our math syntax, adding class="code math" # class MathFilter < HTML::Pipeline::Filter CSS_MATH = 'pre.code.language-math' @@ -15,14 +15,42 @@ module Banzai CSS_CODE = 'code' XPATH_CODE = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_CODE).freeze + # These are based on the Pandoc heuristics, + # https://pandoc.org/MANUAL.html#extension-tex_math_dollars + # Note: at this time, using a dollar sign literal, `\$` inside + # a math statement does not work correctly. + # Corresponds to the "$...$" syntax + DOLLAR_INLINE_PATTERN = %r{ + (?<matched>\$(?<math>(?:\S[^$\n]*?\S|[^$\s]))\$)(?:[^\d]|$) + }x.freeze + + # Corresponds to the "$$...$$" syntax + DOLLAR_DISPLAY_INLINE_PATTERN = %r{ + (?<matched>\$\$\ *(?<math>[^$\n]+?)\ *\$\$) + }x.freeze + + # Corresponds to the $$\n...\n$$ syntax + DOLLAR_DISPLAY_BLOCK_PATTERN = %r{ + ^(?<matched>\$\$\ *\n(?<math>.*)\n\$\$\ *)$ + }x.freeze + + # Order dependent. Handle the `$$` syntax before the `$` syntax + DOLLAR_MATH_PIPELINE = [ + { pattern: DOLLAR_DISPLAY_INLINE_PATTERN, tag: :code, style: :display }, + { pattern: DOLLAR_DISPLAY_BLOCK_PATTERN, tag: :pre, style: :display }, + { pattern: DOLLAR_INLINE_PATTERN, tag: :code, style: :inline } + ].freeze + + # Do not recognize math inside these tags + IGNORED_ANCESTOR_TAGS = %w[pre code tt].to_set + # Attribute indicating inline or display math. STYLE_ATTRIBUTE = 'data-math-style' # Class used for tagging elements that should be rendered TAG_CLASS = 'js-render-math' - INLINE_CLASSES = "code math #{TAG_CLASS}" - + MATH_CLASSES = "code math #{TAG_CLASS}" DOLLAR_SIGN = '$' # Limit to how many nodes can be marked as math elements. @@ -31,8 +59,48 @@ module Banzai RENDER_NODES_LIMIT = 50 def call - nodes_count = 0 + @nodes_count = 0 + + process_dollar_pipeline if Feature.enabled?(:markdown_dollar_math, group) + + process_dollar_backtick_inline + process_math_codeblock + + doc + end + + def process_dollar_pipeline + doc.xpath('descendant-or-self::text()').each do |node| + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + + node_html = node.to_html + next unless node_html.match?(DOLLAR_INLINE_PATTERN) || + node_html.match?(DOLLAR_DISPLAY_INLINE_PATTERN) || + node_html.match?(DOLLAR_DISPLAY_BLOCK_PATTERN) + + temp_doc = Nokogiri::HTML.fragment(node_html) + DOLLAR_MATH_PIPELINE.each do |pipeline| + temp_doc.xpath('child::text()').each do |temp_node| + html = temp_node.to_html + temp_node.content.scan(pipeline[:pattern]).each do |matched, math| + html.sub!(matched, math_html(tag: pipeline[:tag], style: pipeline[:style], math: math)) + @nodes_count += 1 + break if @nodes_count >= RENDER_NODES_LIMIT + end + + temp_node.replace(html) + + break if @nodes_count >= RENDER_NODES_LIMIT + end + end + + node.replace(temp_doc) + end + end + + # Corresponds to the "$`...`$" syntax + def process_dollar_backtick_inline doc.xpath(XPATH_CODE).each do |code| closing = code.next opening = code.previous @@ -44,22 +112,38 @@ module Banzai closing.content.first == DOLLAR_SIGN && opening.content.last == DOLLAR_SIGN - code[:class] = INLINE_CLASSES + code[:class] = MATH_CLASSES code[STYLE_ATTRIBUTE] = 'inline' closing.content = closing.content[1..] opening.content = opening.content[0..-2] - nodes_count += 1 - break if nodes_count >= RENDER_NODES_LIMIT + @nodes_count += 1 + break if @nodes_count >= RENDER_NODES_LIMIT end end + end + # corresponds to the "```math...```" syntax + def process_math_codeblock doc.xpath(XPATH_MATH).each do |el| el[STYLE_ATTRIBUTE] = 'display' el[:class] += " #{TAG_CLASS}" end + end - doc + private + + def math_html(tag:, math:, style:) + case tag + when :code + "<code class=\"#{MATH_CLASSES}\" data-math-style=\"#{style}\">#{math}</code>" + when :pre + "<pre class=\"#{MATH_CLASSES}\" data-math-style=\"#{style}\"><code>#{math}</code></pre>" + end + end + + def group + context[:group] || context[:project]&.group end end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index cbcd547120d..82f6247cf03 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -31,7 +31,8 @@ module Banzai private def lang_tag - @lang_tag ||= Gitlab::Utils::Nokogiri.css_to_xpath('pre[lang="plantuml"] > code').freeze + @lang_tag ||= Gitlab::Utils::Nokogiri + .css_to_xpath('pre[lang="plantuml"] > code, pre > code[lang="plantuml"]').freeze end def settings diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index 97ef71036a2..37734f6a45a 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -15,6 +15,8 @@ module Banzai include RequestStoreReferenceCache include OutputSafety + REFERENCE_TYPE_DATA_ATTRIBUTE = 'data-reference-type=' + class << self # Implement in child class # Example: self.reference_type = :merge_request @@ -132,13 +134,19 @@ module Banzai def data_attribute(attributes = {}) attributes = attributes.reject { |_, v| v.nil? } - attributes[:reference_type] ||= self.class.reference_type + # "data-reference-type=" attribute got moved into a constant because we need + # to use it on ReferenceRewriter class to detect if the markdown contains any reference + reference_type_attribute = "#{REFERENCE_TYPE_DATA_ATTRIBUTE}#{escape_once(self.class.reference_type)} " + attributes[:container] ||= 'body' attributes[:placement] ||= 'top' attributes.delete(:original) if context[:no_original_data] + attributes.map do |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") - end.join(' ') + end + .join(' ') + .prepend(reference_type_attribute) end def ignore_ancestor_query diff --git a/lib/banzai/pipeline/markup_pipeline.rb b/lib/banzai/pipeline/markup_pipeline.rb index 17a73f29afb..330914f7238 100644 --- a/lib/banzai/pipeline/markup_pipeline.rb +++ b/lib/banzai/pipeline/markup_pipeline.rb @@ -9,8 +9,8 @@ module Banzai Filter::AssetProxyFilter, Filter::ExternalLinkFilter, Filter::PlantumlFilter, - Filter::SyntaxHighlightFilter, - Filter::KrokiFilter + Filter::KrokiFilter, + Filter::SyntaxHighlightFilter ] end diff --git a/lib/bulk_imports/file_downloads/filename_fetch.rb b/lib/bulk_imports/file_downloads/filename_fetch.rb new file mode 100644 index 00000000000..b6bb0fd8c81 --- /dev/null +++ b/lib/bulk_imports/file_downloads/filename_fetch.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module BulkImports + module FileDownloads + module FilenameFetch + REMOTE_FILENAME_PATTERN = %r{filename="(?<filename>[^"]+)"}.freeze + FILENAME_SIZE_LIMIT = 255 # chars before the extension + + def raise_error(message) + raise NotImplementedError + end + + private + + # Fetch the remote filename information from the request content-disposition header + # - Raises if the filename does not exist + # - If the filename is longer then 255 chars truncate it + # to be a total of 255 chars (with the extension) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def remote_filename + @remote_filename ||= begin + pattern = BulkImports::FileDownloads::FilenameFetch::REMOTE_FILENAME_PATTERN + name = response_headers['content-disposition'].to_s + .match(pattern) # matches the filename pattern + .then { |match| match&.named_captures || {} } # ensures the match is a hash + .fetch('filename') # fetches the 'filename' key or raise KeyError + + name = File.basename(name) # Ensures to remove path from the filename (../ for instance) + ensure_filename_size(name) # Ensures the filename is within the FILENAME_SIZE_LIMIT + end + rescue KeyError + raise_error 'Remote filename not provided in content-disposition header' + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + + def ensure_filename_size(filename) + limit = BulkImports::FileDownloads::FilenameFetch::FILENAME_SIZE_LIMIT + return filename if filename.length <= limit + + extname = File.extname(filename) + basename = File.basename(filename, extname)[0, limit] + "#{basename}#{extname}" + end + end + end +end diff --git a/lib/bulk_imports/file_downloads/validations.rb b/lib/bulk_imports/file_downloads/validations.rb new file mode 100644 index 00000000000..ae94267a6e8 --- /dev/null +++ b/lib/bulk_imports/file_downloads/validations.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module BulkImports + module FileDownloads + module Validations + def raise_error(message) + raise NotImplementedError + end + + def filepath + raise NotImplementedError + end + + def file_size_limit + raise NotImplementedError + end + + def response_headers + raise NotImplementedError + end + + private + + def validate_filepath + Gitlab::Utils.check_path_traversal!(filepath) + end + + def validate_content_type + content_type = response_headers['content-type'] + + raise_error('Invalid content type') if content_type.blank? || allowed_content_types.exclude?(content_type) + end + + def validate_symlink + return unless File.lstat(filepath).symlink? + + File.delete(filepath) + raise_error 'Invalid downloaded file' + end + + def validate_content_length + validate_size!(response_headers['content-length']) + end + + def validate_size!(size) + if size.blank? + raise_error 'Missing content-length header' + elsif size.to_i > file_size_limit + raise_error format( + "File size %{size} exceeds limit of %{limit}", + size: ActiveSupport::NumberHelper.number_to_human_size(size), + limit: ActiveSupport::NumberHelper.number_to_human_size(file_size_limit) + ) + end + end + end + end +end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index be99fa75ffe..2947dcb4b40 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -31,7 +31,7 @@ module ContainerRegistry def self.deduplicated_size(path) with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client| - client.repository_details(path, sizing: :self_with_descendants)['size_bytes'] + client.repository_details(path&.downcase, sizing: :self_with_descendants)['size_bytes'] end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 76188a937c0..bf44b74cf7b 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -4,7 +4,7 @@ module ContainerRegistry class Tag include Gitlab::Utils::StrongMemoize - attr_reader :repository, :name + attr_reader :repository, :name, :updated_at attr_writer :created_at delegate :registry, :client, to: :repository @@ -97,6 +97,17 @@ module ContainerRegistry instance_variable_set(ivar(:memoized_created_at), date) end + def updated_at=(string_value) + return unless string_value + + @updated_at = + begin + DateTime.iso8601(string_value) + rescue ArgumentError + nil + end + end + def layers return unless manifest diff --git a/lib/error_tracking/sentry_client.rb b/lib/error_tracking/sentry_client.rb index 6a341ddbe86..029389ab5d6 100644 --- a/lib/error_tracking/sentry_client.rb +++ b/lib/error_tracking/sentry_client.rb @@ -10,16 +10,32 @@ module ErrorTracking Error = Class.new(StandardError) MissingKeysError = Class.new(StandardError) + ResponseInvalidSizeError = Class.new(StandardError) + + RESPONSE_SIZE_LIMIT = 1.megabyte attr_accessor :url, :token - def initialize(api_url, token) + def initialize(api_url, token, validate_size_guarded_by_feature_flag: false) @url = api_url @token = token + @validate_size_guarded_by_feature_flag = validate_size_guarded_by_feature_flag + end + + def validate_size_guarded_by_feature_flag? + @validate_size_guarded_by_feature_flag end private + def validate_size(response) + return if Gitlab::Utils::DeepSize.new(response, max_size: RESPONSE_SIZE_LIMIT).valid? + + limit = ActiveSupport::NumberHelper.number_to_human_size(RESPONSE_SIZE_LIMIT) + message = "Sentry API response is too big. Limit is #{limit}." + raise ResponseInvalidSizeError, message + end + def api_urls @api_urls ||= SentryClient::ApiUrls.new(@url) end @@ -86,6 +102,8 @@ module ErrorTracking def handle_response(response) raise_error "Sentry response status code: #{response.code}" unless response.code.between?(200, 204) + validate_size(response.parsed_response) if validate_size_guarded_by_feature_flag? + { body: response.parsed_response, headers: response.headers } end diff --git a/lib/error_tracking/sentry_client/issue.rb b/lib/error_tracking/sentry_client/issue.rb index d0e6bd783f3..3c846eb0635 100644 --- a/lib/error_tracking/sentry_client/issue.rb +++ b/lib/error_tracking/sentry_client/issue.rb @@ -4,7 +4,6 @@ module ErrorTracking class SentryClient module Issue BadRequestError = Class.new(StandardError) - ResponseInvalidSizeError = Class.new(StandardError) SENTRY_API_SORT_VALUE_MAP = { # <accepted_by_client> => <accepted_by_sentry_api> @@ -19,7 +18,9 @@ module ErrorTracking issues = response[:issues] pagination = response[:pagination] - validate_size(issues) + # We check validate size only with feture flag disabled because when + # enabled we already check it when parsing the response. + validate_size(issues) unless validate_size_guarded_by_feature_flag? handle_mapping_exceptions do { @@ -64,13 +65,6 @@ module ErrorTracking }.compact end - def validate_size(issues) - return if Gitlab::Utils::DeepSize.new(issues).valid? - - message = "Sentry API response is too big. Limit is #{Gitlab::Utils::DeepSize.human_default_max_size}." - raise ResponseInvalidSizeError, message - end - def get_issue(issue_id:) http_get(api_urls.issue_url(issue_id))[:body] end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 8c3377fdb80..f14b0a6b9e7 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -131,18 +131,19 @@ class EventFilter finder_query = -> (id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } if order_hint_column.present? - order = Gitlab::Pagination::Keyset::Order.build([ - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: order_hint_column, - order_expression: Event.arel_table[order_hint_column].desc, - nullable: :nulls_last, - distinct: false - ), - Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: :id, - order_expression: Event.arel_table[:id].desc - ) - ]) + order = Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: order_hint_column, + order_expression: Event.arel_table[order_hint_column].desc, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :id, + order_expression: Event.arel_table[:id].desc + ) + ]) finder_query = -> (_order_hint, id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } end diff --git a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template index e984daee0a4..f8bd502ab77 100644 --- a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template +++ b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template @@ -3,5 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::Usage::Metrics::Instrumentations::<%= class_name %>Metric do - it_behaves_like 'a correct instrumented metric value', {}, 1 + let(:expected_value) { 1 } + + it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } end diff --git a/lib/gitlab/abuse.rb b/lib/gitlab/abuse.rb index cc95d3c1e0c..7db99d4b037 100644 --- a/lib/gitlab/abuse.rb +++ b/lib/gitlab/abuse.rb @@ -3,10 +3,10 @@ module Gitlab module Abuse CONFIDENCE_LEVELS = { - certain: 1.0, - likely: 0.8, + certain: 1.0, + likely: 0.8, uncertain: 0.5, - unknown: 0.0 + unknown: 0.0 }.freeze class << self diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 3e09d488bc3..fa025a2658f 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -41,9 +41,9 @@ module Gitlab def options { - "Guest" => GUEST, - "Reporter" => REPORTER, - "Developer" => DEVELOPER, + "Guest" => GUEST, + "Reporter" => REPORTER, + "Developer" => DEVELOPER, "Maintainer" => MAINTAINER } end @@ -62,9 +62,9 @@ module Gitlab def sym_options { - guest: GUEST, - reporter: REPORTER, - developer: DEVELOPER, + guest: GUEST, + reporter: REPORTER, + developer: DEVELOPER, maintainer: MAINTAINER } end @@ -120,9 +120,9 @@ module Gitlab def project_creation_string_options { - 'noone' => NO_ONE_PROJECT_ACCESS, - 'maintainer' => MAINTAINER_PROJECT_ACCESS, - 'developer' => DEVELOPER_MAINTAINER_PROJECT_ACCESS + 'noone' => NO_ONE_PROJECT_ACCESS, + 'maintainer' => MAINTAINER_PROJECT_ACCESS, + 'developer' => DEVELOPER_MAINTAINER_PROJECT_ACCESS } end @@ -147,7 +147,7 @@ module Gitlab def subgroup_creation_string_options { - 'owner' => OWNER_SUBGROUP_ACCESS, + 'owner' => OWNER_SUBGROUP_ACCESS, 'maintainer' => MAINTAINER_SUBGROUP_ACCESS } end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 2d769148c5f..01dcb95eab5 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -149,6 +149,10 @@ module Gitlab severity_mapping.fetch(severity_raw.to_s.downcase, UNMAPPED_SEVERITY) end + def source + monitoring_tool || integration&.name + end + private def plain_gitlab_fingerprint diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 15238b5e50f..18e65779ead 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -6,6 +6,7 @@ module Gitlab module Payload class Generic < Base DEFAULT_TITLE = 'New: Alert' + DEFAULT_SOURCE = 'Generic Alert Endpoint' attribute :description, paths: 'description' attribute :ends_at, paths: 'end_time', type: :time @@ -22,6 +23,14 @@ module Gitlab attribute :plain_gitlab_fingerprint, paths: 'fingerprint' private :plain_gitlab_fingerprint + + def resolved? + ends_at.present? + end + + def source + super || DEFAULT_SOURCE + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index d0d8d68362e..ac9c465bf7d 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -105,9 +105,8 @@ module Gitlab private def use_aggregated_backend? - group.present? && # for now it's only available on the group-level - aggregation.enabled && - Feature.enabled?(:use_vsa_aggregated_tables, group) + # for now it's only available on the group-level + group.present? && aggregation.enabled end def aggregation_attributes diff --git a/lib/gitlab/analytics/date_filler.rb b/lib/gitlab/analytics/date_filler.rb new file mode 100644 index 00000000000..aa3db9f3635 --- /dev/null +++ b/lib/gitlab/analytics/date_filler.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + # This class generates a date => value hash without gaps in the data points. + # + # Simple usage: + # + # > # We have the following data for the last 5 day: + # > input = { 3.days.ago.to_date => 10, Date.today => 5 } + # + # > # Format this data, so we can chart the complete date range: + # > Gitlab::Analytics::DateFiller.new(input, from: 4.days.ago, to: Date.today, default_value: 0).fill + # > { + # > Sun, 28 Aug 2022=>0, + # > Mon, 29 Aug 2022=>10, + # > Tue, 30 Aug 2022=>0, + # > Wed, 31 Aug 2022=>0, + # > Thu, 01 Sep 2022=>5 + # > } + # + # Parameters: + # + # **input** + # A Hash containing data for the series or the chart. The key is a Date object + # or an object which can be converted to Date. + # + # **from** + # Start date of the range + # + # **to** + # End date of the range + # + # **period** + # Specifies the period in wich the dates should be generated. Options: + # + # - :day, generate date-value pair for each day in the given period + # - :week, generate date-value pair for each week (beginning of the week date) + # - :month, generate date-value pair for each week (beginning of the month date) + # + # Note: the Date objects in the `input` should follow the same pattern (beginning of ...) + # + # **default_value** + # + # Which value use when the `input` Hash does not contain data for the given day. + # + # **date_formatter** + # + # How to format the dates in the resulting hash. + class DateFiller + DEFAULT_DATE_FORMATTER = -> (date) { date } + PERIOD_STEPS = { + day: 1.day, + week: 1.week, + month: 1.month + }.freeze + + def initialize( + input, + from:, + to:, + period: :day, + default_value: nil, + date_formatter: DEFAULT_DATE_FORMATTER) + @input = input.transform_keys(&:to_date) + @from = from.to_date + @to = to.to_date + @period = period + @default_value = default_value + @date_formatter = date_formatter + end + + def fill + data = {} + + current_date = from + loop do + transformed_date = transform_date(current_date) + break if transformed_date > to + + formatted_date = date_formatter.call(transformed_date) + + value = input.delete(transformed_date) + data[formatted_date] = value.nil? ? default_value : value + + current_date = (current_date + PERIOD_STEPS.fetch(period)).to_date + end + + raise "Input contains values which doesn't fall under the given period!" if input.any? + + data + end + + private + + attr_reader :input, :from, :to, :period, :default_value, :date_formatter + + def transform_date(date) + case period + when :day + date.beginning_of_day.to_date + when :week + date.beginning_of_week.to_date + when :month + date.beginning_of_month.to_date + else + raise "Unknown period given: #{period}" + end + end + end + end +end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index a2d79b189a3..507f94d87a5 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -16,40 +16,43 @@ module Gitlab # and only do that when it's needed. def rate_limits # rubocop:disable Metrics/AbcSize { - issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute }, - notes_create: { threshold: -> { application_settings.notes_create_limit }, interval: 1.minute }, - project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, - project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute }, + issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute }, + notes_create: { threshold: -> { application_settings.notes_create_limit }, interval: 1.minute }, + project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, + project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute }, project_repositories_archive: { threshold: 5, interval: 1.minute }, - project_generate_new_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, - project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, - project_testing_hook: { threshold: 5, interval: 1.minute }, - play_pipeline_schedule: { threshold: 1, interval: 1.minute }, - raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, - group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, - group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, - group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, - group_testing_hook: { threshold: 5, interval: 1.minute }, - profile_add_new_email: { threshold: 5, interval: 1.minute }, - web_hook_calls: { interval: 1.minute }, - web_hook_calls_mid: { interval: 1.minute }, - web_hook_calls_low: { interval: 1.minute }, - users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, - username_exists: { threshold: 20, interval: 1.minute }, - user_sign_up: { threshold: 20, interval: 1.minute }, - user_sign_in: { threshold: 5, interval: 10.minutes }, - profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, - profile_update_username: { threshold: 10, interval: 1.minute }, - update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, - auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, - search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, - search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, - gitlab_shell_operation: { threshold: 600, interval: 1.minute }, - pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, - temporary_email_failure: { threshold: 50, interval: 1.day }, - project_testing_integration: { threshold: 5, interval: 1.minute }, - email_verification: { threshold: 10, interval: 10.minutes }, - email_verification_code_send: { threshold: 10, interval: 1.hour } + project_generate_new_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, + project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, + project_testing_hook: { threshold: 5, interval: 1.minute }, + play_pipeline_schedule: { threshold: 1, interval: 1.minute }, + raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, + group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, + group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, + group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, + group_testing_hook: { threshold: 5, interval: 1.minute }, + profile_add_new_email: { threshold: 5, interval: 1.minute }, + web_hook_calls: { interval: 1.minute }, + web_hook_calls_mid: { interval: 1.minute }, + web_hook_calls_low: { interval: 1.minute }, + users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, + username_exists: { threshold: 20, interval: 1.minute }, + user_sign_up: { threshold: 20, interval: 1.minute }, + user_sign_in: { threshold: 5, interval: 10.minutes }, + profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, + profile_update_username: { threshold: 10, interval: 1.minute }, + update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, + auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, + search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, + search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, + gitlab_shell_operation: { threshold: 600, interval: 1.minute }, + pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, + temporary_email_failure: { threshold: 300, interval: 1.day }, + permanent_email_failure: { threshold: 5, interval: 1.day }, + project_testing_integration: { threshold: 5, interval: 1.minute }, + email_verification: { threshold: 10, interval: 10.minutes }, + email_verification_code_send: { threshold: 10, interval: 1.hour }, + namespace_exists: { threshold: 20, interval: 1.minute }, + fetch_google_ip_list: { threshold: 10, interval: 1.minute } }.freeze end @@ -130,16 +133,16 @@ module Gitlab # @param logger [Logger] Logger to log request to a specific log file. Defaults to Gitlab::AuthLogger def log_request(request, type, current_user, logger = Gitlab::AuthLogger) request_information = { - message: 'Application_Rate_Limiter_Request', - env: type, - remote_ip: request.ip, + message: 'Application_Rate_Limiter_Request', + env: type, + remote_ip: request.ip, request_method: request.request_method, - path: request.fullpath + path: request.fullpath } if current_user request_information.merge!({ - user_id: current_user.id, + user_id: current_user.id, username: current_user.username }) end diff --git a/lib/gitlab/application_rate_limiter/increment_per_action.rb b/lib/gitlab/application_rate_limiter/increment_per_action.rb index c99d03f1344..a3343c8a97c 100644 --- a/lib/gitlab/application_rate_limiter/increment_per_action.rb +++ b/lib/gitlab/application_rate_limiter/increment_per_action.rb @@ -5,9 +5,9 @@ module Gitlab class IncrementPerAction < BaseStrategy def increment(cache_key, expiry) with_redis do |redis| - redis.pipelined do - redis.incr(cache_key) - redis.expire(cache_key, expiry) + redis.pipelined do |pipeline| + pipeline.incr(cache_key) + pipeline.expire(cache_key, expiry) end.first end end diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb index 8b4197cfff9..7a68dd104a8 100644 --- a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb +++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb @@ -9,10 +9,10 @@ module Gitlab def increment(cache_key, expiry) with_redis do |redis| - redis.pipelined do - redis.sadd(cache_key, resource_key) - redis.expire(cache_key, expiry) - redis.scard(cache_key) + redis.pipelined do |pipeline| + pipeline.sadd(cache_key, resource_key) + pipeline.expire(cache_key, expiry) + pipeline.scard(cache_key) end.last end end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index c96be19f02d..4a6e4e2e06e 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -117,7 +117,7 @@ module Gitlab # Only capture real users for successful authentication events. user: author_if_user, user_name: @author.name, - ip_address: @ip_address, + ip_address: Gitlab::RequestContext.instance.client_ip || @author.current_sign_in_ip, result: AuthenticationEvent.results[:success], provider: @authentication_provider } diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb new file mode 100644 index 00000000000..af5dc9f4b44 --- /dev/null +++ b/lib/gitlab/audit/type/definition.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + module Type + class Definition + include ActiveModel::Validations + + attr_reader :path + attr_reader :attributes + + validate :validate_schema + validate :validate_file_name + + InvalidAuditEventTypeError = Class.new(StandardError) + + AUDIT_EVENT_TYPE_SCHEMA_PATH = Rails.root.join('config', 'audit_events', 'types', 'type_schema.json') + AUDIT_EVENT_TYPE_SCHEMA = JSONSchemer.schema(AUDIT_EVENT_TYPE_SCHEMA_PATH) + + # The PARAMS in config/audit_events/types/type_schema.json + PARAMS = %i[ + name + description + introduced_by_issue + introduced_by_mr + group + milestone + saved_to_database + streamed + ].freeze + + PARAMS.each do |param| + define_method(param) do + attributes[param] + end + end + + def initialize(path, opts = {}) + @path = path + @attributes = {} + + # assign nil, for all unknown opts + PARAMS.each do |param| + @attributes[param] = opts[param] + end + end + + def key + name.to_sym + end + + private + + def validate_schema + schema_errors = AUDIT_EVENT_TYPE_SCHEMA + .validate(attributes.to_h.deep_stringify_keys) + .map { |error| JSONSchemer::Errors.pretty(error) } + + errors.add(:base, schema_errors) if schema_errors.present? + end + + def validate_file_name + # ignoring Style/GuardClause because if we move this into one line, we cause Layout/LineLength errors + # rubocop:disable Style/GuardClause + unless File.basename(path, ".yml") == name + errors.add(:base, "Audit event type '#{name}' has an invalid path: '#{path}'. " \ + "'#{name}' must match the filename") + end + # rubocop:enable Style/GuardClause + end + + class << self + def paths + @paths ||= [Rails.root.join('config', 'audit_events', 'types', '*.yml')] + end + + def definitions + # We lazily load all definitions + @definitions ||= load_all! + end + + def get(key) + definitions[key.to_sym] + end + + private + + def load_all! + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + + def load_all_from_path!(definitions, glob_path) + Dir.glob(glob_path).each do |path| + definition = load_from_file(path) + + if previous = definitions[definition.key] + raise InvalidAuditEventTypeError, "Audit event type '#{definition.key}' " \ + "is already defined in '#{previous.path}'" + end + + definitions[definition.key] = definition + end + end + + def load_from_file(path) + definition = File.read(path) + definition = YAML.safe_load(definition) + definition.deep_symbolize_keys! + + new(path, definition).tap(&:validate!) + rescue StandardError => e + raise InvalidAuditEventTypeError, "Invalid definition for `#{path}`: #{e.message}" + end + end + end + end + end +end + +Gitlab::Audit::Type::Definition.prepend_mod diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 82c6411c712..9dafd59561a 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -7,8 +7,8 @@ module Gitlab class Config NET_LDAP_ENCRYPTION_METHOD = { simple_tls: :simple_tls, - start_tls: :start_tls, - plain: nil + start_tls: :start_tls, + plain: nil }.freeze attr_accessor :provider, :options @@ -193,11 +193,11 @@ module Gitlab def default_attributes { - 'username' => %W(#{uid} uid sAMAccountName userid).uniq, - 'email' => %w(mail email userPrincipalName), - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' + 'username' => %W(#{uid} uid sAMAccountName userid).uniq, + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' } end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 37f92792d2d..82a5aad360c 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -33,7 +33,7 @@ module Gitlab end def password - @password ||= Gitlab::Utils.force_utf8(::User.random_password.downcase) + @password ||= Gitlab::Utils.force_utf8(::User.random_password) end def location @@ -103,7 +103,7 @@ module Gitlab { username: username, - email: email + email: email } end end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 1a25ed10d81..2ce8677c8b7 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,14 +5,14 @@ module Gitlab module OAuth class Provider LABELS = { - "alicloud" => "AliCloud", - "dingtalk" => "DingTalk", - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google", - "azure_oauth2" => "Azure AD", + "alicloud" => "AliCloud", + "dingtalk" => "DingTalk", + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google", + "azure_oauth2" => "Azure AD", "azure_activedirectory_v2" => "Azure AD v2", - 'atlassian_oauth2' => 'Atlassian' + 'atlassian_oauth2' => 'Atlassian' }.freeze def self.authentication(user, provider) @@ -68,7 +68,9 @@ module Gitlab nil end else - provider = Gitlab.config.omniauth.providers.find { |provider| provider.name == name } + provider = Gitlab.config.omniauth.providers.find do |provider| + provider.name == name || (provider.name == 'openid_connect' && provider.args.name == name) + end merge_provider_args_with_defaults!(provider) provider diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 7d9c4c0d7c1..1fed2b263da 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -240,11 +240,11 @@ module Gitlab valid_username = Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } { - name: name.strip.presence || valid_username, - username: valid_username, - email: email, - password: auth_hash.password, - password_confirmation: auth_hash.password, + name: name.strip.presence || valid_username, + username: valid_username, + email: email, + password: auth_hash.password, + password_confirmation: auth_hash.password, password_automatically_set: true } end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb index 9cf1b2247a7..88ad48c3db7 100644 --- a/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb @@ -34,7 +34,7 @@ module Gitlab end def body - { username: user.username, + { username: user.username, token_code: @otp_code } end diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index ff6dc7313bb..322dfa74d09 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -57,3 +57,5 @@ module Gitlab end end end + +Gitlab::Auth::UserAccessDeniedReason.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb new file mode 100644 index 00000000000..2ee0594d0a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `vulnerability_reads.casted_cluster_agent_id` column + class BackfillClusterAgentsHasVulnerabilities < Gitlab::BackgroundMigration::BatchedMigrationJob + VULNERABILITY_READS_JOIN = <<~SQL + INNER JOIN vulnerability_reads + ON vulnerability_reads.casted_cluster_agent_id = cluster_agents.id AND + vulnerability_reads.project_id = cluster_agents.project_id AND + vulnerability_reads.report_type = 7 + SQL + + RELATION = ->(relation) do + relation + .where(has_vulnerabilities: false) + end + + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: RELATION + ) do |sub_batch| + sub_batch + .joins(VULNERABILITY_READS_JOIN) + .update_all(has_vulnerabilities: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb index b9151343d6a..2d64b7378be 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb @@ -9,6 +9,7 @@ module Gitlab # Migration only version of MergeRequest table class MergeRequest < ::ApplicationRecord include EachBatch + validates :suggested_reviewers, json_schema: { filename: 'merge_request_suggested_reviewers' } CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index 814f5a897a9..ce4c4a28b37 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -22,7 +22,7 @@ module Gitlab ProjectFeature.connection.execute( <<~SQL UPDATE project_features pf - SET package_registry_access_level = (CASE p.packages_enabled + SET package_registry_access_level = (CASE p.packages_enabled WHEN true THEN (CASE p.visibility_level WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC} WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED} diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb new file mode 100644 index 00000000000..815c346bb39 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fills the `issues.namespace_id` by setting it to corresponding project.project_namespace_id + class BackfillProjectNamespaceOnIssues < BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { + relation.joins("INNER JOIN projects ON projects.id = issues.project_id") + .select("issues.id AS issue_id, projects.project_namespace_id").where(issues: { namespace_id: nil }) + } + ) do |sub_batch| + connection.execute <<~SQL + UPDATE issues + SET namespace_id = projects.project_namespace_id + FROM (#{sub_batch.to_sql}) AS projects(issue_id, project_namespace_id) + WHERE issues.id = issue_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index 05e2ed72fb3..c49ef9d10f5 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -212,8 +212,8 @@ module Gitlab def build_attributes_for_project(project) { project_id: project.id, - shard_id: find_shard_id(project.repository_storage), - disk_path: project.disk_path + shard_id: find_shard_id(project.repository_storage), + disk_path: project.disk_path } end diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 728b60f7a0e..0c41d6af209 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -10,16 +10,12 @@ module Gitlab vulnerability_reads.project_id = cluster_agents.project_id SQL - RELATION = ->(relation) do - relation - .where(report_type: 7) - end + CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 + + scope_to ->(relation) { relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) } def perform - each_sub_batch( - operation_name: :update_all, - batching_scope: RELATION - ) do |sub_batch| + each_sub_batch(operation_name: :update_all) do |sub_batch| sub_batch .joins(CLUSTER_AGENTS_JOIN) .update_all('casted_cluster_agent_id = CAST(vulnerability_reads.cluster_agent_id AS bigint)') diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index 32962f2bb89..86d53ad798d 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -4,11 +4,9 @@ module Gitlab module BackgroundMigration # Backfills the `issues.work_item_type_id` column, replacing any # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` - class BackfillWorkItemTypeIdForIssues + class BackfillWorkItemTypeIdForIssues < BatchedMigrationJob # Basic AR model for issues table class MigrationIssue < ApplicationRecord - include ::EachBatch - self.table_name = 'issues' scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } @@ -16,29 +14,27 @@ module Gitlab MAX_UPDATE_RETRIES = 3 - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, base_type, base_type_id) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id, base_type) + scope_to ->(relation) { + relation.where(issue_type: base_type) + } + + job_arguments :base_type, :base_type_id - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } + ) do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) # The query need to be reconstructed because .each_batch modifies the default scope # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 reconstructed_sub_batch = MigrationIssue.unscoped.base_query(base_type).where(id: first..last) - batch_metrics.time_operation(:update_all) do - update_with_retry(reconstructed_sub_batch, base_type_id) - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) + update_with_retry(reconstructed_sub_batch, base_type_id) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private # Retry mechanism required as update statements on the issues table will randomly take longer than @@ -64,10 +60,6 @@ module Gitlab def update_batch(sub_batch, base_type_id) sub_batch.update_all(work_item_type_id: base_type_id) end - - def relation_scoped_to_range(source_table, source_key_column, start_id, end_id, base_type) - MigrationIssue.where(source_key_column => start_id..end_id).base_query(base_type) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb deleted file mode 100644 index 7d5fef67c25..00000000000 --- a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module BatchingStrategies - # Batching class to use for back-filling issue's work_item_type_id for a single issue type. - # Batches will be scoped to records where the foreign key is NULL and only of a given issue type - # - # If no more batches exist in the table, returns nil. - class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments:, job_class: nil) - issue_type = job_arguments.first - - relation.where(issue_type: issue_type) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb index 9ad119310f7..72da2b5a2b7 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb @@ -3,18 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for back-filling project_statistic's container_registry_size. - # Batches will be scoped to records where the project_ids are migrated - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy < PrimaryKeyBatchingStrategy - MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze - - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(created_at: MIGRATION_PHASE_1_ENDED_AT..).or( - relation.where(migration_state: 'import_done') - ).select(:project_id).distinct - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb index f0d015198dc..c2fa00f66de 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb @@ -3,16 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for back-filling vulnerability_read's casted_cluster_agent_id from cluster_agent_id. - # Batches will be scoped to records where the report_type belongs to cluster_image_scanning. - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class BackfillVulnerabilityReadsClusterAgentBatchingStrategy < PrimaryKeyBatchingStrategy - CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 - - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb index e1855b6cfee..9504d4eec11 100644 --- a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb @@ -3,14 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for setting state in vulnerabilitites table. - # Batches will be scoped to records where the dismissed_at is set. - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class DismissedVulnerabilitiesStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where.not(dismissed_at: nil) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 1ffa4a052e5..43352b1bf91 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -22,8 +22,8 @@ module Gitlab def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) model_class = define_batchable_model(table_name, connection: connection) - quoted_column_name = model_class.connection.quote_column_name(column_name) - relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + arel_column = model_class.arel_table[column_name] + relation = model_class.where(arel_column.gteq(batch_min_value)) if job_class relation = filter_batch(relation, @@ -32,11 +32,10 @@ module Gitlab ) end - relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop - next_batch_bounds = batch.pick(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")) + next_batch_bounds = batch.pick(arel_column.minimum, arel_column.maximum) break end @@ -44,15 +43,6 @@ module Gitlab next_batch_bounds end - # Deprecated - # - # Use `scope_to` to define additional filters on the migration job class. - # - # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#adding-additional-filters. - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation - end - private def filter_batch(relation, table_name:, column_name:, job_class:, job_arguments: []) diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb new file mode 100644 index 00000000000..49525479637 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478 + class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb new file mode 100644 index 00000000000..739197898d9 --- /dev/null +++ b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete approval rules + # as this feature exists only in EE + class DeleteApprovalRulesWithVulnerability < BatchedMigrationJob + def perform + end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability.prepend_mod_with('Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb new file mode 100644 index 00000000000..35ac42f76ab --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidGroupMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) do + relation.where(source_type: 'Namespace') + .joins('LEFT OUTER JOIN namespaces ON members.source_id = namespaces.id') + .where(namespaces: { id: nil }) + end + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + invalid_ids = sub_batch.map(&:id) + Gitlab::AppLogger.info({ message: 'Removing invalid group member records', + deleted_count: invalid_ids.size, ids: invalid_ids }) + + sub_batch.delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb new file mode 100644 index 00000000000..3c60f765c29 --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) { relation.where(source_type: 'Project') } + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + invalid_project_members = sub_batch + .joins('LEFT OUTER JOIN projects ON members.source_id = projects.id') + .where(projects: { id: nil }) + invalid_ids = invalid_project_members.pluck(:id) + + # the actual delete + deleted_count = invalid_project_members.delete_all + + Gitlab::AppLogger.info({ message: 'Removing invalid project member records', + deleted_count: deleted_count, + ids: invalid_ids }) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb new file mode 100644 index 00000000000..824054b31f2 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for public projects created after 17/02/2022 + class DisableLegacyOpenSourceLicenceForRecentPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + THRESHOLD_DATE = '2022-02-17 09:00:00' + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :disable_legacy_open_source_licence_for_recent_public_projects, + batching_scope: ->(relation) { + relation.where(visibility_level: PUBLIC).where('created_at >= ?', THRESHOLD_DATE) + } + ) do |sub_batch| + ProjectSetting.where(project_id: sub_batch) + .where(legacy_open_source_license_available: true) + .update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb new file mode 100644 index 00000000000..6e4d5d8ddcb --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for projects less than 1 MB + class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } + + def perform + each_sub_batch(operation_name: :disable_legacy_open_source_license_for_projects_less_than_one_mb) do |sub_batch| + updates = { legacy_open_source_license_available: false, updated_at: Time.current } + + sub_batch + .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id') + .where('project_statistics.repository_size < ?', 1.megabyte) + .update_all(updates) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb index 3605b157f4f..2bf631c6c7d 100644 --- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb +++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb @@ -11,7 +11,7 @@ module Gitlab @user = user @verification_from_mail = Gitlab.config.gitlab.email_from - mail( + mail_with_locale( template_path: 'unconfirm_mailer', template_name: 'unconfirm_notification_email', to: @user.notification_email_or_default, diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 72380af2c53..9a42d035285 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -58,7 +58,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r development: "a143e9e2-41b3-47bc-9a19-081d089229f4", test: "a143e9e2-41b3-47bc-9a19-081d089229f4", staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", - production: "58dc0f06-936c-43b3-93bb-71693f1b6570" + production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb new file mode 100644 index 00000000000..d30263976e8 --- /dev/null +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. + # These job artifacts will not be deleted and will have their `expire_at` removed. + class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob + # The migration would have backfilled `expire_at` + # to midnight on the 22nd of the month of the local timezone, + # storing it as UTC time in the database. + # + # If the timezone setting has changed since the migration, + # the `expire_at` stored in the database could have changed to a different local time other than midnight. + # For example: + # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00. + # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 + # on the previous day (21st). + # + # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark + # on the dates 21, 22, 23 of the month will not be deleted. + # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets + EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL + EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) + AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) + AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0 + SQL + + scope_to ->(relation) { + relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE) + .or(relation.where(file_type: 3)) + } + + def perform + each_sub_batch( + operation_name: :update_all + ) do |sub_batch| + sub_batch.update_all(expire_at: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb new file mode 100644 index 00000000000..5b1d630bb03 --- /dev/null +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removes obsolete wiki notes + class RemoveSelfManagedWikiNotes < BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :delete_all + ) do |sub_batch| + sub_batch.where(noteable_type: 'Wiki').delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb new file mode 100644 index 00000000000..718fb0aaa71 --- /dev/null +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Renames all system notes created when an issuable task is checked/unchecked + # from `task` into `checklist item` + # `marked the task **Task 1** as incomplete` => `marked the checklist item **Task 1** as incomplete` + class RenameTaskSystemNoteToChecklistItem < BatchedMigrationJob + REPLACE_REGEX = '\Amarked\sthe\stask' + TEXT_REPLACEMENT = 'marked the checklist item' + + scope_to ->(relation) { + relation.where(system_note_metadata: { action: :task }) + } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + ApplicationRecord.connection.execute <<~SQL + UPDATE notes + SET note = REGEXP_REPLACE(notes.note,'#{REPLACE_REGEX}', '#{TEXT_REPLACEMENT}') + FROM (#{sub_batch.select(:note_id).to_sql}) AS metadata_fields(note_id) + WHERE notes.id = note_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index fd6cbcb8d05..a0cfeed618a 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -6,11 +6,10 @@ module Gitlab class SetCorrectVulnerabilityState < BatchedMigrationJob DISMISSED_STATE = 2 + scope_to ->(relation) { relation.where.not(dismissed_at: nil) } + def perform - each_sub_batch( - operation_name: :update_vulnerabilities_state, - batching_scope: -> (relation) { relation.where.not(dismissed_at: nil) } - ) do |sub_batch| + each_sub_batch(operation_name: :update_vulnerabilities_state) do |sub_batch| sub_batch.update_all(state: DISMISSED_STATE) end end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index 81b01395542..c8520993b8e 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,6 +3,7 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab + # rubocop:disable Rails/ApplicationController class BaseDoorkeeperController < ActionController::Base include Gitlab::Allowable include EnforcesTwoFactorAuthentication @@ -12,4 +13,5 @@ module Gitlab helper_method :can? end + # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb index 7b11d6bc9ff..48b6ca59367 100644 --- a/lib/gitlab/cache/helpers.rb +++ b/lib/gitlab/cache/helpers.rb @@ -57,9 +57,19 @@ module Gitlab # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @return [String] def cached_object(object, presenter:, presenter_args:, context:, expires_in:) - cache.fetch(contextual_cache_key(presenter, object, context), expires_in: expires_in) do - Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + misses = 0 + + json = cache.fetch(contextual_cache_key(presenter, object, context), expires_in: expires_in) do + time_action(render_type: :object) do + misses += 1 + + Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + end end + + increment_cache_metric(render_type: :object, total_count: 1, miss_count: misses) + + json end # Used for fetching or rendering multiple objects @@ -71,10 +81,18 @@ module Gitlab # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @return [Array<String>] def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:) + misses = 0 + json = fetch_multi(presenter, collection, context: context, expires_in: expires_in) do |obj| - Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + time_action(render_type: :collection) do + misses += 1 + + Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + end end + increment_cache_metric(render_type: :collection, total_count: collection.length, miss_count: misses) + json.values end @@ -106,6 +124,57 @@ module Gitlab contextual_cache_key(presenter, object, context) end end + + def increment_cache_metric(render_type:, total_count:, miss_count:) + return unless Feature.enabled?(:add_timing_to_certain_cache_actions) + return unless caller_id + + metric_name = :cached_object_operations_total + hit_count = total_count - miss_count + + current_transaction&.increment( + metric_name, + hit_count, + { caller_id: caller_id, render_type: render_type, cache_hit: true } + ) + + current_transaction&.increment( + metric_name, + miss_count, + { caller_id: caller_id, render_type: render_type, cache_hit: false } + ) + end + + def time_action(render_type:, &block) + if Feature.enabled?(:add_timing_to_certain_cache_actions) + real_start = Gitlab::Metrics::System.monotonic_time + + presented_object = yield + + real_duration_histogram(render_type).observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + + presented_object + else + yield + end + end + + def real_duration_histogram(render_type) + Gitlab::Metrics.histogram( + :gitlab_presentable_object_cacheless_render_real_duration_seconds, + 'Duration of generating presentable objects to be cached in real time', + { caller_id: caller_id, render_type: render_type }, + [0.1, 0.5, 1, 2] + ) + end + + def current_transaction + @current_transaction ||= ::Gitlab::Metrics::WebTransaction.current + end + + def caller_id + @caller_id ||= Gitlab::ApplicationContext.current_context_attribute(:caller_id) + end end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 10233cf4228..2ab702aa4f9 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -19,11 +19,11 @@ module Gitlab }.freeze STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 }.freeze def self.convert(ansi, state = nil) diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb index 79b42a5f5bf..fdd49df1e24 100644 --- a/lib/gitlab/ci/ansi2json/parser.rb +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -20,11 +20,11 @@ module Gitlab }.freeze STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 }.freeze def self.bold?(mask) diff --git a/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb deleted file mode 100644 index 690a47097c6..00000000000 --- a/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Artifacts - module Adapters - class ZipStream - MAX_DECOMPRESSED_SIZE = 100.megabytes - MAX_FILES_PROCESSED = 50 - - attr_reader :stream - - InvalidStreamError = Class.new(StandardError) - - def initialize(stream) - raise InvalidStreamError, "Stream is required" unless stream - - @stream = stream - @files_processed = 0 - end - - def each_blob - Zip::InputStream.open(stream) do |zio| - while entry = zio.get_next_entry - break if at_files_processed_limit? - next unless should_process?(entry) - - @files_processed += 1 - - yield entry.get_input_stream.read - end - end - end - - private - - def should_process?(entry) - file?(entry) && !too_large?(entry) - end - - def file?(entry) - # Check the file name as a workaround for incorrect - # file type detection when using InputStream - # https://github.com/rubyzip/rubyzip/issues/533 - entry.file? && !entry.name.end_with?('/') - end - - def too_large?(entry) - entry.size > MAX_DECOMPRESSED_SIZE - end - - def at_files_processed_limit? - @files_processed >= MAX_FILES_PROCESSED - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 641aa71fb4e..a1a8e9288c7 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -32,7 +32,18 @@ module Gitlab end def build_attributes - attributes.merge(pipeline_attributes) + attributes.merge(pipeline_attributes, ci_stage_attributes) + end + + def ci_stage_attributes + { + ci_stage: ::Ci::Stage.new( + name: attributes[:stage], + position: attributes[:stage_idx], + pipeline: pipeline_attributes[:pipeline], + project: pipeline_attributes[:project] + ) + } end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index e2b54797dc8..aebd81e7b07 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -24,7 +24,7 @@ module Gitlab private def worktree_paths(context) - return unless context.project + return [] unless context.project if @top_level_only context.top_level_worktree_paths diff --git a/lib/gitlab/ci/config/entry/current_variables.rb b/lib/gitlab/ci/config/entry/current_variables.rb new file mode 100644 index 00000000000..3b6721ec92d --- /dev/null +++ b/lib/gitlab/ci/config/entry/current_variables.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents CI/CD variables. + # The class will be renamed to `Variables` when removing the FF `ci_variables_refactoring_to_variable`. + # + class CurrentVariables < ::Gitlab::Config::Entry::ComposableHash + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Hash + end + + # Enable these lines when removing the FF `ci_variables_refactoring_to_variable` + # and renaming this class to `Variables`. + # def self.default(**) + # {} + # end + + def value + @entries.to_h do |key, entry| + [key.to_s, entry.value] + end + end + + def value_with_data + @entries.to_h do |key, entry| + [key.to_s, entry.value_with_data] + end + end + + private + + def composable_class(_name, _config) + Entry::Variable + end + + def composable_metadata + { allowed_value_data: opt(:allowed_value_data) } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 96ba3553b46..a727da87308 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -54,7 +54,7 @@ module Gitlab validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true - validates :auto_stop_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser }, allow_nil: true + validates :auto_stop_in, type: String, allow_nil: true end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 613f7ff3370..84e31ca1fc6 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -11,10 +11,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Imageable validations do - validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS, - if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: IMAGEABLE_LEGACY_ALLOWED_KEYS, - unless: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS end def value @@ -25,7 +22,7 @@ module Gitlab name: @config[:name], entrypoint: @config[:entrypoint], ports: (ports_value if ports_defined?), - pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) + pull_policy: pull_policy_value }.compact else {} diff --git a/lib/gitlab/ci/config/entry/imageable.rb b/lib/gitlab/ci/config/entry/imageable.rb index f045ee3d549..1aecfee9ab9 100644 --- a/lib/gitlab/ci/config/entry/imageable.rb +++ b/lib/gitlab/ci/config/entry/imageable.rb @@ -13,7 +13,6 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable IMAGEABLE_ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze - IMAGEABLE_LEGACY_ALLOWED_KEYS = %i[name entrypoint ports].freeze included do include ::Gitlab::Config::Entry::Validatable @@ -47,10 +46,6 @@ module Gitlab opt(:with_image_ports) end - def ci_docker_image_pull_policy_enabled? - ::Feature.enabled?(:ci_docker_image_pull_policy) - end - def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/entry/legacy_variables.rb b/lib/gitlab/ci/config/entry/legacy_variables.rb new file mode 100644 index 00000000000..5379f707537 --- /dev/null +++ b/lib/gitlab/ci/config/entry/legacy_variables.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents environment variables. + # This is legacy implementation and will be removed with the FF `ci_variables_refactoring_to_variable`. + # + class LegacyVariables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_VALUE_DATA = %i[value description].freeze + + validations do + validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? + validates :config, variables: true, unless: :use_value_data? + end + + def value + @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } + end + + def value_with_data + @config.to_h { |key, value| [key.to_s, expand_value(value)] } + end + + def use_value_data? + opt(:use_value_data) + end + + private + + def expand_value(value) + if value.is_a?(Hash) + { value: value[:value].to_s, description: value[:description] }.compact + else + { value: value.to_s } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 78794f524f4..2d2032b1d8c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -29,7 +29,7 @@ module Gitlab in: %i[only except start_in], message: 'key may not be used with `rules`' }, - if: :has_rules? + if: :has_rules? with_options allow_nil: true do validates :extends, array_of_strings_or_string: true @@ -120,7 +120,7 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - job_variables: variables_value.to_h, + job_variables: variables_entry.value_with_data, root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index ff11c757dfa..57e89bd7bc5 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -48,9 +48,10 @@ module Gitlab description: 'Script that will be executed after each job.', reserved: true + # use_value_data will be removed with the FF ci_variables_refactoring_to_variable entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { use_value_data: true }, + metadata: { use_value_data: true, allowed_value_data: %i[value description] }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 0e19447dff8..4b3a9990df4 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -11,14 +11,9 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Imageable ALLOWED_KEYS = %i[command alias variables].freeze - LEGACY_ALLOWED_KEYS = %i[command alias variables].freeze validations do - validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS, - if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: LEGACY_ALLOWED_KEYS + IMAGEABLE_LEGACY_ALLOWED_KEYS, - unless: :ci_docker_image_pull_policy_enabled? - + validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } @@ -43,7 +38,7 @@ module Gitlab { name: @config } elsif hash? @config.merge( - pull_policy: (pull_policy_value if ci_docker_image_pull_policy_enabled?) + pull_policy: pull_policy_value ).compact else {} diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb new file mode 100644 index 00000000000..253888aadeb --- /dev/null +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a CI/CD variable. + # + class Variable < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleVariable, if: -> (config) { SimpleVariable.applies_to?(config) } + strategy :ComplexVariable, if: -> (config) { ComplexVariable.applies_to?(config) } + + class SimpleVariable < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + class << self + def applies_to?(config) + Gitlab::Config::Entry::Validators::AlphanumericValidator.validate(config) + end + end + + validations do + validates :key, alphanumeric: true + validates :config, alphanumeric: true + end + + def value + @config.to_s + end + + def value_with_data + { value: @config.to_s } + end + end + + class ComplexVariable < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + class << self + def applies_to?(config) + config.is_a?(Hash) + end + end + + validations do + validates :key, alphanumeric: true + validates :config_value, alphanumeric: true, allow_nil: false, if: :config_value_defined? + validates :config_description, alphanumeric: true, allow_nil: false, if: :config_description_defined? + + validate do + allowed_value_data = Array(opt(:allowed_value_data)) + + if allowed_value_data.any? + extra_keys = config.keys - allowed_value_data + + errors.add(:config, "uses invalid data keys: #{extra_keys.join(', ')}") if extra_keys.present? + else + errors.add(:config, "must be a string") + end + end + end + + def value + config_value.to_s + end + + def value_with_data + { value: value, description: config_description }.compact + end + + def config_value + @config[:value] + end + + def config_description + @config[:description] + end + + def config_value_defined? + config.key?(:value) + end + + def config_description_defined? + config.key?(:description) + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["variable definition must be either a string or a hash"] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index efb469ee32a..0284958d9d4 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -5,43 +5,21 @@ module Gitlab class Config module Entry ## - # Entry that represents environment variables. + # Entry that represents CI/CD variables. + # CurrentVariables will be renamed to this class when removing the FF `ci_variables_refactoring_to_variable`. # - class Variables < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - - ALLOWED_VALUE_DATA = %i[value description].freeze - - validations do - validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? - validates :config, variables: true, unless: :use_value_data? - end - - def value - @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } + class Variables + def self.new(...) + if YamlProcessor::FeatureFlags.enabled?(:ci_variables_refactoring_to_variable) + CurrentVariables.new(...) + else + LegacyVariables.new(...) + end end def self.default(**) {} end - - def value_with_data - @config.to_h { |key, value| [key.to_s, expand_value(value)] } - end - - def use_value_data? - opt(:use_value_data) - end - - private - - def expand_value(value) - if value.is_a?(Hash) - { value: value[:value].to_s, description: value[:description] } - else - { value: value.to_s, description: nil } - end - end end end end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 278353220e4..4e01688a955 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -8,7 +8,7 @@ module Gitlab def reserved_claims super.merge( iss: Settings.gitlab.base_url, - sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", + sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", aud: Settings.gitlab.base_url ) end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index deb20a2138c..aa594ca4049 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -6,7 +6,6 @@ module Gitlab module Sbom class Cyclonedx SUPPORTED_SPEC_VERSIONS = %w[1.4].freeze - COMPONENT_ATTRIBUTES = %w[type name version].freeze def parse!(blob, sbom_report) @report = sbom_report @@ -62,10 +61,17 @@ module Gitlab end def parse_components - data['components']&.each do |component| - next unless supported_component_type?(component['type']) + data['components']&.each do |component_data| + type = component_data['type'] + next unless supported_component_type?(type) - report.add_component(component.slice(*COMPONENT_ATTRIBUTES)) + component = ::Gitlab::Ci::Reports::Sbom::Component.new( + type: type, + name: component_data['name'], + version: component_data['version'] + ) + + report.add_component(component) end end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb index ad04b3257f9..00ca723b258 100644 --- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -21,11 +21,11 @@ module Gitlab def source return unless required_attributes_present? - { - 'type' => :dependency_scanning, - 'data' => data, - 'fingerprint' => fingerprint - } + ::Gitlab::Ci::Reports::Sbom::Source.new( + type: :dependency_scanning, + data: data, + fingerprint: fingerprint + ) end private diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 13a159f3745..da7faaab6ff 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -7,16 +7,16 @@ module Gitlab class Common SecurityReportParserError = Class.new(Gitlab::Ci::Parsers::ParserError) - def self.parse!(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) - new(json_data, report, vulnerability_finding_signatures_enabled, validate: validate).parse! + def self.parse!(json_data, report, signatures_enabled: false, validate: false) + new(json_data, report, signatures_enabled: signatures_enabled, validate: validate).parse! end - def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) + def initialize(json_data, report, signatures_enabled: false, validate: false) @json_data = json_data @report = report @project = report.project @validate = validate - @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + @signatures_enabled = signatures_enabled end def parse! @@ -26,7 +26,7 @@ module Gitlab raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) - create_scanner + create_scanner(top_level_scanner_data) create_scan create_analyzer @@ -77,7 +77,7 @@ module Gitlab report_data, report.version, project: @project, - scanner: top_level_scanner + scanner: top_level_scanner_data ) end @@ -89,8 +89,8 @@ module Gitlab @report_version ||= report_data['version'] end - def top_level_scanner - @top_level_scanner ||= report_data.dig('scan', 'scanner') + def top_level_scanner_data + @top_level_scanner_data ||= report_data.dig('scan', 'scanner') end def scan_data @@ -119,7 +119,7 @@ module Gitlab evidence = create_evidence(data['evidence']) signatures = create_signatures(tracking_data(data)) - if @vulnerability_finding_signatures_enabled && !signatures.empty? + if @signatures_enabled && !signatures.empty? # NOT the signature_sha - the compare key is hashed # to create the project_fingerprint highest_priority_signature = signatures.max_by(&:priority) @@ -138,7 +138,7 @@ module Gitlab evidence: evidence, severity: parse_severity_level(data['severity']), confidence: parse_confidence_level(data['confidence']), - scanner: create_scanner(data['scanner']), + scanner: create_scanner(top_level_scanner_data || data['scanner']), scan: report&.scan, identifiers: identifiers, flags: flags, @@ -149,7 +149,7 @@ module Gitlab details: data['details'] || {}, signatures: signatures, project_id: @project.id, - vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled)) + vulnerability_finding_signatures_enabled: @signatures_enabled)) end def create_signatures(tracking) @@ -208,7 +208,7 @@ module Gitlab report.analyzer = ::Gitlab::Ci::Reports::Security::Analyzer.new(**params) end - def create_scanner(scanner_data = top_level_scanner) + def create_scanner(scanner_data) return unless scanner_data.is_a?(Hash) report.add_scanner( diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index c075ada725a..28d6620e5c4 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = [].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..db4c7ab1425 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json @@ -0,0 +1,977 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json new file mode 100644 index 00000000000..641cfc82e48 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json @@ -0,0 +1,911 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..59aa172444d --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json new file mode 100644 index 00000000000..0e4c866794a --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json @@ -0,0 +1,1287 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + }, + "discovered_at": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", + "examples": [ + "2020-01-28T03:26:02.956" + ] + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json new file mode 100644 index 00000000000..652c2f48fe4 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json @@ -0,0 +1,968 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "dependency_files", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json new file mode 100644 index 00000000000..40d4d9f5287 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json @@ -0,0 +1,869 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json new file mode 100644 index 00000000000..cfde126dd7b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json @@ -0,0 +1,892 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..7ccb39a2b8e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/cluster-image-scanning-report-format.json @@ -0,0 +1,946 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/container-scanning-report-format.json new file mode 100644 index 00000000000..2517832853e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/container-scanning-report-format.json @@ -0,0 +1,880 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..a2f9eb12992 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/coverage-fuzzing-report-format.json @@ -0,0 +1,836 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dast-report-format.json new file mode 100644 index 00000000000..10fafaf8975 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dast-report-format.json @@ -0,0 +1,1241 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dependency-scanning-report-format.json new file mode 100644 index 00000000000..ade1ce9ea8f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dependency-scanning-report-format.json @@ -0,0 +1,944 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/sast-report-format.json new file mode 100644 index 00000000000..9fae45d728e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/sast-report-format.json @@ -0,0 +1,831 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/secret-detection-report-format.json new file mode 100644 index 00000000000..fca00e17f26 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/secret-detection-report-format.json @@ -0,0 +1,854 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 999ffff85d2..d95ecff85cd 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -8,7 +8,9 @@ module Gitlab JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze - def parse!(xml_data, test_suite, job:) + def parse!(xml_data, test_report, job:) + test_suite = test_report.get_suite(job.test_suite_name) + root = Hash.from_xml(xml_data) total_parsed = 0 max_test_cases = job.max_test_cases_per_report diff --git a/lib/gitlab/ci/pipeline/chain/assign_partition.rb b/lib/gitlab/ci/pipeline/chain/assign_partition.rb new file mode 100644 index 00000000000..4b8efe13d44 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/assign_partition.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class AssignPartition < Chain::Base + include Chain::Helpers + + def perform! + @pipeline.partition_id = find_partition_id + end + + def break? + @pipeline.errors.any? + end + + private + + def find_partition_id + if @command.creates_child_pipeline? + @command.parent_pipeline_partition_id + else + ::Ci::Pipeline.current_partition_value + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 0a6f6fd740c..14c320f77bf 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -80,6 +80,10 @@ module Gitlab bridge&.parent_pipeline end + def parent_pipeline_partition_id + parent_pipeline.partition_id if creates_child_pipeline? + end + def creates_child_pipeline? bridge&.triggers_child_pipeline? end @@ -117,8 +121,14 @@ module Gitlab end def observe_jobs_count_in_alive_pipelines + jobs_count = if Feature.enabled?(:ci_limit_active_jobs_early, project) + project.all_pipelines.jobs_count_in_alive_pipelines + else + project.all_pipelines.builds_count_in_alive_pipelines + end + metrics.active_jobs_histogram - .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines) + .observe({ plan: project.actual_plan_name }, jobs_count) end def increment_pipeline_failure_reason_counter(reason) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index 3c150ca26bb..a14dec48619 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -7,6 +7,7 @@ module Gitlab module Config class Content < Chain::Base include Chain::Helpers + include ::Gitlab::Utils::StrongMemoize SOURCES = [ Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter, @@ -18,10 +19,10 @@ module Gitlab ].freeze def perform! - if config = find_config - @pipeline.build_pipeline_config(content: config.content) - @command.config_content = config.content - @pipeline.config_source = config.source + if pipeline_config&.exists? + @pipeline.build_pipeline_config(content: pipeline_config.content) + @command.config_content = pipeline_config.content + @pipeline.config_source = pipeline_config.source else error('Missing CI config file') end @@ -33,7 +34,19 @@ module Gitlab private - def find_config + def pipeline_config + strong_memoize(:pipeline_config) do + next legacy_find_config if ::Feature.disabled?(:ci_project_pipeline_config_refactoring, project) + + ::Gitlab::Ci::ProjectConfig.new( + project: project, sha: @pipeline.sha, + custom_content: @command.content, + pipeline_source: @command.source, pipeline_source_bridge: @command.bridge + ) + end + end + + def legacy_find_config sources.each do |source| config = source.new(@pipeline, @command) return config if config.exists? diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb index 8bc172f93d3..69dca1568b6 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/source.rb @@ -6,6 +6,7 @@ module Gitlab module Chain module Config class Content + # When removing ci_project_pipeline_config_refactoring, this and its subclasses will be removed. class Source include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index 245ef32f06b..3dd9b85d9b2 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -18,7 +18,9 @@ module Gitlab def ensure_environment(build) return unless build.instance_of?(::Ci::Build) && build.has_environment? - environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource + environment = ::Gitlab::Ci::Pipeline::Seed::Environment + .new(build, merge_request: @command.merge_request) + .to_resource if environment.persisted? build.persisted_environment = environment diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 6e95c7988fc..915e48828d2 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -57,7 +57,8 @@ module Gitlab }.compact Gitlab::HTTP.post( - validation_service_url, timeout: validation_service_timeout, + validation_service_url, + timeout: validation_service_timeout, headers: headers, body: validation_service_payload.to_json ) @@ -96,13 +97,17 @@ module Gitlab last_sign_in_ip: current_user.last_sign_in_ip, sign_in_count: current_user.sign_in_count }, + credit_card: { + similar_cards_count: current_user.credit_card_validation&.similar_records&.count.to_i, + similar_holder_names_count: current_user.credit_card_validation&.similar_holder_names_count.to_i + }, pipeline: { sha: pipeline.sha, ref: pipeline.ref, type: pipeline.source }, builds: builds_validation_payload, - total_builds_count: current_user.pipelines.jobs_count_in_alive_pipelines + total_builds_count: current_user.pipelines.builds_count_in_alive_pipelines } end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 93106b96af2..2e4267e986b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -148,7 +148,9 @@ module Gitlab ref: @pipeline.ref, tag: @pipeline.tag, trigger_request: @pipeline.legacy_trigger, - protected: @pipeline.protected_ref? + protected: @pipeline.protected_ref?, + partition_id: @pipeline.partition_id, + metadata_attributes: { partition_id: @pipeline.partition_id } } end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 6bcc71a808b..8353bc523bf 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -5,17 +5,21 @@ module Gitlab module Pipeline module Seed class Environment < Seed::Base - attr_reader :job + attr_reader :job, :merge_request - def initialize(job) + delegate :simple_variables, to: :job + + def initialize(job, merge_request: nil) @job = job + @merge_request = merge_request end def to_resource environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| # Initialize the attributes at creation - environment.auto_stop_in = auto_stop_in + environment.auto_stop_in = expanded_auto_stop_in environment.tier = deployment_tier + environment.merge_request = merge_request end end @@ -36,6 +40,12 @@ module Gitlab def expanded_environment_name job.expanded_environment_name end + + def expanded_auto_stop_in + return unless auto_stop_in + + ExpandVariables.expand(auto_stop_in, -> { simple_variables.sort_and_expand_all }) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 7cf6466cf4b..1c4247bd5ee 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -25,7 +25,8 @@ module Gitlab { name: @attributes.fetch(:name), position: @attributes.fetch(:index), pipeline: @pipeline, - project: @pipeline.project } + project: @pipeline.project, + partition_id: @pipeline.partition_id } end def seeds diff --git a/lib/gitlab/ci/processable_object_hierarchy.rb b/lib/gitlab/ci/processable_object_hierarchy.rb new file mode 100644 index 00000000000..1122361e27e --- /dev/null +++ b/lib/gitlab/ci/processable_object_hierarchy.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProcessableObjectHierarchy < ::Gitlab::ObjectHierarchy + private + + def middle_table + ::Ci::BuildNeed.arel_table + end + + def from_tables(cte) + [objects_table, cte.table, middle_table] + end + + def parent_id_column(_cte) + middle_table[:name] + end + + def ancestor_conditions(cte) + middle_table[:name].eq(objects_table[:name]).and( + middle_table[:build_id].eq(cte.table[:id]) + ) + end + + def descendant_conditions(cte) + middle_table[:build_id].eq(objects_table[:id]).and( + middle_table[:name].eq(cte.table[:name]) + ) + end + end + end +end diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb new file mode 100644 index 00000000000..ded6877ef29 --- /dev/null +++ b/lib/gitlab/ci/project_config.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + # Locates project CI config + class ProjectConfig + # The order of sources is important: + # - EE uses Compliance first since it must be used first if compliance templates are enabled. + # (see ee/lib/ee/gitlab/ci/project_config.rb) + # - Parameter is used by on-demand security scanning which passes the actual CI YAML to use as argument. + # - Bridge is used for downstream pipelines since the config is defined in the bridge job. If lower in priority, + # it would evaluate the project's YAML file instead. + # - Repository / ExternalProject / Remote: their order is not important between each other. + # - AutoDevops is used as default option if nothing else is found and if AutoDevops is enabled. + SOURCES = [ + ProjectConfig::Parameter, + ProjectConfig::Bridge, + ProjectConfig::Repository, + ProjectConfig::ExternalProject, + ProjectConfig::Remote, + ProjectConfig::AutoDevops + ].freeze + + def initialize(project:, sha:, custom_content: nil, pipeline_source: nil, pipeline_source_bridge: nil) + @config = find_config(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + end + + delegate :content, :source, to: :@config, allow_nil: true + + def exists? + !!@config&.exists? + end + + private + + def find_config(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + sources.each do |source| + config = source.new(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + return config if config.exists? + end + + nil + end + + def sources + SOURCES + end + end + end +end + +Gitlab::Ci::ProjectConfig.prepend_mod_with('Gitlab::Ci::ProjectConfig') diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb new file mode 100644 index 00000000000..c6905f480a2 --- /dev/null +++ b/lib/gitlab/ci/project_config/auto_devops.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class AutoDevops < Source + def content + strong_memoize(:content) do + next unless project&.auto_devops_enabled? + + template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) + YAML.dump('include' => [{ 'template' => template.full_name }]) + end + end + + def source + :auto_devops_source + end + + private + + def template_name + 'Auto-DevOps' + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/bridge.rb b/lib/gitlab/ci/project_config/bridge.rb new file mode 100644 index 00000000000..c342ab2c215 --- /dev/null +++ b/lib/gitlab/ci/project_config/bridge.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Bridge < Source + def content + return unless pipeline_source_bridge + + pipeline_source_bridge.yaml_for_downstream + end + + def source + :bridge_source + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb new file mode 100644 index 00000000000..0ed5d6fa226 --- /dev/null +++ b/lib/gitlab/ci/project_config/external_project.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class ExternalProject < Source + def content + strong_memoize(:content) do + next unless external_project_path? + + path_file, path_project, ref = extract_location_tokens + + config_location = { 'project' => path_project, 'file' => path_file } + config_location['ref'] = ref if ref.present? + + YAML.dump('include' => [config_location]) + end + end + + def source + :external_project_source + end + + private + + # Example: path/to/.gitlab-ci.yml@another-group/another-project + def external_project_path? + ci_config_path =~ /\A.+(yml|yaml)@.+\z/ + end + + # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname + def extract_location_tokens + path_file, path_project = ci_config_path.split('@', 2) + + if path_project.include? ":" + project, ref = path_project.split(':', 2) + [path_file, project, ref] + else + [path_file, path_project] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/parameter.rb b/lib/gitlab/ci/project_config/parameter.rb new file mode 100644 index 00000000000..69e699c27f1 --- /dev/null +++ b/lib/gitlab/ci/project_config/parameter.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Parameter < Source + def content + strong_memoize(:content) do + next unless custom_content.present? + + custom_content + end + end + + def source + :parameter_source + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb new file mode 100644 index 00000000000..cf1292706d2 --- /dev/null +++ b/lib/gitlab/ci/project_config/remote.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Remote < Source + def content + strong_memoize(:content) do + next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) + + YAML.dump('include' => [{ 'remote' => ci_config_path }]) + end + end + + def source + :remote_source + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb new file mode 100644 index 00000000000..435ad4d42fe --- /dev/null +++ b/lib/gitlab/ci/project_config/repository.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Repository < Source + def content + strong_memoize(:content) do + next unless file_in_repository? + + YAML.dump('include' => [{ 'local' => ci_config_path }]) + end + end + + def source + :repository_source + end + + private + + def file_in_repository? + return unless project + return unless sha + + project.repository.gitlab_ci_yml_for(sha, ci_config_path).present? + rescue GRPC::NotFound, GRPC::Internal + nil + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb new file mode 100644 index 00000000000..ebe5728163b --- /dev/null +++ b/lib/gitlab/ci/project_config/source.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Source + include Gitlab::Utils::StrongMemoize + + def initialize(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + @project = project + @sha = sha + @custom_content = custom_content + @pipeline_source = pipeline_source + @pipeline_source_bridge = pipeline_source_bridge + end + + def exists? + strong_memoize(:exists) do + content.present? + end + end + + def content + raise NotImplementedError + end + + def source + raise NotImplementedError + end + + private + + attr_reader :project, :sha, :custom_content, :pipeline_source, :pipeline_source_bridge + + def ci_config_path + @ci_config_path ||= project.ci_config_path_or_default + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb index 6d57e05aa63..88b3b14d5c9 100644 --- a/lib/gitlab/ci/reports/coverage_report_generator.rb +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -35,7 +35,7 @@ module Gitlab private def report_builds - @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) + @pipeline.latest_report_builds_in_self_and_project_descendants(::Ci::JobArtifact.of_report_type(:coverage)) end end end diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 86b9be274cc..198b34451b4 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -7,10 +7,10 @@ module Gitlab class Component attr_reader :component_type, :name, :version - def initialize(component = {}) - @component_type = component['type'] - @name = component['name'] - @version = component['version'] + def initialize(type:, name:, version:) + @component_type = type + @name = name + @version = version end end end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb index dc6b3153e51..4f84d12f78c 100644 --- a/lib/gitlab/ci/reports/sbom/report.rb +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -17,11 +17,11 @@ module Gitlab end def set_source(source) - self.source = Source.new(source) + self.source = source end def add_component(component) - components << Component.new(component) + components << component end private diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index 60bf30b65a5..ea0fb8d4fbb 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -7,10 +7,10 @@ module Gitlab class Source attr_reader :source_type, :data, :fingerprint - def initialize(source = {}) - @source_type = source['type'] - @data = source['data'] - @fingerprint = source['fingerprint'] + def initialize(type:, data:, fingerprint:) + @source_type = type + @data = data + @fingerprint = fingerprint end end end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb index 1ac66a0c671..918df163ede 100644 --- a/lib/gitlab/ci/reports/security/scanner.rb +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -7,13 +7,13 @@ module Gitlab class Scanner ANALYZER_ORDER = { "bundler_audit" => 1, - "retire.js" => 2, + "retire.js" => 2, "gemnasium" => 3, "gemnasium-maven" => 3, "gemnasium-python" => 3, "bandit" => 1, "spotbugs" => 1, - "semgrep" => 2 + "semgrep" => 2 }.freeze attr_accessor :external_id, :name, :vendor, :version diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 5d60aa8f540..a136044c124 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -31,6 +31,7 @@ module Gitlab downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', + reached_max_pipeline_hierarchy_size: 'downstream pipeline tree is too large', project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', ci_quota_exceeded: 'no more CI minutes available', @@ -39,7 +40,8 @@ module Gitlab builds_disabled: 'project builds are disabled', environment_creation_failure: 'environment creation failure', deployment_rejected: 'deployment rejected', - ip_restriction_failure: 'IP address restriction failure' + ip_restriction_failure: 'IP address restriction failure', + failed_outdated_deployment_job: 'failed outdated deployment job' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index f0ddc4b4916..539e1a6385d 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..70f85382967 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,244 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ +# +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DS_EXCLUDED_ANALYZERS: "" + DS_EXCLUDED_PATHS: "spec, test, tests, tmp" + DS_MAJOR_VERSION: 3 + +dependency_scanning: + stage: test + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + dependencies: [] + rules: + - when: never + +.ds-analyzer: + extends: dependency_scanning + allow_failure: true + variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/$DS_ANALYZER_NAME:$DS_MAJOR_VERSION" + # DS_ANALYZER_NAME is an undocumented variable used in job definitions + # to inject the analyzer name in the image name. + DS_ANALYZER_NAME: "" + image: + name: "$DS_ANALYZER_IMAGE$DS_IMAGE_SUFFIX" + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +.cyclonedx-reports: + artifacts: + paths: + - "**/gl-sbom-*.cdx.json" + +.gemnasium-shared-rule: + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + +gemnasium-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium" + GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + +.gemnasium-maven-shared-rule: + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' + +gemnasium-maven-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium-maven" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + +.gemnasium-python-shared-rule: + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + +gemnasium-python-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium-python" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE + +bundler-audit-dependency_scanning: + extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "bundler-audit" + DS_MAJOR_VERSION: 2 + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/347491" + - exit 1 + rules: + - when: never + +retire-js-dependency_scanning: + extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "retire.js" + DS_MAJOR_VERSION: 2 + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/289830" + - exit 1 + rules: + - when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 1a2a8b4edb4..78fe108e8b9 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index cb8818357a2..bc2e1fed0d4 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..e47f669c2e2 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,48 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html +# +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + + LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. + LICENSE_MANAGEMENT_VERSION: 4 + +license_scanning: + stage: test + image: + name: "$SECURE_ANALYZERS_PREFIX/license-finder:$LICENSE_MANAGEMENT_VERSION" + entrypoint: [""] + variables: + LM_REPORT_VERSION: '2.1' + SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD + allow_failure: true + script: + - /run.sh analyze . + artifacts: + reports: + license_scanning: gl-license-scanning-report.json + dependencies: [] + rules: + - if: $LICENSE_MANAGEMENT_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index dd164c00724..a6d47e31de2 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -36,19 +36,12 @@ sast: bandit-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.4" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.py' + - when: never brakeman-sast: extends: .sast-analyzer @@ -69,23 +62,12 @@ brakeman-sast: eslint-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.4" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' + - when: never flawfinder-sast: extends: .sast-analyzer @@ -125,19 +107,12 @@ kubesec-sast: gosec-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 3 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.4" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.go' + - when: never .mobsf-sast: extends: .sast-analyzer @@ -261,6 +236,8 @@ semgrep-sast: - '**/*.c' - '**/*.go' - '**/*.java' + - '**/*.cs' + - '**/*.html' sobelow-sast: extends: .sast-analyzer @@ -297,6 +274,5 @@ spotbugs-sast: - if: $CI_COMMIT_BRANCH exists: - '**/*.groovy' - - '**/*.java' - '**/*.scala' - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index c6938920ea4..c0ca821ebff 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -36,24 +36,12 @@ sast: bandit-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.3" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.py' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.py' + - when: never brakeman-sast: extends: .sast-analyzer @@ -80,32 +68,12 @@ brakeman-sast: eslint-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.3" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' + - when: never flawfinder-sast: extends: .sast-analyzer @@ -138,6 +106,15 @@ flawfinder-sast: - '**/*.cp' - '**/*.cxx' +gosec-sast: + extends: .sast-analyzer + script: + - echo "This job was deprecated in GitLab 15.0 and removed in GitLab 15.2" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 + rules: + - when: never + kubesec-sast: extends: .sast-analyzer image: @@ -159,27 +136,6 @@ kubesec-sast: - if: $CI_COMMIT_BRANCH && $SCAN_KUBERNETES_MANIFESTS == 'true' -gosec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 3 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.go' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.go' - .mobsf-sast: extends: .sast-analyzer image: @@ -323,7 +279,7 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SERACH_MAX_DEPTH: 20 + SEARCH_MAX_DEPTH: 20 SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: @@ -341,6 +297,8 @@ semgrep-sast: - '**/*.c' - '**/*.go' - '**/*.java' + - '**/*.html' + - '**/*.cs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. @@ -353,6 +311,8 @@ semgrep-sast: - '**/*.c' - '**/*.go' - '**/*.java' + - '**/*.html' + - '**/*.cs' sobelow-sast: extends: .sast-analyzer @@ -394,7 +354,6 @@ spotbugs-sast: - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.groovy' - - '**/*.java' - '**/*.scala' - '**/*.kt' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -402,6 +361,5 @@ spotbugs-sast: - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. exists: - '**/*.groovy' - - '**/*.java' - '**/*.scala' - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Katalon.gitlab-ci.yml b/lib/gitlab/ci/templates/Katalon.gitlab-ci.yml new file mode 100644 index 00000000000..c8939c8f5a2 --- /dev/null +++ b/lib/gitlab/ci/templates/Katalon.gitlab-ci.yml @@ -0,0 +1,65 @@ +# This template is provided and maintained by Katalon, an official Technology Partner with GitLab. +# +# Use this template to run a Katalon Studio test from this repository. +# You can: +# - Copy and paste this template into a new `.gitlab-ci.yml` file. +# - Add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# In either case, you must also select which job you want to run, `.katalon_tests` +# or `.katalon_tests_with_artifacts` (see configuration below), and add that configuration +# to a new job with `extends:`. For example: +# +# Katalon-tests: +# extends: +# - .katalon_tests_with_artifacts +# +# Requirements: +# - A Katalon Studio project with the content saved in the root GitLab repository folder. +# - An active KRE license. +# - A valid Katalon API key. +# +# CI/CD variables, set in the project CI/CD settings: +# - KATALON_TEST_SUITE_PATH: The default path is `Test Suites/<Your Test Suite Name>`. +# Defines which test suite to run. +# - KATALON_API_KEY: The Katalon API key. +# - KATALON_PROJECT_DIR: Optional. Add if the project is in another location. +# - KATALON_ORG_ID: Optional. Add if you are part of multiple Katalon orgs. +# Set to the Org ID that has KRE licenses assigned. For more info on the Org ID, +# see https://support.katalon.com/hc/en-us/articles/4724459179545-How-to-get-Organization-ID- + +.katalon_tests: + # Use the latest version of the Katalon Runtime Engine. You can also use other versions of the + # Katalon Runtime Engine by specifying another tag, for example `katalonstudio/katalon:8.1.2` + # or `katalonstudio/katalon:8.3.0`. + image: 'katalonstudio/katalon' + services: + - docker:dind + variables: + # Specify the Katalon Studio project directory. By default, it is stored under the root project folder. + KATALON_PROJECT_DIR: $CI_PROJECT_DIR + + # The following bash script has two different versions, one if you set the KATALON_ORG_ID + # CI/CD variable, and the other if you did not set it. If you have more than one org in + # admin.katalon.com you must set the KATALON_ORG_ID variable with an ORG ID or + # the Katalon Test Suite fails to run. + # + # You can update or add additional `katalonc` commands below. To see all of the arguments + # `katalonc` supports, go to https://docs.katalon.com/katalon-studio/docs/console-mode-execution.html + script: + - |- + if [[ $KATALON_ORG_ID == "" ]]; then + katalonc.sh -projectPath=$KATALON_PROJECT_DIR -apiKey=$KATALON_API_KEY -browserType="Chrome" -retry=0 -statusDelay=20 -testSuitePath="$KATALON_TEST_SUITE_PATH" -reportFolder=Reports/ + else + katalonc.sh -projectPath=$KATALON_PROJECT_DIR -apiKey=$KATALON_API_KEY -browserType="Chrome" -retry=0 -statusDelay=20 -orgID=$KATALON_ORG_ID -testSuitePath="$KATALON_TEST_SUITE_PATH" -reportFolder=Reports/ + fi + +# Upload the artifacts and make the junit report accessible under the Pipeline Tests +.katalon_tests_with_artifacts: + extends: .katalon_tests + artifacts: + when: always + paths: + - Reports/ + reports: + junit: + Reports/*/*/*/*.xml diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 3d7883fb87a..79a08c33fdf 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -11,12 +11,12 @@ # # Requirements: # - A `test` stage to be present in the pipeline. -# - You must define the image to be scanned in the DOCKER_IMAGE variable. If DOCKER_IMAGE is the +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the # same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. -# - Container registry credentials defined by `DOCKER_USER` and `DOCKER_PASSWORD` variables if the +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the # image to be scanned is in a private registry. # - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the -# DOCKERFILE_PATH variable. +# CS_DOCKERFILE_PATH variable. # # Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f7b1d12b3b3 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,68 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 3a956ebfc49..9a40a23b276 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -9,7 +9,7 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/terraform:1.1.9" + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/1.1:v0.43.0" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 64c784f43cb..fb0d300338b 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -38,7 +38,7 @@ publish: # Compare the version in package.json to all published versions. # If the package.json version has not yet been published, run `npm publish`. - | - if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then + if [[ "$(npm view ${NPM_PACKAGE_NAME} versions)" != *"'${NPM_PACKAGE_VERSION}'"* ]]; then npm publish echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" else diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 95a60b852b8..c5664ef1cfb 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -23,7 +23,6 @@ module Gitlab attr_reader :job - delegate :old_trace, to: :job delegate :can_attempt_archival_now?, :increment_archival_attempts!, :archival_attempts_message, :archival_attempts_available?, to: :trace_metadata @@ -82,7 +81,7 @@ module Gitlab end def live? - job.trace_chunks.any? || current_path.present? || old_trace.present? + job.trace_chunks.any? || current_path.present? end def read(&block) @@ -111,7 +110,6 @@ module Gitlab # Erase the live trace erase_trace_chunks! FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace - job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace ensure @current_path = nil end @@ -162,8 +160,6 @@ module Gitlab Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") - elsif old_trace - StringIO.new(old_trace) end end @@ -210,11 +206,6 @@ module Gitlab archive_stream!(stream) FileUtils.rm(current_path) end - elsif old_trace - StringIO.new(old_trace, 'rb').tap do |stream| - archive_stream!(stream) - job.erase_old_trace! - end end end diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 95dff83506d..528d72c9bcc 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -118,7 +118,7 @@ module Gitlab def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_JOB_NAME', value: job.name) - variables.append(key: 'CI_JOB_STAGE', value: job.stage) + variables.append(key: 'CI_JOB_STAGE', value: job.stage_name) variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action? variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request @@ -127,7 +127,7 @@ module Gitlab # legacy variables variables.append(key: 'CI_BUILD_NAME', value: job.name) - variables.append(key: 'CI_BUILD_STAGE', value: job.stage) + variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? end diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index 7cc727bb3ea..16e3afd8620 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -6,24 +6,24 @@ module Gitlab module Helpers class << self def merge_variables(current_vars, new_vars) - current_vars = transform_from_yaml_variables(current_vars) - new_vars = transform_from_yaml_variables(new_vars) + return current_vars if new_vars.blank? - transform_to_yaml_variables( - current_vars.merge(new_vars) - ) - end + current_vars = transform_to_array(current_vars) if current_vars.is_a?(Hash) + new_vars = transform_to_array(new_vars) if new_vars.is_a?(Hash) - def transform_to_yaml_variables(vars) - vars.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + (new_vars + current_vars).uniq { |var| var[:key] } end - def transform_from_yaml_variables(vars) - return vars.stringify_keys.transform_values(&:to_s) if vars.is_a?(Hash) + def transform_to_array(vars) + return [] if vars.blank? - vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + vars.map do |key, data| + if data.is_a?(Hash) + { key: key.to_s, **data.except(:key) } + else + { key: key.to_s, value: data } + end + end end def inherit_yaml_variables(from:, to:, inheritance:) @@ -35,7 +35,7 @@ module Gitlab def apply_inheritance(variables, inheritance) case inheritance when true then variables - when false then {} + when false then [] when Array then variables.select { |var| inheritance.include?(var[:key]) } end end diff --git a/lib/gitlab/ci/yaml_processor/feature_flags.rb b/lib/gitlab/ci/yaml_processor/feature_flags.rb index f03db9d0e6b..50d37f6e4a0 100644 --- a/lib/gitlab/ci/yaml_processor/feature_flags.rb +++ b/lib/gitlab/ci/yaml_processor/feature_flags.rb @@ -5,10 +5,10 @@ module Gitlab class YamlProcessor module FeatureFlags ACTOR_KEY = 'ci_yaml_processor_feature_flag_actor' + CORRECT_USAGE_KEY = 'ci_yaml_processor_feature_flag_correct_usage' NO_ACTOR_VALUE = :no_actor - - NoActorError = Class.new(StandardError) NO_ACTOR_MESSAGE = "Actor not set. Ensure to call `enabled?` inside `with_actor` block" + NoActorError = Class.new(StandardError) class << self # Cache a feature flag actor as thread local variable so @@ -31,6 +31,15 @@ module Gitlab ::Feature.enabled?(feature_flag, current_actor) end + def ensure_correct_usage + previous = Thread.current[CORRECT_USAGE_KEY] + Thread.current[CORRECT_USAGE_KEY] = true + + yield + ensure + Thread.current[CORRECT_USAGE_KEY] = previous + end + private def current_actor @@ -39,10 +48,22 @@ module Gitlab value rescue NoActorError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + handle_missing_actor(e) nil end + + def handle_missing_actor(exception) + if ensure_correct_usage? + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + else + Gitlab::ErrorTracking.track_exception(exception) + end + end + + def ensure_correct_usage? + Thread.current[CORRECT_USAGE_KEY] == true + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 4bd1ac3b67f..f203f88442d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -43,7 +43,7 @@ module Gitlab end def root_variables - @root_variables ||= transform_to_yaml_variables(variables) + @root_variables ||= transform_to_array(variables) end def jobs @@ -70,7 +70,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], # yaml_variables is calculated with using job_variables in Seed::Build - job_variables: transform_to_yaml_variables(job[:job_variables]), + job_variables: transform_to_array(job[:job_variables]), root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], @@ -114,7 +114,7 @@ module Gitlab Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: root_variables, - to: transform_to_yaml_variables(job[:job_variables]), + to: job[:job_variables], inheritance: job.fetch(:root_variables_inheritance, true) ) end @@ -137,8 +137,8 @@ module Gitlab job[:release] end - def transform_to_yaml_variables(variables) - ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) + def transform_to_array(variables) + ::Gitlab::Ci::Variables::Helpers.transform_to_array(variables) end end end diff --git a/lib/gitlab/cleanup/personal_access_tokens.rb b/lib/gitlab/cleanup/personal_access_tokens.rb new file mode 100644 index 00000000000..a1e4b5765c2 --- /dev/null +++ b/lib/gitlab/cleanup/personal_access_tokens.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class PersonalAccessTokens + # By default tokens that haven't been used for over 1 year will be revoked + DEFAULT_TIME_PERIOD = 1.year + # To prevent inadvertently revoking all tokens, we provide a minimum time + MINIMUM_TIME_PERIOD = 1.day + + attr_reader :logger, :cut_off_date, :revocation_time, :group + + def initialize(cut_off_date: DEFAULT_TIME_PERIOD.ago.beginning_of_day, logger: nil, group_full_path:) + @cut_off_date = cut_off_date + + # rubocop: disable CodeReuse/ActiveRecord + @group = Group.find_by_full_path(group_full_path) + # rubocop: enable CodeReuse/ActiveRecord + + raise "Group with full_path #{group_full_path} not found" unless @group + raise "Invalid time: #{@cut_off_date}" unless @cut_off_date <= MINIMUM_TIME_PERIOD.ago + + # Use a static revocation time to make correlation of revoked + # tokens easier, should it be needed. + @revocation_time = Time.current.utc + @logger = logger || Gitlab::AppJsonLogger + + raise "Invalid logger: #{@logger}" unless @logger.respond_to?(:info) && @logger.respond_to?(:warn) + end + + def run!(dry_run: true, revoke_active_tokens: false) + # rubocop:disable Rails/Output + if dry_run + puts "Dry running. No changes will be made" + elsif revoke_active_tokens + puts "Revoking used and unused access tokens created before #{cut_off_date}..." + else + puts "Revoking access tokens last used and created before #{cut_off_date}..." + end + # rubocop:enable Rails/Output + + tokens_to_revoke = revocable_tokens(revoke_active_tokens) + + # rubocop:disable Cop/InBatches + tokens_to_revoke.in_batches do |access_tokens| + revoke_batch(access_tokens, dry_run) + end + # rubocop:enable Cop/InBatches + end + + private + + def revocable_tokens(revoke_active_tokens) + if revoke_active_tokens + PersonalAccessToken + .active + .owner_is_human + .created_before(cut_off_date) + .for_users(group.users) + else + PersonalAccessToken + .active + .owner_is_human + .last_used_before_or_unused(cut_off_date) + .for_users(group.users) + end + end + + def revoke_batch(access_tokens, dry_run) + # Capture a simplified set of attributes for logging and for + # determining when an error has led some records to not be + # updated + attrs = access_tokens.as_json(only: [:id, :user_id]) + + # Use `update_all` to bypass any validations which might + # prevent revocation. Manually specify updated_at. + affected_row_count = dry_run ? 0 : access_tokens.update_all(revoked: true, updated_at: @revocation_time) + + message = { + dry_run: dry_run, + message: "Revoke token batch", + token_count: attrs.size, + updated_count: affected_row_count, + tokens: attrs, + group_full_path: group.full_path + } + + # rubocop:disable Rails/Output + if dry_run + puts "Dry run complete. #{attrs.size} rows would be affected" + logger.info(message) + elsif affected_row_count.eql?(attrs.size) + puts "Finished. #{attrs.size} rows affected" + logger.info(message) + else + # :nocov: + puts "ERROR. #{affected_row_count} tokens deleted, #{attrs.size} tokens should have been deleted" + logger.warn(message) + # :nocov: + end + # rubocop:enable Rails/Output + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 8e624215065..7104de2a3c3 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -17,7 +17,6 @@ module Gitlab def closed_by_message(message) return [] if message.nil? - return [] unless @project.autoclose_referenced_issues closing_statements = [] message.scan(ISSUE_CLOSING_REGEX) do @@ -27,8 +26,9 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) @extractor.issues.reject do |issue| - # Don't extract issues from the project this project was forked from - @extractor.project.forked_from?(issue.project) + @extractor.project.forked_from?(issue.project) || + !issue.project.autoclose_referenced_issues || + !issue.project.issues_enabled? end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index e423d1f17da..be08ada9d2f 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -4,6 +4,11 @@ require_relative '../utils' # Gitlab::Utils module Gitlab module Cluster + # We take advantage of the fact that the application is pre-loaded in the primary + # process. If it's a pre-fork server like Puma, this will be the Puma master process. + # Otherwise it is the worker itself such as for Sidekiq. + PRIMARY_PID = $$ + # # LifecycleEvents lets Rails initializers register application startup hooks # that are sensitive to forking. For example, to defer the creation of diff --git a/lib/gitlab/config/entry/composable_hash.rb b/lib/gitlab/config/entry/composable_hash.rb index 9531b7e56fd..0b892fd4552 100644 --- a/lib/gitlab/config/entry/composable_hash.rb +++ b/lib/gitlab/config/entry/composable_hash.rb @@ -25,9 +25,9 @@ module Gitlab entry_class_name = entry_class.name.demodulize.underscore factory = ::Gitlab::Config::Entry::Factory.new(entry_class) - .value(config || {}) + .value(config.nil? ? {} : config) .with(key: name, parent: self, description: "#{name} #{entry_class_name} definition") # rubocop:disable CodeReuse/ActiveRecord - .metadata(name: name) + .metadata(composable_metadata.merge(name: name)) @entries[name] = factory.create! end @@ -38,9 +38,15 @@ module Gitlab end end + private + def composable_class(name, config) opt(:composable_class) end + + def composable_metadata + {} + end end end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index cc24ae837f3..337cfbc5287 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -304,6 +304,7 @@ module Gitlab end end + # This will be removed with the FF `ci_variables_refactoring_to_variable`. class VariablesValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -336,6 +337,18 @@ module Gitlab end end + class AlphanumericValidator < ActiveModel::EachValidator + def self.validate(value) + value.is_a?(String) || value.is_a?(Symbol) || value.is_a?(Integer) + end + + def validate_each(record, attribute, value) + unless self.class.validate(value) + record.errors.add(attribute, 'must be an alphanumeric string') + end + end + end + class ExpressionValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid? diff --git a/lib/gitlab/container_repository/tags/cache.rb b/lib/gitlab/container_repository/tags/cache.rb index ff457fb9219..47a6e67a5a1 100644 --- a/lib/gitlab/container_repository/tags/cache.rb +++ b/lib/gitlab/container_repository/tags/cache.rb @@ -48,14 +48,14 @@ module Gitlab ::Gitlab::Redis::Cache.with do |redis| # we use a pipeline instead of a MSET because each tag has # a specific ttl - redis.pipelined do + redis.pipelined do |pipeline| cacheable_tags.each do |tag| created_at = tag.created_at # ttl is the max_ttl_in_seconds reduced by the number # of seconds that the tag has already existed ttl = max_ttl_in_seconds - (now - created_at).seconds ttl = ttl.to_i - redis.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0 + pipeline.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0 end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 91e6fc11a53..4640f85bb0a 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -24,7 +24,7 @@ module Gitlab # Leaving this way to have backward compatibility build_id: build.id, build_name: build.name, - build_stage: build.stage, + build_stage: build.stage_name, build_status: build.status, build_created_at: build.created_at, build_started_at: build.started_at, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 2c124b07006..320ebe5e80f 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -52,7 +52,8 @@ module Gitlab runner: :tags, job_artifacts_archive: [], user: [], - metadata: [] + metadata: [], + ci_stage: [] } } ) @@ -110,7 +111,7 @@ module Gitlab def build_hook_attrs(build) { id: build.id, - stage: build.stage, + stage: build.stage_name, name: build.name, status: build.status, created_at: build.created_at, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8703365b678..dd84127459d 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -242,7 +242,8 @@ module Gitlab # in such cases it is fine to ignore such connections return unless db_config - primary_model = self.database_base_models.fetch(db_config.name.to_sym) + db_config_name = db_config.name.delete_suffix(LoadBalancing::LoadBalancer::REPLICA_SUFFIX) + primary_model = self.database_base_models.fetch(db_config_name.to_sym) self.schemas_to_base_models.select do |_, child_models| child_models.any? do |child_model| diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 6aed1eed994..45f52765d0f 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -8,6 +8,7 @@ module Gitlab BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" MAXIMUM_FAILED_RATIO = 0.5 MINIMUM_JOBS = 50 + FINISHED_PROGRESS_VALUE = 100 self.table_name = :batched_background_migrations @@ -24,6 +25,7 @@ module Gitlab scope :queue_order, -> { order(id: :asc) } scope :queued, -> { with_statuses(:active, :paused) } + scope :ordered_by_created_at_desc, -> { order(created_at: :desc) } # on_hold_until is a temporary runtime status which puts execution "on hold" scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') } @@ -57,11 +59,11 @@ module Gitlab state :finalizing, value: 5 event :pause do - transition any => :paused + transition [:active, :paused] => :paused end event :execute do - transition any => :active + transition [:active, :paused, :failed] => :active end event :finish do @@ -231,7 +233,15 @@ module Gitlab "BatchedMigration[id: #{id}]" end + # Computes an estimation of the progress of the migration in percents. + # + # Because `total_tuple_count` is an estimation of the tuples based on DB statistics + # when the migration is complete there can actually be more or less tuples that initially + # estimated as `total_tuple_count` so the progress may not show 100%. For that reason when + # we know migration completed successfully, we just return the 100 value def progress + return FINISHED_PROGRESS_VALUE if finished? + return unless total_tuple_count.to_i > 0 100 * migrated_tuple_count / total_tuple_count diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb index 9a283074b32..506d2996ad5 100644 --- a/lib/gitlab/database/background_migration/health_status.rb +++ b/lib/gitlab/database/background_migration/health_status.rb @@ -18,7 +18,7 @@ module Gitlab indicator.new(migration.health_context).evaluate rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, - job_class_name: migration.job_class_name) + job_class_name: migration.job_class_name) Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") end diff --git a/lib/gitlab/database/batch_average_counter.rb b/lib/gitlab/database/batch_average_counter.rb new file mode 100644 index 00000000000..9cb1e34ab67 --- /dev/null +++ b/lib/gitlab/database/batch_average_counter.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class BatchAverageCounter + COLUMN_FALLBACK = 0 + DEFAULT_BATCH_SIZE = 1_000 + FALLBACK = -1 + MAX_ALLOWED_LOOPS = 10_000 + OFFSET_BY_ONE = 1 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + + attr_reader :relation, :column + + def initialize(relation, column) + @relation = relation + @column = wrap_column(relation, column) + end + + def count(batch_size: nil) + raise 'BatchAverageCounter can not be run inside a transaction' if transaction_open? + + batch_size = batch_size.presence || DEFAULT_BATCH_SIZE + + start = column_start + finish = column_finish + + total_sum = 0 + total_records = 0 + + batch_start = start + + while batch_start < finish + begin + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end) + + # We use `sum` and `count` instead of `average` here to not run into an "average of averages" + # problem as batches will have different sizes, so we are essentially summing up the values for + # each batch separately, and then dividing that result on the total number of records. + batch_sum, batch_count = batch_relation.pick(column.sum, column.count) + + total_sum += batch_sum.to_i + total_records += batch_count + + batch_start = batch_end + rescue ActiveRecord::QueryCanceled => error # rubocop:disable Database/RescueQueryCanceled + # retry with a safe batch size & warmer cache + if batch_size >= 2 * DEFAULT_BATCH_SIZE + batch_size /= 2 + else + log_canceled_batch_fetch(batch_start, batch_relation.to_sql, error) + + return FALLBACK + end + end + + sleep(SLEEP_TIME_IN_SECONDS) + end + + return FALLBACK if total_records == 0 + + total_sum.to_f / total_records + end + + private + + def column_start + relation.unscope(:group, :having).minimum(column) || COLUMN_FALLBACK + end + + def column_finish + (relation.unscope(:group, :having).maximum(column) || COLUMN_FALLBACK) + OFFSET_BY_ONE + end + + def build_relation_batch(start, finish) + relation.where(column.between(start...finish)) + end + + def log_canceled_batch_fetch(batch_start, query, error) + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: relation.table_name, + operation: 'average', + start: batch_start, + query: query, + message: "Query has been canceled with message: #{error.message}" + ) + end + + def transaction_open? + relation.connection.transaction_open? + end + + def wrap_column(relation, column) + return column if column.is_a?(Arel::Attributes::Attribute) + + relation.arel_table[column] + end + end + end +end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 92a41bb36ee..7a064fb4005 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -35,6 +35,10 @@ module Gitlab BatchCounter.new(relation, column: column).count(batch_size: batch_size, start: start, finish: finish) end + def batch_count_with_timeout(relation, column = nil, batch_size: nil, start: nil, finish: nil, timeout: nil, partial_results: nil) + BatchCounter.new(relation, column: column).count_with_timeout(batch_size: batch_size, start: start, finish: finish, timeout: timeout, partial_results: partial_results) + end + def batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size, start: start, finish: finish) end @@ -44,7 +48,7 @@ module Gitlab end def batch_average(relation, column, batch_size: nil, start: nil, finish: nil) - BatchCounter.new(relation, column: nil, operation: :average, operation_args: [column]).count(batch_size: batch_size, start: start, finish: finish) + BatchAverageCounter.new(relation, column).count(batch_size: batch_size) end class << self diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 522b598cd9d..abb62140503 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -6,7 +6,6 @@ module Gitlab FALLBACK = -1 MIN_REQUIRED_BATCH_SIZE = 1_250 DEFAULT_SUM_BATCH_SIZE = 1_000 - DEFAULT_AVERAGE_BATCH_SIZE = 1_000 MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep ALLOWED_MODES = [:itself, :distinct].freeze @@ -27,12 +26,19 @@ module Gitlab def unwanted_configuration?(finish, batch_size, start) (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || - (@operation == :average && batch_size < DEFAULT_AVERAGE_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || start >= finish end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) + result = count_with_timeout(batch_size: batch_size, mode: mode, start: start, finish: finish, timeout: nil) + + return FALLBACK if result[:status] != :completed + + result[:count] + end + + def count_with_timeout(batch_size: nil, mode: :itself, start: nil, finish: nil, timeout: nil, partial_results: nil) raise 'BatchCount can not be run inside a transaction' if transaction_open? check_mode!(mode) @@ -44,12 +50,20 @@ module Gitlab finish = actual_finish(finish) raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) + return { status: :bad_config } if unwanted_configuration?(finish, batch_size, start) - results = nil + results = partial_results batch_start = start + start_time = ::Gitlab::Metrics::System.monotonic_time.seconds + while batch_start < finish + + # Timeout elapsed, return partial result so the caller can continue later + if timeout && ::Gitlab::Metrics::System.monotonic_time.seconds - start_time > timeout + return { status: :timeout, partial_results: results, continue_from: batch_start } + end + begin batch_end = [batch_start + batch_size, finish].min batch_relation = build_relation_batch(batch_start, batch_end, mode) @@ -62,14 +76,14 @@ module Gitlab batch_size /= 2 else log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) - return FALLBACK + return { status: :cancelled } end end sleep(SLEEP_TIME_IN_SECONDS) end - results + { status: :completed, count: results } end def transaction_open? @@ -94,7 +108,6 @@ module Gitlab def batch_size_for_mode_and_operation(mode, operation) return DEFAULT_SUM_BATCH_SIZE if operation == :sum - return DEFAULT_AVERAGE_BATCH_SIZE if operation == :average mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE end @@ -132,10 +145,6 @@ module Gitlab message: "Query has been canceled with message: #{error.message}" ) end - - def not_group_by_query? - !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? - end end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index d05eee7d6e6..5725d7a4503 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -91,6 +91,7 @@ ci_job_artifact_states: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci ci_namespace_mirrors: :gitlab_ci +ci_partitions: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci @@ -182,6 +183,7 @@ design_management_versions: :gitlab_main design_user_mentions: :gitlab_main detached_partitions: :gitlab_shared diff_note_positions: :gitlab_main +dora_configurations: :gitlab_main dora_daily_metrics: :gitlab_main draft_notes: :gitlab_main elastic_index_settings: :gitlab_main @@ -228,6 +230,7 @@ geo_repository_deleted_events: :gitlab_main geo_repository_renamed_events: :gitlab_main geo_repository_updated_events: :gitlab_main geo_reset_checksum_events: :gitlab_main +ghost_user_migrations: :gitlab_main gitlab_subscription_histories: :gitlab_main gitlab_subscriptions: :gitlab_main gpg_keys: :gitlab_main @@ -315,6 +318,7 @@ merge_request_diff_details: :gitlab_main merge_request_diff_files: :gitlab_main merge_request_diffs: :gitlab_main merge_request_metrics: :gitlab_main +merge_request_predictions: :gitlab_main merge_request_reviewers: :gitlab_main merge_requests_closing_issues: :gitlab_main merge_requests: :gitlab_main @@ -380,6 +384,7 @@ packages_events: :gitlab_main packages_helm_file_metadata: :gitlab_main packages_maven_metadata: :gitlab_main packages_npm_metadata: :gitlab_main +packages_rpm_metadata: :gitlab_main packages_nuget_dependency_link_metadata: :gitlab_main packages_nuget_metadata: :gitlab_main packages_package_file_build_infos: :gitlab_main @@ -399,6 +404,7 @@ plans: :gitlab_main pool_repositories: :gitlab_main postgres_async_indexes: :gitlab_shared postgres_autovacuum_activity: :gitlab_shared +postgres_constraints: :gitlab_shared postgres_foreign_keys: :gitlab_shared postgres_index_bloat_estimates: :gitlab_shared postgres_indexes: :gitlab_shared @@ -479,6 +485,7 @@ sbom_components: :gitlab_main sbom_occurrences: :gitlab_main sbom_component_versions: :gitlab_main sbom_sources: :gitlab_main +sbom_vulnerable_component_versions: :gitlab_main schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main @@ -549,6 +556,7 @@ user_statuses: :gitlab_main user_synced_attributes_metadata: :gitlab_main verification_codes: :gitlab_main vulnerabilities: :gitlab_main +vulnerability_advisories: :gitlab_main vulnerability_exports: :gitlab_main vulnerability_external_issue_links: :gitlab_main vulnerability_feedback: :gitlab_main diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index cd483d616bb..fe75cd763b4 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -5,42 +5,63 @@ module Gitlab class LockWritesManager TRIGGER_FUNCTION_NAME = 'gitlab_schema_prevent_write' - def initialize(table_name:, connection:, database_name:, logger: nil) + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger + @dry_run = dry_run + end + + def table_locked_for_writes?(table_name) + query = <<~SQL + SELECT COUNT(*) from information_schema.triggers + WHERE event_object_table = '#{table_name}' + AND trigger_name = '#{write_trigger_name(table_name)}' + SQL + + connection.select_value(query) == 3 end def lock_writes + if table_locked_for_writes?(table_name) + logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes" + return + end + logger&.info "Database: '#{database_name}', Table: '#{table_name}': Lock Writes".color(:yellow) - sql = <<-SQL - DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; + sql_statement = <<~SQL CREATE TRIGGER #{write_trigger_name(table_name)} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON #{table_name} FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); SQL - with_retries(connection) do - connection.execute(sql) - end + execute_sql_statement(sql_statement) end def unlock_writes logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green) - sql = <<-SQL - DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name} + sql_statement = <<~SQL + DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; SQL - with_retries(connection) do - connection.execute(sql) - end + execute_sql_statement(sql_statement) end private - attr_reader :table_name, :connection, :database_name, :logger + attr_reader :table_name, :connection, :database_name, :logger, :dry_run + + def execute_sql_statement(sql) + if dry_run + logger&.info sql + else + with_retries(connection) do + connection.execute(sql) + end + end + end def with_retries(connection, &block) with_statement_timeout_retries do diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index db39524f4f6..e574422ce11 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -936,13 +936,14 @@ module Gitlab def revert_backfill_conversion_of_integer_to_bigint(table, columns, primary_key: :id) columns = Array.wrap(columns) - conditions = ActiveRecord::Base.sanitize_sql([ - 'job_class_name = :job_class_name AND table_name = :table_name AND column_name = :column_name AND job_arguments = :job_arguments', - job_class_name: 'CopyColumnUsingBackgroundMigrationJob', - table_name: table, - column_name: primary_key, - job_arguments: [columns, columns.map { |column| convert_to_bigint_column(column) }].to_json - ]) + conditions = ActiveRecord::Base.sanitize_sql( + [ + 'job_class_name = :job_class_name AND table_name = :table_name AND column_name = :column_name AND job_arguments = :job_arguments', + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: table, + column_name: primary_key, + job_arguments: [columns, columns.map { |column| convert_to_bigint_column(column) }].to_json + ]) execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index a9440cafd30..76982a9da9b 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -40,7 +40,7 @@ module Gitlab instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) batch_names = (1..).each.lazy.map { |i| "batch_#{i}" } - jobs.shuffle.each do |j| + jobs.each do |j| break if run_until <= Time.current instrumentation.observe(version: nil, diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index f7713237b38..6da2e098d43 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -15,6 +15,7 @@ module Gitlab def jobs_by_migration_name traditional_background_migrations.group_by { |j| class_name_for_job(j) } + .transform_values(&:shuffle) end private diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index f38d847b0e8..c27ae6a2c5d 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -4,6 +4,7 @@ module Gitlab module Database module Migrations class TestBatchedBackgroundRunner < BaseBackgroundRunner + include Gitlab::Database::DynamicModelHelpers attr_reader :connection def initialize(result_dir:, connection:) @@ -18,31 +19,81 @@ module Gitlab .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) - all_migration_jobs = [] + smallest_batch_start = migration.next_min_value - min_value = migration.next_min_value + table_max_value = define_batchable_model(migration.table_name, connection: connection) + .maximum(migration.column_name) - while (next_bounds = batching_strategy.next_batch( - migration.table_name, - migration.column_name, - batch_min_value: min_value, - batch_size: migration.batch_size, - job_arguments: migration.job_arguments - )) + largest_batch_start = table_max_value - migration.batch_size + + # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 + # to pick actual batches to sample. + variance = largest_batch_start - smallest_batch_start + + batch_starts = uniform_fractions + .lazy # frac varies from 0 to 1, values in smallest_batch_start..largest_batch_start + .map { |frac| (variance * frac).to_i + smallest_batch_start } + + # Track previously run batches so that we stop sampling if a new batch would intersect an older one + completed_batches = [] + + jobs_to_sample = batch_starts + # Stop sampling if a batch would intersect a previous batch + .take_while { |start| completed_batches.none? { |batch| batch.cover?(start) } } + .map do |batch_start| + next_bounds = batching_strategy.next_batch( + migration.table_name, + migration.column_name, + batch_min_value: batch_start, + batch_size: migration.batch_size, + job_arguments: migration.job_arguments + ) batch_min, batch_max = next_bounds - all_migration_jobs << migration.create_batched_job!(batch_min, batch_max) - min_value = batch_max + 1 + job = migration.create_batched_job!(batch_min, batch_max) + + completed_batches << (batch_min..batch_max) + + job end - [migration.job_class_name, all_migration_jobs] + [migration.job_class_name, jobs_to_sample] end end def run_job(job) Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) end + + def uniform_fractions + Enumerator.new do |y| + # Generates equally distributed fractions between 0 and 1, with increasing detail as more are pulled from + # the enumerator. + # 0, 1 (special case) + # 1/2 + # 1/4, 3/4 + # 1/8, 3/8, 5/8, 7/8 + # etc. + # The pattern here is at each outer loop, the denominator multiplies by 2, and at each inner loop, + # the numerator counts up all odd numbers 1 <= n < denominator. + y << 0 + y << 1 + + # denominators are each increasing power of 2 + denominators = (1..).lazy.map { |exponent| 2**exponent } + + denominators.each do |denominator| + # Numerators at the current step are all odd numbers between 1 and the denominator + numerators = (1..denominator).step(2) + + numerators.each do |numerator| + next_frac = numerator.fdiv(denominator) + y << next_frac + end + end + end + end end end end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 92825d41599..6314aff9914 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -33,6 +33,18 @@ module Gitlab PartitionManager.new(model).sync_partitions end + unless only_on + models_to_sync.each do |model| + next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) + + Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + if connection_name != model.connection_db_config.name + PartitionManager.new(model, connection: connection).sync_partitions + end + end + end + end + Gitlab::AppLogger.info(message: 'Finished sync of dynamic postgres partitions') end diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb new file mode 100644 index 00000000000..f45cf02ec9b --- /dev/null +++ b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class ConvertTableToFirstListPartition + UnableToPartition = Class.new(StandardError) + + include Gitlab::Database::MigrationHelpers + + SQL_STATEMENT_SEPARATOR = ";\n\n" + + attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value + + def initialize(migration_context:, table_name:, parent_table_name:, partitioning_column:, zero_partition_value:) + @migration_context = migration_context + @connection = migration_context.connection + @table_name = table_name + @parent_table_name = parent_table_name + @partitioning_column = partitioning_column + @zero_partition_value = zero_partition_value + end + + def prepare_for_partitioning + assert_existing_constraints_partitionable + + add_partitioning_check_constraint + end + + def revert_preparation_for_partitioning + migration_context.remove_check_constraint(table_name, partitioning_constraint.name) + end + + def partition + assert_existing_constraints_partitionable + assert_partitioning_constraint_present + create_parent_table + attach_foreign_keys_to_parent + + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.execute(sql_to_convert_table) + end + end + + def revert_partitioning + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.execute(<<~SQL) + ALTER TABLE #{connection.quote_table_name(parent_table_name)} + DETACH PARTITION #{connection.quote_table_name(table_name)}; + SQL + + alter_sequences_sql = alter_sequence_statements(old_table: parent_table_name, new_table: table_name) + .join(SQL_STATEMENT_SEPARATOR) + + migration_context.execute(alter_sequences_sql) + + # This takes locks for all the foreign keys that the parent table had. + # However, those same locks were taken while detaching the partition, and we can't avoid that. + # If we dropped the foreign key before detaching the partition to avoid this locking, + # the drop would cascade to the child partitions and drop their foreign keys as well + migration_context.drop_table(parent_table_name) + end + + add_partitioning_check_constraint + end + + private + + attr_reader :connection, :migration_context + + delegate :quote_table_name, :quote_column_name, to: :connection + + def sql_to_convert_table + # The critical statement here is the attach_table_to_parent statement. + # The following statements could be run in a later transaction, + # but they acquire the same locks so it's much faster to incude them + # here. + [ + attach_table_to_parent_statement, + alter_sequence_statements(old_table: table_name, new_table: parent_table_name), + remove_constraint_statement + ].flatten.join(SQL_STATEMENT_SEPARATOR) + end + + def table_identifier + "#{connection.current_schema}.#{table_name}" + end + + def assert_existing_constraints_partitionable + violating_constraints = Gitlab::Database::PostgresConstraint + .by_table_identifier(table_identifier) + .primary_or_unique_constraints + .not_including_column(partitioning_column) + .to_a + + return if violating_constraints.empty? + + violation_messages = violating_constraints.map { |c| "#{c.name} on (#{c.column_names.join(', ')})" } + + raise UnableToPartition, <<~MSG + Constraints on #{table_name} are incompatible with partitioning on #{partitioning_column} + + All primary key and unique constraints must include the partitioning column. + Violations: + #{violation_messages.join("\n")} + MSG + end + + def partitioning_constraint + constraints_on_column = Gitlab::Database::PostgresConstraint + .by_table_identifier(table_identifier) + .check_constraints + .valid + .including_column(partitioning_column) + + constraints_on_column.to_a.find do |constraint| + constraint.definition == "CHECK ((#{partitioning_column} = #{zero_partition_value}))" + end + end + + def assert_partitioning_constraint_present + return if partitioning_constraint + + raise UnableToPartition, <<~MSG + Table #{table_name} is not ready for partitioning. + Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value}) + MSG + end + + def add_partitioning_check_constraint + return if partitioning_constraint.present? + + check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}" + # Any constraint name would work. The constraint is found based on its definition before partitioning + migration_context.add_check_constraint(table_name, check_body, 'partitioning_constraint') + + raise UnableToPartition, 'Error adding partitioning constraint' unless partitioning_constraint.present? + end + + def create_parent_table + migration_context.execute(<<~SQL) + CREATE TABLE IF NOT EXISTS #{quote_table_name(parent_table_name)} ( + LIKE #{quote_table_name(table_name)} INCLUDING ALL + ) PARTITION BY LIST(#{quote_column_name(partitioning_column)}) + SQL + end + + def attach_foreign_keys_to_parent + migration_context.foreign_keys(table_name).each do |fk| + # At this point no other connection knows about the parent table. + # Thus the only contended lock in the following transaction is on fk.to_table. + # So a deadlock is impossible. + + # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist. + # Don't try to recreate it in that case + if migration_context.foreign_keys(parent_table_name) + .any? { |p_fk| p_fk.options[:name] == fk.options[:name] } + next + end + + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.add_foreign_key(parent_table_name, fk.to_table, **fk.options) + end + end + end + + def attach_table_to_parent_statement + <<~SQL + ALTER TABLE #{quote_table_name(parent_table_name)} + ATTACH PARTITION #{table_name} + FOR VALUES IN (#{zero_partition_value}) + SQL + end + + def alter_sequence_statements(old_table:, new_table:) + sequences_owned_by(old_table).map do |seq_info| + seq_name, column_name = seq_info.values_at(:name, :column_name) + <<~SQL.chomp + ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)} + SQL + end + end + + def remove_constraint_statement + <<~SQL + ALTER TABLE #{quote_table_name(parent_table_name)} + DROP CONSTRAINT #{quote_table_name(partitioning_constraint.name)} + SQL + end + + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/373887 + def sequences_owned_by(table_name) + sequence_data = connection.exec_query(<<~SQL, nil, [table_name]) + SELECT seq_pg_class.relname AS seq_name, + dep_pg_class.relname AS table_name, + pg_attribute.attname AS col_name + FROM pg_class seq_pg_class + INNER JOIN pg_depend ON seq_pg_class.oid = pg_depend.objid + INNER JOIN pg_class dep_pg_class ON pg_depend.refobjid = dep_pg_class.oid + INNER JOIN pg_attribute ON dep_pg_class.oid = pg_attribute.attrelid + AND pg_depend.refobjsubid = pg_attribute.attnum + WHERE seq_pg_class.relkind = 'S' + AND dep_pg_class.relname = $1 + SQL + + sequence_data.map do |seq_info| + name, column_name = seq_info.values_at('seq_name', 'col_name') + { name: name, column_name: column_name } + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index aac91eaadb1..55ca9ff8645 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -10,12 +10,15 @@ module Gitlab MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' RETAIN_DETACHED_PARTITIONS_FOR = 1.week - def initialize(model) + def initialize(model, connection: nil) @model = model - @connection_name = model.connection.pool.db_config.name + @connection = connection || model.connection + @connection_name = @connection.pool.db_config.name end def sync_partitions + return skip_synching_partitions unless table_partitioned? + Gitlab::AppLogger.info( message: "Checking state of dynamic postgres partitions", table_name: model.table_name, @@ -43,9 +46,7 @@ module Gitlab private - attr_reader :model - - delegate :connection, to: :model + attr_reader :model, :connection def missing_partitions return [] unless connection.table_exists?(model.table_name) @@ -129,6 +130,20 @@ module Gitlab connection: connection ).run(&block) end + + def table_partitioned? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(model.table_name).present? + end + end + + def skip_synching_partitions + Gitlab::AppLogger.warn( + message: "Skipping synching partitions", + table_name: model.table_name, + connection_name: @connection_name + ) + end end end end diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb index 23ac73a0e53..4e38eea963b 100644 --- a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -8,7 +8,7 @@ module Gitlab def self.from_sql(table, partition_name, definition) # A list partition can support multiple values, but we only support a single number - matches = definition.match(/FOR VALUES IN \('(?<value>\d+)'\)/) + matches = definition.match(/FOR VALUES IN \('?(?<value>\d+)'?\)/) raise ArgumentError, 'Unknown partition definition' unless matches @@ -29,17 +29,21 @@ module Gitlab @partition_name || "#{table}_#{value}" end + def data_size + execute("SELECT pg_table_size(#{quote(full_partition_name)})").first['pg_table_size'] + end + def to_sql <<~SQL CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} - PARTITION OF #{conn.quote_table_name(table)} - FOR VALUES IN (#{conn.quote(value)}) + PARTITION OF #{quote_table_name(table)} + FOR VALUES IN (#{quote(value)}) SQL end def to_detach_sql <<~SQL - ALTER TABLE #{conn.quote_table_name(table)} + ALTER TABLE #{quote_table_name(table)} DETACH PARTITION #{fully_qualified_partition} SQL end @@ -63,8 +67,14 @@ module Gitlab private + delegate :execute, :quote, :quote_table_name, to: :conn, private: true + + def full_partition_name + "%s.%s" % [Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA, partition_name] + end + def fully_qualified_partition - "%s.%s" % [conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), conn.quote_table_name(partition_name)] + quote_table_name(full_partition_name) end def conn diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 4b5349f0327..5bb34a86d43 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -14,7 +14,7 @@ module Gitlab @next_partition_if = next_partition_if @detach_partition_if = detach_partition_if - ensure_partitioning_column_ignored! + ensure_partitioning_column_ignored_or_readonly! end def current_partitions @@ -26,7 +26,7 @@ module Gitlab def missing_partitions if no_partitions_exist? [initial_partition] - elsif next_partition_if.call(active_partition.value) + elsif next_partition_if.call(active_partition) [next_partition] else [] @@ -44,7 +44,7 @@ module Gitlab def extra_partitions possibly_extra = current_partitions[0...-1] # Never consider the most recent partition - extra = possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + extra = possibly_extra.take_while { |p| detach_partition_if.call(p) } default_value = current_default_value if extra.any? { |p| p.value == default_value } @@ -128,12 +128,17 @@ module Gitlab Integer(value) end - def ensure_partitioning_column_ignored! - unless model.ignored_columns.include?(partitioning_key.to_s) - raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" + def ensure_partitioning_column_ignored_or_readonly! + unless key_ignored_or_readonly? + raise "Add #{partitioning_key} to #{model.name}.ignored_columns or " \ + "mark it as readonly to use it with SlidingListStrategy" end end + def key_ignored_or_readonly? + model.ignored_columns.include?(partitioning_key.to_s) || model.readonly_attribute?(partitioning_key.to_s) + end + def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( klass: self.class, diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index c9a3b5caf79..15b542cf089 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -77,8 +77,42 @@ module Gitlab end end + # Finds duplicate indexes for a given schema and table. This finds + # indexes where the index definition is identical but the names are + # different. Returns an array of arrays containing duplicate index name + # pairs. + # + # Example: + # + # find_duplicate_indexes('table_name_goes_here') + def find_duplicate_indexes(table_name, schema_name: connection.current_schema) + find_indexes(table_name, schema_name: schema_name) + .group_by { |r| r['index_id'] } + .select { |_, v| v.size > 1 } + .map { |_, indexes| indexes.map { |index| index['index_name'] } } + end + private + def find_indexes(table_name, schema_name: connection.current_schema) + indexes = connection.select_all(<<~SQL, 'SQL', [schema_name, table_name]) + SELECT n.nspname AS schema_name, + c.relname AS table_name, + i.relname AS index_name, + regexp_replace(pg_get_indexdef(i.oid), 'INDEX .*? USING', '_') AS index_id + FROM pg_index x + JOIN pg_class c ON c.oid = x.indrelid + JOIN pg_class i ON i.oid = x.indexrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE (c.relkind = ANY (ARRAY['r'::"char", 'm'::"char", 'p'::"char"])) + AND (i.relkind = ANY (ARRAY['i'::"char", 'I'::"char"])) + AND n.nspname = $1 + AND c.relname = $2; + SQL + + indexes.to_a + end + def find_partitioned_table(table_name) partitioned_table = Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name) diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index a541ecf5316..695a5d7ec77 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -251,6 +251,54 @@ module Gitlab create_sync_trigger(source_table_name, trigger_name, function_name) end + def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:prepare_constraint_for_list_partitioning) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).prepare_for_partitioning + end + + def revert_preparing_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:revert_preparing_constraint_for_list_partitioning) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).revert_preparation_for_partitioning + end + + def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:convert_table_to_first_list_partition) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).partition + end + + def revert_converting_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:revert_converting_table_to_first_list_partition) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).revert_partitioning + end + private def assert_table_is_allowed(table_name) diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb new file mode 100644 index 00000000000..fa590914332 --- /dev/null +++ b/lib/gitlab/database/postgres_constraint.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Backed by the postgres_constraints view + class PostgresConstraint < SharedModel + IDENTIFIER_REGEX = /^\w+\.\w+$/.freeze + self.primary_key = :oid + + scope :check_constraints, -> { where(constraint_type: 'c') } + scope :primary_key_constraints, -> { where(constraint_type: 'p') } + scope :unique_constraints, -> { where(constraint_type: 'u') } + scope :primary_or_unique_constraints, -> { where(constraint_type: %w[u p]) } + + scope :including_column, ->(column) { where("? = ANY(column_names)", column) } + scope :not_including_column, ->(column) { where.not("? = ANY(column_names)", column) } + + scope :valid, -> { where(constraint_valid: true) } + + scope :by_table_identifier, ->(identifier) do + unless identifier =~ IDENTIFIER_REGEX + raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" + end + + where(table_identifier: identifier) + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb new file mode 100644 index 00000000000..c2d5dfc1a15 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + module Ci + # The purpose of this analyzer is to detect queries not going through a partitioning routing table + class PartitioningAnalyzer < Database::QueryAnalyzers::Base + RoutingTableNotUsedError = Class.new(QueryAnalyzerError) + + ENABLED_TABLES = %w[ + ci_builds_metadata + ].freeze + + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + ::Feature.enabled?(:ci_partitioning_analyze_queries, type: :ops) + end + + def analyze(parsed) + analyze_legacy_tables_usage(parsed) + end + + private + + def analyze_legacy_tables_usage(parsed) + detected = ENABLED_TABLES & (parsed.pg.dml_tables + parsed.pg.select_tables) + + return if detected.none? + + ::Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + RoutingTableNotUsedError.new("Detected non-partitioned table use #{detected.inspect}: #{parsed.sql}") + ) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index 06e2b114c91..b4b9161f0c2 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -14,7 +14,7 @@ module Gitlab class << self def enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:query_analyzer_gitlab_schema_metrics) + Feature.enabled?(:query_analyzer_gitlab_schema_metrics, type: :ops) end def analyze(parsed) diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index e0cb803b872..3b1751c863d 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -33,7 +33,7 @@ module Gitlab def self.enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:detect_cross_database_modification) + Feature.enabled?(:detect_cross_database_modification, type: :ops) end def self.requires_tracking?(parsed) diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb index 3ea7277571f..33c965cb150 100644 --- a/lib/gitlab/database/reflection.rb +++ b/lib/gitlab/database/reflection.rb @@ -114,7 +114,7 @@ module Gitlab 'PostgreSQL on Amazon RDS' => { statement: 'SHOW rds.extensions', error: /PG::UndefinedObject/ }, # Based on https://cloud.google.com/sql/docs/postgres/flags#postgres-c this should be specific # to Cloud SQL for PostgreSQL - 'Cloud SQL for PostgreSQL' => { statement: 'SHOW cloudsql.iam_authentication', error: /PG::UndefinedObject/ }, + 'Cloud SQL for PostgreSQL' => { statement: 'SHOW cloudsql.iam_authentication', error: /PG::UndefinedObject/ }, # Based on # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-extensions diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index b96dffc99ac..aba45fcc57b 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -27,7 +27,7 @@ module Gitlab # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) - Gitlab::Database::AsyncIndexes.drop_pending_indexes! if Feature.enabled?(:database_async_index_destruction, type: :ops) + Gitlab::Database::AsyncIndexes.drop_pending_indexes! automatic_reindexing end diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb new file mode 100644 index 00000000000..9f096904d31 --- /dev/null +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TablesSortedByForeignKeys + include TSort + + def initialize(connection, tables) + @connection = connection + @tables = tables + end + + def execute + strongly_connected_components + end + + private + + def tsort_each_node(&block) + tables_dependencies.each_key(&block) + end + + def tsort_each_child(node, &block) + tables_dependencies[node].each(&block) + end + + # it maps the tables to the tables that depend on it + def tables_dependencies + @tables.to_h do |table_name| + [table_name, all_foreign_keys[table_name]&.map(&:from_table).to_a] + end + end + + def all_foreign_keys + @all_foreign_keys ||= @tables.flat_map do |table_name| + @connection.foreign_keys(table_name) + end.group_by(&:to_table) + end + end + end +end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb new file mode 100644 index 00000000000..164520fbab3 --- /dev/null +++ b/lib/gitlab/database/tables_truncate.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TablesTruncate + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + + def initialize(database_name:, min_batch_size:, logger: nil, until_table: nil, dry_run: false) + @database_name = database_name + @min_batch_size = min_batch_size + @logger = logger + @until_table = until_table + @dry_run = dry_run + end + + def execute + raise "Cannot truncate legacy tables in single-db setup" unless Gitlab::Database.has_config?(:ci) + raise "database is not supported" unless %w[main ci].include?(database_name) + + logger&.info "DRY RUN:" if dry_run + + connection = Gitlab::Database.database_base_models[database_name].connection + + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + tables_to_truncate = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| + (GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection)).include?(schema_name) + end.keys + + tables_sorted = Gitlab::Database::TablesSortedByForeignKeys.new(connection, tables_to_truncate).execute + # Checking if all the tables have the write-lock triggers + # to make sure we are deleting the right tables on the right database. + tables_sorted.flatten.each do |table_name| + query = <<~SQL + SELECT COUNT(*) from information_schema.triggers + WHERE event_object_table = '#{table_name}' + AND trigger_name = 'gitlab_schema_write_trigger_for_#{table_name}' + SQL + + if connection.select_value(query) == 0 + raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" + end + end + + if until_table + table_index = tables_sorted.find_index { |tables_group| tables_group.include?(until_table) } + raise "The table '#{until_table}' is not within the truncated tables" if table_index.nil? + + tables_sorted = tables_sorted[0..table_index] + end + + # min_batch_size is the minimum number of new tables to truncate at each stage. + # But in each stage we have also have to truncate the already truncated tables in the previous stages + logger&.info "Truncating legacy tables for the database #{database_name}" + truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + end + + private + + attr_accessor :database_name, :min_batch_size, :logger, :dry_run, :until_table + + def truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + truncated_tables = [] + + tables_sorted.flatten.each do |table| + sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + logger&.info(sql_statement) + connection.execute(sql_statement) unless dry_run + end + + # We do the truncation in stages to avoid high IO + # In each stage, we truncate the new tables along with the already truncated + # tables before. That's because PostgreSQL doesn't allow to truncate any table (A) + # without truncating any other table (B) that has a Foreign Key pointing to the table (A). + # even if table (B) is empty, because it has been already truncated in a previous stage. + tables_sorted.in_groups_of(min_batch_size, false).each do |tables_groups| + new_tables_to_truncate = tables_groups.flatten + logger&.info "= New tables to truncate: #{new_tables_to_truncate.join(', ')}" + truncated_tables.push(*new_tables_to_truncate).tap(&:sort!) + sql_statements = [ + "SET LOCAL statement_timeout = 0", + "SET LOCAL lock_timeout = 0", + "TRUNCATE TABLE #{truncated_tables.join(', ')} RESTRICT" + ] + + sql_statements.each { |sql_statement| logger&.info(sql_statement) } + + next if dry_run + + connection.transaction do + sql_statements.each { |sql_statement| connection.execute(sql_statement) } + end + end + end + end + end +end diff --git a/lib/gitlab/database_importers/security/training_providers/importer.rb b/lib/gitlab/database_importers/security/training_providers/importer.rb new file mode 100644 index 00000000000..aa6a9f29c6d --- /dev/null +++ b/lib/gitlab/database_importers/security/training_providers/importer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module Security + module TrainingProviders + module Importer + KONTRA_DATA = { + name: 'Kontra', + description: "Kontra Application Security provides interactive developer security education that + enables engineers to quickly learn security best practices + and fix issues in their code by analysing real-world software security vulnerabilities.", + url: "https://application.security/api/webhook/gitlab/exercises/search" + }.freeze + + SCW_DATA = { + name: 'Secure Code Warrior', + description: "Resolve vulnerabilities faster and confidently with + highly relevant and bite-sized secure coding learning.", + url: "https://integration-api.securecodewarrior.com/api/v1/trial" + }.freeze + + module Security + class TrainingProvider < ApplicationRecord + self.table_name = 'security_training_providers' + end + end + + def self.upsert_providers + current_time = Time.current + timestamps = { created_at: current_time, updated_at: current_time } + + Security::TrainingProvider.upsert_all( + [KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps)], + unique_by: :index_security_training_providers_on_unique_name + ) + end + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index badebabb192..6d8395d048d 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -6,9 +6,9 @@ module Gitlab class Compare < Base def initialize(compare, project:, diff_options:, diff_refs: nil) super(compare, - project: project, + project: project, diff_options: diff_options, - diff_refs: diff_refs) + diff_refs: diff_refs) end def unfold_diff_lines(positions) diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 7cfe0086f57..084ce63e36a 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::Utils::Gzip include Gitlab::Utils::StrongMemoize - EXPIRATION = 1.day + EXPIRATION = 1.hour VERSION = 2 delegate :diffable, to: :@diff_collection @@ -82,6 +82,16 @@ module Gitlab private + def expiration + return 1.day unless Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) + + if Feature.enabled?(:highlight_diffs_short_renewable_expiration, diffable.project) + EXPIRATION + else + 8.hours + end + end + def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) @@ -125,9 +135,9 @@ module Gitlab # def write_to_redis_hash(hash) Gitlab::Redis::Cache.with do |redis| - redis.pipelined do + redis.pipelined do |pipeline| hash.each do |diff_file_id, highlighted_diff_lines_hash| - redis.hset( + pipeline.hset( key, diff_file_id, gzip_compress(highlighted_diff_lines_hash.to_json) @@ -137,8 +147,7 @@ module Gitlab end # HSETs have to have their expiration date manually updated - # - redis.expire(key, EXPIRATION) + pipeline.expire(key, expiration) end record_memory_usage(fetch_memory_usage(redis, key)) @@ -188,11 +197,19 @@ module Gitlab return {} unless file_paths.any? results = [] + cache_key = key + highlight_diffs_renewable_expiration_enabled = Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) + expiration_period = expiration Gitlab::Redis::Cache.with do |redis| - results = redis.hmget(key, file_paths) + redis.pipelined do |pipeline| + results = pipeline.hmget(cache_key, file_paths) + pipeline.expire(key, expiration_period) if highlight_diffs_renewable_expiration_enabled + end end + results = results.value + record_hit_ratio(results) results.map! do |result| diff --git a/lib/gitlab/doorkeeper_secret_storing/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/pbkdf2_sha512.rb deleted file mode 100644 index 4bfb5f9e64c..00000000000 --- a/lib/gitlab/doorkeeper_secret_storing/pbkdf2_sha512.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DoorkeeperSecretStoring - class Pbkdf2Sha512 < ::Doorkeeper::SecretStoring::Base - STRETCHES = 20_000 - # An empty salt is used because we need to look tokens up solely by - # their hashed value. Additionally, tokens are always cryptographically - # pseudo-random and unique, therefore salting provides no - # additional security. - SALT = '' - - def self.transform_secret(plain_secret) - return plain_secret unless Feature.enabled?(:hash_oauth_tokens) - - Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) - end - - ## - # Determines whether this strategy supports restoring - # secrets from the database. This allows detecting users - # trying to use a non-restorable strategy with +reuse_access_tokens+. - def self.allows_restoring_secrets? - false - end - end - end -end diff --git a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb new file mode 100644 index 00000000000..e0884557496 --- /dev/null +++ b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +module Gitlab + module DoorkeeperSecretStoring + module Secret + class Pbkdf2Sha512 < ::Doorkeeper::SecretStoring::Base + STRETCHES = 20_000 + # An empty salt is used because we need to look tokens up solely by + # their hashed value. Additionally, tokens are always cryptographically + # pseudo-random and unique, therefore salting provides no + # additional security. + SALT = '' + + def self.transform_secret(plain_secret, stored_as_hash = false) + return plain_secret if Feature.disabled?(:hash_oauth_secrets) && !stored_as_hash + + Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) + end + + ## + # Determines whether this strategy supports restoring + # secrets from the database. This allows detecting users + # trying to use a non-restorable strategy with +reuse_access_tokens+. + def self.allows_restoring_secrets? + false + end + + ## + # Securely compare the given +input+ value with a +stored+ value + # processed by +transform_secret+. + def self.secret_matches?(input, stored) + stored_as_hash = stored.starts_with?('$pbkdf2-') + transformed_input = transform_secret(input, stored_as_hash) + ActiveSupport::SecurityUtils.secure_compare transformed_input, stored + end + end + end + end +end diff --git a/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb new file mode 100644 index 00000000000..f9e6d4076f3 --- /dev/null +++ b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module DoorkeeperSecretStoring + module Token + class Pbkdf2Sha512 < ::Doorkeeper::SecretStoring::Base + STRETCHES = 20_000 + # An empty salt is used because we need to look tokens up solely by + # their hashed value. Additionally, tokens are always cryptographically + # pseudo-random and unique, therefore salting provides no + # additional security. + SALT = '' + + def self.transform_secret(plain_secret) + return plain_secret unless Feature.enabled?(:hash_oauth_tokens) + + Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) + end + + ## + # Determines whether this strategy supports restoring + # secrets from the database. This allows detecting users + # trying to use a non-restorable strategy with +reuse_access_tokens+. + def self.allows_restoring_secrets? + false + end + end + end + end +end diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index b67ca8d8a7d..931276588f0 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -20,8 +20,8 @@ module Gitlab sanitize_exif_if_needed(content, tmp.path) file = { - tempfile: tmp, - filename: attachment.filename, + tempfile: tmp, + filename: attachment.filename, content_type: attachment.content_type } diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb index 6a0471ef9c5..ca99dd12c8e 100644 --- a/lib/gitlab/email/message/in_product_marketing/team.rb +++ b/lib/gitlab/email/message/in_product_marketing/team.rb @@ -42,18 +42,18 @@ module Gitlab [ s_('InProductMarketing|Did you know teams that use GitLab are far more efficient?'), list([ - s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), - s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') - ]) + s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), + s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') + ]) ].join("\n"), s_("InProductMarketing|We know a thing or two about efficiency and we don't want to keep that to ourselves. Sign up for a free trial of GitLab Ultimate and your teams will be on it from day one."), [ s_('InProductMarketing|Stop wondering and use GitLab to answer questions like:'), list([ - s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), - s_('InProductMarketing|How many days does it take our team to complete various tasks?'), - s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') - ]) + s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), + s_('InProductMarketing|How many days does it take our team to complete various tasks?'), + s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') + ]) ].join("\n") ][series] end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index d55cf3202a6..293aa3b53bf 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -79,11 +79,11 @@ module Gitlab @action_name ||= case @action when :create - 'pushed new' + s_('Notify|pushed new') when :delete - 'deleted' + s_('Notify|deleted') else - 'pushed to' + s_('Notify|pushed to') end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index f539d627dcb..2b36b1c99bd 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -17,13 +17,13 @@ module Gitlab def emoji_image_tag(name, src) image_options = { - class: 'emoji', - src: src, - title: ":#{name}:", - alt: ":#{name}:", + class: 'emoji', + src: src, + title: ":#{name}:", + alt: ":#{name}:", height: 20, - width: 20, - align: 'absmiddle' + width: 20, + align: 'absmiddle' } ActionController::Base.helpers.tag(:img, image_options) diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index f26ab6e3ed1..34c674c3003 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -71,6 +71,21 @@ module Gitlab encode_utf8(data, replace: UNICODE_REPLACEMENT_CHARACTER) end + # This method escapes unsupported UTF-8 characters instead of deleting them + def encode_utf8_with_escaping!(message) + return encode!(message) if Feature.disabled?(:escape_gitaly_refs) + + message = force_encode_utf8(message) + return message if message.valid_encoding? + + unless message.valid_encoding? + message = message.chars.map { |char| char.valid_encoding? ? char : escape_chars(char) }.join + end + + # encode and clean the bad chars + message.replace clean(message) + end + def encode_utf8(message, replace: "") message = force_encode_utf8(message) return message if message.valid_encoding? @@ -145,6 +160,15 @@ module Gitlab message.force_encoding("UTF-8") end + # Escapes \x80 - \xFF characters not supported by UTF-8 + def escape_chars(char) + bytes = char.bytes + + return char unless bytes.one? + + "%#{bytes.first.to_s(16).upcase}" + end + def clean(message, replace: "") message.encode( "UTF-16BE", diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index a1918ee6ad5..f6431483a15 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -97,12 +97,12 @@ module Gitlab def add_instrument_for_cache_hit(status, route, request) payload = { etag_route: route.name, - params: request.filtered_parameters, - headers: request.headers, - format: request.format.ref, - method: request.request_method, - path: request.filtered_path, - status: status + params: request.filtered_parameters, + headers: request.headers, + format: request.format.ref, + method: request.request_method, + path: request.filtered_path, + status: status } ActiveSupport::Notifications.instrument( diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 44c6984c09b..437d577e70e 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -16,9 +16,9 @@ module Gitlab etags = keys.map { generate_etag } Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do + redis.pipelined do |pipeline| keys.each_with_index do |key, i| - redis.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 8a5432025d8..142d0e55593 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -70,9 +70,9 @@ module Gitlab logger = Gitlab::ExperimentationLogger.build logger.warn message: 'Subject must conform to the rollout strategy', - experiment_key: experiment_key, - subject: subject.class.to_s, - rollout_strategy: rollout_strategy(experiment_key) + experiment_key: experiment_key, + subject: subject.class.to_s, + rollout_strategy: rollout_strategy(experiment_key) end def valid_subject_for_rollout_strategy?(experiment_key, subject) diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb index 509daeb0248..c06711d16f8 100644 --- a/lib/gitlab/external_authorization/cache.rb +++ b/lib/gitlab/external_authorization/cache.rb @@ -20,8 +20,8 @@ module Gitlab def store(new_access, new_reason, new_refreshed_at) ::Gitlab::Redis::Cache.with do |redis| - redis.pipelined do - redis.mapped_hmset( + redis.pipelined do |pipeline| + pipeline.mapped_hmset( cache_key, { access: new_access.to_s, @@ -30,7 +30,7 @@ module Gitlab } ) - redis.expire(cache_key, VALIDITY_TIME) + pipeline.expire(cache_key, VALIDITY_TIME) end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 612865ed1be..ca1a2b2a077 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -129,15 +129,15 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - iid: bug['ixBug'], - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, assignee_ids: [assignee_id], - state: bug['fOpen'] == 'true' ? 'opened' : 'closed', - created_at: date, - updated_at: DateTime.parse(bug['dtLastUpdated']) + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) @@ -184,11 +184,11 @@ module Gitlab ) note = Note.create!( - project_id: project.id, - noteable_type: "Issue", - noteable_id: issue.id, - author_id: author_id, - note: body + project_id: project.id, + noteable_type: "Issue", + noteable_id: issue.id, + author_id: author_id, + note: body ) note.update_attribute(:created_at, date) diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 40dcac5f46f..0c13ab604bc 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -55,7 +55,12 @@ module Gitlab end def needs_rewrite? - strong_memoize(:needs_rewrite) { @text_html.include?('data-reference-type=') } + strong_memoize(:needs_rewrite) do + reference_type_attribute = + Banzai::Filter::References::ReferenceFilter::REFERENCE_TYPE_DATA_ATTRIBUTE + + @text_html.include?(reference_type_attribute) + end end private diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4b9f2ababc8..4b877bf44da 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -25,7 +25,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') + encode_utf8_with_escaping!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') end def branch_name(ref) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 003cc87d65a..72f7413500f 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -230,7 +230,6 @@ module Gitlab private def encode_diff_to_utf8(replace_invalid_utf8_chars) - return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol) return unless replace_invalid_utf8_chars && diff_should_be_converted? @diff = Gitlab::EncodingHelper.encode_utf8_with_replacement_character(@diff) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ad655fedb6d..f1cd75258be 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,7 +403,7 @@ module Gitlab wrapped_gitaly_errors do gitaly_blob_client.list_blobs(revisions, limit: REV_LIST_COMMIT_LIMIT, - with_paths: with_paths, dynamic_timeout: dynamic_timeout) + with_paths: with_paths, dynamic_timeout: dynamic_timeout) end end @@ -701,7 +701,9 @@ module Gitlab # Delete the specified branch from the repository # Note: No Git hooks are executed for this action def delete_branch(branch_name) - write_ref(branch_name, Gitlab::Git::BLANK_SHA) + branch_name = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{branch_name}" unless branch_name.start_with?("refs/") + + delete_refs(branch_name) rescue CommandError => e raise DeleteBranchError, e end @@ -913,8 +915,29 @@ module Gitlab true end + # Creates a commit + # + # @param [User] user The committer of the commit. + # @param [String] branch_name: The name of the branch to be created/updated. + # @param [String] message: The commit message. + # @param [Array<Hash>] actions: An array of files to be added/updated/removed. + # @option actions: [Symbol] :action One of :create, :create_dir, :update, :move, :delete, :chmod + # @option actions: [String] :file_path The path of the file or directory being added/updated/removed. + # @option actions: [String] :previous_path The path of the file being moved. Only used for the :move action. + # @option actions: [String,IO] :content The file content for :create or :update + # @option actions: [String] :encoding One of text, base64 + # @option actions: [Boolean] :execute_filemode True sets the executable filemode on the file. + # @option actions: [Boolean] :infer_content True uses the existing file contents instead of using content on move. + # @param [String] author_email: The authors email, if unspecified the committers email is used. + # @param [String] author_name: The authors name, if unspecified the committers name is used. + # @param [String] start_branch_name: The name of the branch to be used as the parent of the commit. Only used if start_sha: is unspecified. + # @param [String] start_sha: The sha to be used as the parent of the commit. + # @param [Gitlab::Git::Repository] start_repository: The repository that contains the start branch or sha. Defaults to use this repository. + # @param [Boolean] force: Force update the branch. + # @return [Gitlab::Git::OperationService::BranchUpdate] + # # rubocop:disable Metrics/ParameterLists - def multi_action( + def commit_files( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, start_branch_name: nil, start_sha: nil, start_repository: nil, @@ -989,8 +1012,8 @@ module Gitlab gitaly_ref_client.branch_names_contains_sha(sha) end - def tag_names_contains_sha(sha) - gitaly_ref_client.tag_names_contains_sha(sha) + def tag_names_contains_sha(sha, limit: 0) + gitaly_ref_client.tag_names_contains_sha(sha, limit: limit) end def search_files_by_content(query, ref, options = {}) @@ -1011,16 +1034,20 @@ module Gitlab end def search_files_by_name(query, ref) - safe_query = Regexp.escape(query.sub(%r{^/*}, "")) + safe_query = query.sub(%r{^/*}, "") ref ||= root_ref return [] if empty? || safe_query.blank? - gitaly_repository_client.search_files_by_name(ref, safe_query) + gitaly_repository_client.search_files_by_name(ref, safe_query).map do |file| + Gitlab::EncodingHelper.encode_utf8(file) + end end def search_files_by_regexp(filter, ref = 'HEAD') - gitaly_repository_client.search_files_by_regexp(ref, filter) + gitaly_repository_client.search_files_by_regexp(ref, filter).map do |file| + Gitlab::EncodingHelper.encode_utf8(file) + end end def find_commits_by_message(query, ref, path, limit, offset) @@ -1031,6 +1058,24 @@ module Gitlab end end + def list_commits_by(query, ref, author: nil, before: nil, after: nil, limit: 1000) + params = { + author: author, + ignore_case: true, + commit_message_patterns: query, + before: before, + after: after, + reverse: false, + pagination_params: { limit: limit } + } + + wrapped_gitaly_errors do + gitaly_commit_client + .list_commits([ref], params) + .map { |c| commit(c) } + end + end + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25, literal_pathspec: false) wrapped_gitaly_errors do gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit, literal_pathspec: literal_pathspec) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 25895dc6728..5ed5158eeea 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -63,7 +63,7 @@ module Gitlab end def init_from_gitaly - @name = encode!(@raw_tag.name.dup) + @name = encode_utf8_with_escaping!(@raw_tag.name.dup) @target = @raw_tag.id @message = message_from_gitaly_tag diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 4bab94968d7..2228fcb886e 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -70,18 +70,6 @@ module Gitlab @repository.exists? end - def write_page(name, format, content, commit_details) - wrapped_gitaly_errors do - gitaly_write_page(name, format, content, commit_details) - end - end - - def update_page(page_path, title, format, content, commit_details) - wrapped_gitaly_errors do - gitaly_update_page(page_path, title, format, content, commit_details) - end - end - def list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) wrapped_gitaly_errors do gitaly_list_pages( @@ -113,21 +101,13 @@ module Gitlab @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end - def gitaly_write_page(name, format, content, commit_details) - gitaly_wiki_client.write_page(name, format, content, commit_details) - end - - def gitaly_update_page(page_path, title, format, content, commit_details) - gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) - end - def gitaly_find_page(title:, version: nil, dir: nil, load_content: true) return unless title.present? wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir, load_content: load_content) return unless wiki_page - Gitlab::Git::WikiPage.new(wiki_page, version) + Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) rescue GRPC::InvalidArgument nil end @@ -143,7 +123,7 @@ module Gitlab end gitaly_pages.map do |wiki_page, version| - Gitlab::Git::WikiPage.new(wiki_page, version) + Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) end end end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index a1f3d64ccde..57b7e7d53dd 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -5,17 +5,31 @@ module Gitlab class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :historical, :formatted_data - # This class abstracts away Gitlab::GitalyClient::WikiPage - def initialize(gitaly_page, version) - @url_path = gitaly_page.url_path - @title = gitaly_page.title - @format = gitaly_page.format - @path = gitaly_page.path - @raw_data = gitaly_page.raw_data - @name = gitaly_page.name - @historical = gitaly_page.historical? + class << self + # Abstracts away Gitlab::GitalyClient::WikiPage + def from_gitaly_wiki_page(gitaly_page, version) + new( + url_path: gitaly_page.url_path, + title: gitaly_page.title, + format: gitaly_page.format, + path: gitaly_page.path, + raw_data: gitaly_page.raw_data, + name: gitaly_page.name, + historical: gitaly_page.historical?, + version: version + ) + end + end - @version = version + def initialize(hash) + @url_path = hash[:url_path] + @title = hash[:title] + @format = hash[:format] + @path = hash[:path] + @raw_data = hash[:raw_data] + @name = hash[:name] + @historical = hash[:historical] + @version = hash[:version] end def historical? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0f306a9825d..312d1dddff1 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -232,7 +232,7 @@ module Gitlab msg.paths.map do |path| Gitlab::Git::ChangedPath.new( status: path.status, - path: EncodingHelper.encode!(path.path) + path: EncodingHelper.encode!(path.path) ) end end @@ -251,14 +251,23 @@ module Gitlab consume_commits_response(response) end - def list_commits(revisions, reverse: false, pagination_params: nil) + def list_commits(revisions, params = {}) request = Gitaly::ListCommitsRequest.new( repository: @gitaly_repo, revisions: Array.wrap(revisions), - reverse: reverse, - pagination_params: pagination_params + reverse: !!params[:reverse], + ignore_case: params[:ignore_case], + pagination_params: params[:pagination_params] ) + if params[:commit_message_patterns] + request.commit_message_patterns += Array.wrap(params[:commit_message_patterns]) + end + + request.author = encode_binary(params[:author]) if params[:author] + request.before = GitalyClient.timestamp(params[:before]) if params[:before] + request.after = GitalyClient.timestamp(params[:after]) if params[:after] + response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -396,12 +405,12 @@ module Gitlab def find_commits(options) request = Gitaly::FindCommitsRequest.new( - repository: @gitaly_repo, - limit: options[:limit], - offset: options[:offset], - follow: options[:follow], - skip_merges: options[:skip_merges], - all: !!options[:all], + repository: @gitaly_repo, + limit: options[:limit], + offset: options[:offset], + follow: options[:follow], + skip_merges: options[:skip_merges], + all: !!options[:all], first_parent: !!options[:first_parent], global_options: parse_global_options!(options), disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index c5c6ec1cdfa..7835fb32f59 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -85,8 +85,20 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) - rescue GRPC::FailedPrecondition => ex - raise Gitlab::Git::Repository::InvalidRef, ex + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :custom_hook + raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), + fallback_message: e.details) + else + if e.code == GRPC::Core::StatusCodes::FAILED_PRECONDITION + raise Gitlab::Git::Repository::InvalidRef, e + end + + raise + end end def user_update_branch(branch_name, user, newrev, oldrev) @@ -410,9 +422,9 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :operation_service, - :user_commit_files, req_enum, timeout: GitalyClient.long_timeout, - remote_storage: start_repository&.storage) + response = GitalyClient.call( + @repository.storage, :operation_service, :user_commit_files, req_enum, + timeout: GitalyClient.long_timeout, remote_storage: start_repository&.storage) if (pre_receive_error = response.pre_receive_error.presence) raise Gitlab::Git::PreReceiveError, pre_receive_error diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 42f9c165610..bb6bc3121bd 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -7,7 +7,8 @@ module Gitlab TAGS_SORT_KEY = { 'name' => Gitaly::FindAllTagsRequest::SortBy::Key::REFNAME, - 'updated' => Gitaly::FindAllTagsRequest::SortBy::Key::CREATORDATE + 'updated' => Gitaly::FindAllTagsRequest::SortBy::Key::CREATORDATE, + 'version' => Gitaly::FindAllTagsRequest::SortBy::Key::VERSION_REFNAME }.freeze TAGS_SORT_DIRECTION = { @@ -104,7 +105,7 @@ module Gitlab return unless branch target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) - Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) + Gitlab::Git::Branch.new(@repository, branch.name.dup, branch.target_commit.id, target_commit) end def find_tag(tag_name) @@ -258,7 +259,7 @@ module Gitlab end def sort_tags_by_param(sort_by) - match = sort_by.match(/^(?<key>name|updated)_(?<direction>asc|desc)$/) + match = sort_by.match(/^(?<key>name|updated|version)_(?<direction>asc|desc)$/) return unless match @@ -269,14 +270,23 @@ module Gitlab end def consume_find_local_branches_response(response) - response.flat_map do |message| - message.branches.map do |gitaly_branch| - Gitlab::Git::Branch.new( - @repository, - encode!(gitaly_branch.name.dup), - gitaly_branch.commit_id, - commit_from_local_branches_response(gitaly_branch) - ) + if Feature.enabled?(:gitaly_simplify_find_local_branches_response, type: :undefined) + response.flat_map do |message| + message.local_branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) + end + end + else + response.flat_map do |message| + message.branches.map do |gitaly_branch| + Gitlab::Git::Branch.new( + @repository, + gitaly_branch.name.dup, + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) + ) + end end end end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb index 36bda67c26e..48fd0e66354 100644 --- a/lib/gitlab/gitaly_client/server_service.rb +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -26,6 +26,19 @@ module Gitlab storage_specific(disk_statistics) end + def readiness_check + request = Gitaly::ReadinessCheckRequest.new(timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@storage, :server_service, :readiness_check, request, timeout: GitalyClient.default_timeout) + + return { success: true } if response.ok_response + + failed_checks = response.failure_response.failed_checks.map do |failed_check| + ["#{failed_check.name}: #{failed_check.error_message}"] + end + + { success: false, message: failed_checks.join("\n") } + end + private def storage_specific(response) diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb new file mode 100644 index 00000000000..b71d5f753f2 --- /dev/null +++ b/lib/gitlab/github_import/attachments_downloader.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class AttachmentsDownloader + include ::Gitlab::ImportExport::CommandLineUtil + include ::BulkImports::FileDownloads::FilenameFetch + include ::BulkImports::FileDownloads::Validations + + DownloadError = Class.new(StandardError) + + FILENAME_SIZE_LIMIT = 255 # chars before the extension + DEFAULT_FILE_SIZE_LIMIT = 25.megabytes + TMP_DIR = File.join(Dir.tmpdir, 'github_attachments').freeze + + attr_reader :file_url, :filename, :file_size_limit + + def initialize(file_url, file_size_limit: DEFAULT_FILE_SIZE_LIMIT) + @file_url = file_url + @file_size_limit = file_size_limit + + filename = URI(file_url).path.split('/').last + @filename = ensure_filename_size(filename) + end + + def perform + validate_content_length + validate_filepath + + file = download + validate_symlink + file + end + + def delete + FileUtils.rm_rf File.dirname(filepath) + end + + private + + def raise_error(message) + raise DownloadError, message + end + + def response_headers + @response_headers ||= + Gitlab::HTTP.perform_request(Net::HTTP::Head, file_url, {}).headers + end + + def download + file = File.open(filepath, 'wb') + Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, stream_body: true) { |batch| file.write(batch) } + file + end + + def filepath + @filepath ||= begin + dir = File.join(TMP_DIR, SecureRandom.uuid) + mkdir_p dir + File.join(dir, filename) + end + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 11a41149274..6cff15a204f 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -76,11 +76,15 @@ module Gitlab each_object(:pull_request_reviews, repo_name, iid) end + def repos(options = {}) + octokit.repos(nil, options).map(&:to_h) + end + # Returns the details of a GitHub repository. # # name - The path (in the form `owner/repository`) of the repository. def repository(name) - with_rate_limit { octokit.repo(name) } + with_rate_limit { octokit.repo(name).to_h } end def pull_request(repo_name, iid) @@ -99,6 +103,14 @@ module Gitlab each_object(:releases, *args) end + def branches(*args) + each_object(:branches, *args) + end + + def branch_protection(repo_name, branch_name) + with_rate_limit { octokit.branch_protection(repo_name, branch_name) } + end + # Fetches data from the GitHub API and yields a Page object for every page # of data, without loading all of them into memory. # @@ -167,7 +179,7 @@ module Gitlab end def search_repos_by_name(name, options = {}) - with_retry { octokit.search_repositories(search_query(str: name, type: :name), options) } + with_retry { octokit.search_repositories(search_query(str: name, type: :name), options).to_h } end def search_query(str:, type:, include_collaborations: true, include_orgs: true) diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb index 9ab1d916d33..8218acf2bfb 100644 --- a/lib/gitlab/github_import/importer/events/base_importer.rb +++ b/lib/gitlab/github_import/importer/events/base_importer.rb @@ -29,6 +29,19 @@ module Gitlab def issuable_db_id(object) IssuableFinder.new(project, object).database_id end + + def issuable_type(issue_event) + merge_request_event?(issue_event) ? MergeRequest.name : Issue.name + end + + def merge_request_event?(issue_event) + issue_event.issuable_type == MergeRequest.name + end + + def resource_event_belongs_to(issue_event) + belongs_to_key = merge_request_event?(issue_event) ? :merge_request_id : :issue_id + { belongs_to_key => issuable_db_id(issue_event) } + end end end end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index c8f6335e4a8..b75d41f40de 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -7,22 +7,22 @@ module Gitlab class ChangedAssignee < BaseImporter def execute(issue_event) assignee_id = author_id(issue_event, author_key: :assignee) - assigner_id = author_id(issue_event, author_key: :assigner) + author_id = author_id(issue_event, author_key: :actor) - note_body = parse_body(issue_event, assigner_id, assignee_id) + note_body = parse_body(issue_event, assignee_id) - create_note(issue_event, note_body, assigner_id) + create_note(issue_event, note_body, author_id) end private - def create_note(issue_event, note_body, assigner_id) + def create_note(issue_event, note_body, author_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, - author_id: assigner_id, + author_id: author_id, note: note_body, system_note_metadata: SystemNoteMetadata.new( { @@ -36,12 +36,14 @@ module Gitlab ) end - def parse_body(issue_event, assigner_id, assignee_id) + def parse_body(issue_event, assignee_id) + assignee = User.find(assignee_id).to_reference + Gitlab::I18n.with_default_locale do if issue_event.event == "unassigned" - "unassigned #{User.find(assigner_id).to_reference}" + "unassigned #{assignee}" else - "assigned to #{User.find(assignee_id).to_reference}" + "assigned to #{assignee}" end end end diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 818a9202745..83130d18db9 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -12,13 +12,14 @@ module Gitlab private def create_event(issue_event) - ResourceLabelEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceLabelEvent.create!(attrs) end def label_finder diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb index 3164c041dc3..39b92d88b58 100644 --- a/lib/gitlab/github_import/importer/events/changed_milestone.rb +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -17,14 +17,15 @@ module Gitlab private def create_event(issue_event) - ResourceMilestoneEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), created_at: issue_event.created_at, milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, action: action(issue_event.event), state: DEFAULT_STATE - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceMilestoneEvent.create!(attrs) end def action(event_type) diff --git a/lib/gitlab/github_import/importer/events/changed_reviewer.rb b/lib/gitlab/github_import/importer/events/changed_reviewer.rb new file mode 100644 index 00000000000..17b1fa4ab45 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_reviewer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedReviewer < BaseImporter + def execute(issue_event) + requested_reviewer_id = author_id(issue_event, author_key: :requested_reviewer) + review_requester_id = author_id(issue_event, author_key: :review_requester) + + note_body = parse_body(issue_event, requested_reviewer_id) + + create_note(issue_event, note_body, review_requester_id) + end + + private + + def create_note(issue_event, note_body, review_requester_id) + Note.create!( + system: true, + noteable_type: issuable_type(issue_event), + noteable_id: issuable_db_id(issue_event), + project: project, + author_id: review_requester_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new( + { + action: 'reviewer', + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ), + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def parse_body(issue_event, requested_reviewer_id) + requested_reviewer = User.find(requested_reviewer_id).to_reference + + if issue_event.event == 'review_request_removed' + "#{SystemNotes::IssuablesService.issuable_events[:review_request_removed]}" \ + " #{requested_reviewer}" + else + "#{SystemNotes::IssuablesService.issuable_events[:review_requested]}" \ + " #{requested_reviewer}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index ca8730d0f27..58d9dbf826c 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'closed', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,15 +25,16 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', close_after_error_tracking_resolve: false, close_auto_resolve_prometheus_alert: false, created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 89fc1bdeb09..b56ae186d3c 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -33,7 +33,7 @@ module Gitlab def create_note(issue_event, note_body, user_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, author_id: user_id, diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index 96d112b04c6..fb9e08116ba 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -14,7 +14,7 @@ module Gitlab def note_params(issue_event) { noteable_id: issuable_db_id(issue_event), - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), project_id: project.id, author_id: author_id(issue_event), note: parse_body(issue_event), diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb index b75344bf817..8abeba0777d 100644 --- a/lib/gitlab/github_import/importer/events/reopened.rb +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'reopened', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,12 +25,13 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), state: 'reopened', created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index ef456e56ee1..80749aae93c 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -15,11 +15,7 @@ module Gitlab @client = client end - # TODO: Add MergeRequest events support - # https://gitlab.com/groups/gitlab-org/-/epics/7673 def execute - return if issue_event.issuable_type == 'MergeRequest' - importer = event_importer_class(issue_event) if importer importer.new(project, client).execute(issue_event) @@ -49,6 +45,8 @@ module Gitlab Gitlab::GithubImport::Importer::Events::CrossReferenced when 'assigned', 'unassigned' Gitlab::GithubImport::Importer::Events::ChangedAssignee + when 'review_requested', 'review_request_removed' + Gitlab::GithubImport::Importer::Events::ChangedReviewer end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb new file mode 100644 index 00000000000..16215fdce8e --- /dev/null +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ProtectedBranchImporter + attr_reader :protected_branch, :project, :client + + # protected_branch - An instance of + # `Gitlab::GithubImport::Representation::ProtectedBranch`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(protected_branch, project, client) + @protected_branch = protected_branch + @project = project + @client = client + end + + def execute + # The creator of the project is always allowed to create protected + # branches, so we skip the authorization check in this service class. + ProtectedBranches::CreateService + .new(project, project.creator, params) + .execute(skip_authorization: true) + end + + private + + def params + { + name: protected_branch.id, + push_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + merge_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + allow_force_push: allow_force_push? + } + end + + def allow_force_push? + if ProtectedBranch.protected?(project, protected_branch.id) + ProtectedBranch.allow_force_push?(project, protected_branch.id) && protected_branch.allow_force_pushes + else + protected_branch.allow_force_pushes + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb new file mode 100644 index 00000000000..b5be823d5ab --- /dev/null +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ProtectedBranchesImporter + include ParallelScheduling + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + protected_branches = client.branches(repo).select { |branch| branch.protection&.enabled } + protected_branches.each do |protected_branch| + object = client.branch_protection(repo, protected_branch.name) + next if object.nil? || already_imported?(object) + + yield object + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + mark_as_imported(object) + end + end + + def importer_class + ProtectedBranchImporter + end + + def representation_class + Gitlab::GithubImport::Representation::ProtectedBranch + end + + def sidekiq_worker_class + ImportProtectedBranchWorker + end + + def object_type + :protected_branch + end + + def collection_method + :protected_branches + end + + def id_for_already_imported_cache(protected_branch) + protected_branch.name + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/release_attachments_importer.rb b/lib/gitlab/github_import/importer/release_attachments_importer.rb new file mode 100644 index 00000000000..6419851623c --- /dev/null +++ b/lib/gitlab/github_import/importer/release_attachments_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleaseAttachmentsImporter + attr_reader :release_db_id, :release_description, :project + + # release - An instance of `ReleaseAttachments`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(release_attachments, project, _client = nil) + @release_db_id = release_attachments.release_db_id + @release_description = release_attachments.description + @project = project + end + + def execute + attachment_urls = MarkdownText.fetch_attachment_urls(release_description) + new_description = attachment_urls.reduce(release_description) do |description, url| + new_url = download_attachment(url) + description.gsub(url, new_url) + end + + Release.find(release_db_id).update_column(:description, new_description) + end + + private + + # in: github attachment markdown url + # out: gitlab attachment markdown url + def download_attachment(markdown_url) + url = extract_url_from_markdown(markdown_url) + name_prefix = extract_name_from_markdown(markdown_url) + + downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(url) + file = downloader.perform + uploader = UploadService.new(project, file, FileUploader).execute + "#{name_prefix}(#{uploader.to_h[:url]})" + ensure + downloader&.delete + end + + # in: "" + # out: https://user-images.githubusercontent.com/.. + def extract_url_from_markdown(text) + text.match(%r{https://.*\)$}).to_a[0].chop + end + + # in: "" + # out: ![image-icon] + def extract_name_from_markdown(text) + text.match(%r{^!?\[.*\]}).to_a[0] + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_attachments_importer.rb b/lib/gitlab/github_import/importer/releases_attachments_importer.rb new file mode 100644 index 00000000000..7221c802d83 --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_attachments_importer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesAttachmentsImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + project.releases.select(:id, :description).each_batch(of: BATCH_SIZE, column: :id) do |batch| + batch.each do |release| + next if already_imported?(release) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield release + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(release) + end + end + end + + def representation_class + Representation::ReleaseAttachments + end + + def importer_class + ReleaseAttachmentsImporter + end + + def sidekiq_worker_class + ImportReleaseAttachmentsWorker + end + + def collection_method + :release_attachments + end + + def object_type + :release_attachment + end + + def id_for_already_imported_cache(release) + release.id + end + + def object_representation(object) + representation_class.from_db_record(object) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index aba4729e9c8..708768a60cf 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -17,7 +17,7 @@ module Gitlab # Returns true if we should import the wiki for the project. # rubocop: disable CodeReuse/ActiveRecord def import_wiki? - client_repository&.has_wiki && + client_repository[:has_wiki] && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end @@ -86,7 +86,7 @@ module Gitlab private def default_branch - client_repository&.default_branch + client_repository[:default_branch] end def client_repository diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 8e4015acbbc..8a9ddfc6ec0 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -7,7 +7,7 @@ module Gitlab include ParallelScheduling include SingleEndpointNotesImporting - PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issuable_iid}/%{collection}' BATCH_SIZE = 100 def initialize(project, client, parallel: true) @@ -27,12 +27,20 @@ module Gitlab Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - associated.issue = { 'number' => parent_record.iid } + pull_request = parent_record.is_a? MergeRequest + associated.issue = { 'number' => parent_record.iid, 'pull_request' => pull_request } yield(associated) mark_as_imported(associated) end + # In Github Issues and MergeRequests uses the same API to get their events. + # Even more - they have commonly uniq iid + def each_associated_page(&block) + issues_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + merge_requests_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + end + def importer_class IssueEventImporter end @@ -53,16 +61,20 @@ module Gitlab :issue_timeline end - def parent_collection + def issues_collection project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord end + def merge_requests_collection + project.merge_requests.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + def parent_imported_cache_key "github-importer/issues/#{collection_method}/already-imported/#{project.id}" end - def page_counter_id(issue) - PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + def page_counter_id(issuable) + PROCESSED_PAGE_CACHE_KEY % { issuable_iid: issuable.iid, collection: collection_method } end def id_for_already_imported_cache(event) @@ -74,10 +86,10 @@ module Gitlab end # Cross-referenced events on Github doesn't have id. - def compose_associated_id!(issue, event) + def compose_associated_id!(issuable, event) return if event.event != 'cross-referenced' - event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + event.id = "cross-reference##{issuable.iid}-in-#{event.source.issue.id}" end end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 692016bd005..bf2856bc77f 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# This class includes overriding Kernel#format method +# what makes impossible to use it here +# rubocop:disable Style/FormatString module Gitlab module GithubImport class MarkdownText @@ -8,6 +11,21 @@ module Gitlab ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' + MEDIA_TYPES = %w[gif jpeg jpg mov mp4 png svg webm].freeze + DOC_TYPES = %w[ + csv docx fodg fodp fods fodt gz log md odf odg odp ods + odt pdf pptx tgz txt xls xlsx zip + ].freeze + ALL_TYPES = (MEDIA_TYPES + DOC_TYPES).freeze + + # On github.com we have base url for docs and CDN url for media. + # On github EE as far as we can know there is no CDN urls and media is placed on base url. + # To no escape the escaping symbol we use single quotes instead of double with interpolation. + # rubocop:disable Style/StringConcatenation + CDN_URL_MATCHER = '(!\[.+\]\(%{github_media_cdn}/\d+/(\w|-)+\.(' + MEDIA_TYPES.join('|') + ')\))' + BASE_URL_MATCHER = '(\[.+\]\(%{github_url}/.+/.+/files/\d+/.+\.(' + ALL_TYPES.join('|') + ')\))' + # rubocop:enable Style/StringConcatenation + class << self def format(*args) new(*args).to_s @@ -24,8 +42,20 @@ module Gitlab .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) end + def fetch_attachment_urls(text) + cdn_url_matcher = CDN_URL_MATCHER % { github_media_cdn: Regexp.escape(github_media_cdn) } + doc_url_matcher = BASE_URL_MATCHER % { github_url: Regexp.escape(github_url) } + + text.scan(Regexp.new(cdn_url_matcher)).map(&:first) + + text.scan(Regexp.new(doc_url_matcher)).map(&:first) + end + private + def github_media_cdn + 'https://user-images.githubusercontent.com' + end + # Returns github domain without slash in the end def github_url oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} @@ -63,3 +93,4 @@ module Gitlab end end end +# rubocop:enable Style/FormatString diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8c18c74d24..bf5046de36c 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -63,7 +63,7 @@ module Gitlab # Imports all the objects in sequence in the current thread. def sequential_import each_object_to_import do |object| - repr = representation_class.from_api_response(object, additional_object_data) + repr = object_representation(object) importer_class.new(repr, project, client).execute end @@ -83,7 +83,7 @@ module Gitlab import_arguments = [] each_object_to_import do |object| - repr = representation_class.from_api_response(object, additional_object_data) + repr = object_representation(object) import_arguments << [project.id, repr.to_hash, waiter.key] @@ -210,6 +210,10 @@ module Gitlab {} end + def object_representation(object) + representation_class.from_api_response(object, additional_object_data) + end + def info(project_id, extra = {}) Logger.info(log_attributes(project_id, extra)) end diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb index d2438ee8094..84de4d4798d 100644 --- a/lib/gitlab/github_import/representation/expose_attribute.rb +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -20,6 +20,10 @@ module Gitlab end end end + + def [](key) + respond_to?(key.to_sym) ? attributes[key] : nil + end end end end diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 67a5df73a97..89271a7dcd6 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -10,7 +10,8 @@ module Gitlab attr_reader :attributes expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, - :milestone_title, :issue, :source, :assignee, :assigner, :created_at + :milestone_title, :issue, :source, :assignee, :review_requester, + :requested_reviewer, :created_at # attributes - A Hash containing the event details. The keys of this # Hash (and any nested hashes) must be symbols. @@ -47,7 +48,8 @@ module Gitlab issue: event.issue&.to_h&.symbolize_keys, source: event.source, assignee: user_representation(event.assignee), - assigner: user_representation(event.assigner), + requested_reviewer: user_representation(event.requested_reviewer), + review_requester: user_representation(event.review_requester), created_at: event.created_at ) end @@ -57,7 +59,8 @@ module Gitlab hash = Representation.symbolize_hash(raw_hash) hash[:actor] = user_representation(hash[:actor], source: :hash) hash[:assignee] = user_representation(hash[:assignee], source: :hash) - hash[:assigner] = user_representation(hash[:assigner], source: :hash) + hash[:requested_reviewer] = user_representation(hash[:requested_reviewer], source: :hash) + hash[:review_requester] = user_representation(hash[:review_requester], source: :hash) new(hash) end diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb new file mode 100644 index 00000000000..b80b7cf1076 --- /dev/null +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class ProtectedBranch + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :allow_force_pushes + + # Builds a Branch Protection info from a GitHub API response. + # Resource structure details: + # https://docs.github.com/en/rest/branches/branch-protection#get-branch-protection + # branch_protection - An instance of `Sawyer::Resource` containing the protection details. + def self.from_api_response(branch_protection, _additional_object_data = {}) + branch_name = branch_protection.url.match(%r{/branches/(\S{1,255})/protection$})[1] + + hash = { + id: branch_name, + allow_force_pushes: branch_protection.allow_force_pushes.enabled + } + + new(hash) + end + + # Builds a new Protection using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the raw Protection details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/release_attachments.rb b/lib/gitlab/github_import/representation/release_attachments.rb new file mode 100644 index 00000000000..fd272be2405 --- /dev/null +++ b/lib/gitlab/github_import/representation/release_attachments.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# This class only partly represents Release record from DB and +# is used to connect ReleasesAttachmentsImporter with ReleaseAttachmentsImporter +# without modifying ObjectImporter a lot. +# Attachments are inside release's `description`. +module Gitlab + module GithubImport + module Representation + class ReleaseAttachments + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :release_db_id, :description + + # Builds a event from a GitHub API response. + # + # release - An instance of `Release` model. + def self.from_db_record(release) + new( + release_db_id: release.id, + description: release.description + ) + end + + def self.from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { db_id: release_db_id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 6bc37337799..ab37bc92ee7 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -16,6 +16,7 @@ module Gitlab ].freeze PARALLEL_IMPORTERS = [ + Importer::ProtectedBranchesImporter, Importer::PullRequestsImporter, Importer::IssuesImporter, Importer::DiffNotesImporter, diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 0a3559adde3..aea4059dfbc 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -63,23 +63,27 @@ module Gitlab mark_as_imported(associated) end - def each_associated_page + def each_associated_page(&block) parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |parent_record| - # The page counter needs to be scoped by parent_record to avoid skipping - # pages of notes from already imported parent_record. - page_counter = PageCounter.new(project, page_counter_id(parent_record)) - repo = project.import_source - options = collection_options.merge(page: page_counter.current) + process_batch(batch, &block) + end + end - client.each_page(collection_method, repo, parent_record.iid, options) do |page| - next unless page_counter.set(page.number) + def process_batch(batch) + batch.each do |parent_record| + # The page counter needs to be scoped by parent_record to avoid skipping + # pages of notes from already imported parent_record. + page_counter = PageCounter.new(project, page_counter_id(parent_record)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) - yield parent_record, page - end + client.each_page(collection_method, repo, parent_record.iid, options) do |page| + next unless page_counter.set(page.number) - mark_parent_imported(parent_record) + yield parent_record, page end + + mark_parent_imported(parent_record) end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 6d6a00d260d..1feb0d450f0 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -45,8 +45,10 @@ module Gitlab object&.actor when :assignee object&.assignee - when :assigner - object&.assigner + when :requested_reviewer + object&.requested_reviewer + when :review_requester + object&.review_requester else object&.author end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 5f1802e323c..08a614edb4b 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -56,7 +56,6 @@ module Gitlab push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) - push_frontend_feature_flag(:mr_attention_requests, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb index 40b90310e8b..657364abfdf 100644 --- a/lib/gitlab/graphql/errors.rb +++ b/lib/gitlab/graphql/errors.rb @@ -7,6 +7,7 @@ module Gitlab ArgumentError = Class.new(BaseError) ResourceNotAvailable = Class.new(BaseError) MutationError = Class.new(BaseError) + LimitError = Class.new(BaseError) end end end diff --git a/lib/gitlab/graphql/limit/field_call_count.rb b/lib/gitlab/graphql/limit/field_call_count.rb new file mode 100644 index 00000000000..4165970a2a6 --- /dev/null +++ b/lib/gitlab/graphql/limit/field_call_count.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Limit + class FieldCallCount < ::GraphQL::Schema::FieldExtension + def resolve(object:, arguments:, context:) + raise Gitlab::Graphql::Errors::ArgumentError, 'Limit must be specified.' unless limit + raise Gitlab::Graphql::Errors::LimitError, error_message if increment_call_count(context) > limit + + yield(object, arguments) + end + + private + + def increment_call_count(context) + context[:call_count] ||= {} + context[:call_count][field] ||= 0 + context[:call_count][field] += 1 + end + + def limit + options[:limit] + end + + def error_message + "\"#{field.graphql_name}\" field can be requested only for #{limit} #{field.owner.graphql_name}(s) at a time." + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index b074c273996..987a5e7b74b 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -59,11 +59,15 @@ module Gitlab if before true elsif first - case sliced_nodes - when Array - sliced_nodes.size > limit_value + if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) + limited_nodes.size > limit_value else - sliced_nodes.limit(1).offset(limit_value).exists? # rubocop: disable CodeReuse/ActiveRecord + case sliced_nodes + when Array + sliced_nodes.size > limit_value + else + sliced_nodes.limit(1).offset(limit_value).exists? # rubocop: disable CodeReuse/ActiveRecord + end end else false @@ -89,7 +93,7 @@ module Gitlab # So we're ok loading them into memory here as that's bound to happen # anyway. Having them ready means we can modify the result while # rendering the fields. - @nodes ||= limited_nodes.to_a + @nodes ||= limited_nodes.to_a.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord end def items @@ -116,15 +120,21 @@ module Gitlab end if last - paginated_nodes = LastItems.take_items(sliced_nodes, limit_value + 1) + paginated_nodes = sliced_nodes.last(limit_value + 1) # there is an extra node, so there is a previous page @has_previous_page = paginated_nodes.count > limit_value @has_previous_page ? paginated_nodes.last(limit_value) : paginated_nodes elsif loaded?(sliced_nodes) - sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord + if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) + sliced_nodes.take(limit_value + 1) # rubocop: disable CodeReuse/ActiveRecord + else + sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord + end + elsif Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) + sliced_nodes.limit(limit_value + 1).to_a else - sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord + sliced_nodes.limit(limit_value) end end end @@ -141,7 +151,7 @@ module Gitlab def limit_value # note: only first _or_ last can be specified, not both - @limit_value ||= [first, last, max_page_size].compact.min + @limit_value ||= [first, last, max_page_size, GitlabSchema.default_max_page_size].compact.min end def loaded?(items) diff --git a/lib/gitlab/graphql/pagination/keyset/last_items.rb b/lib/gitlab/graphql/pagination/keyset/last_items.rb deleted file mode 100644 index 960567a6fbc..00000000000 --- a/lib/gitlab/graphql/pagination/keyset/last_items.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Pagination - module Keyset - # This class handles the last(N) ActiveRecord call even if a special ORDER BY configuration is present. - # For the last(N) call, ActiveRecord calls reverse_order, however for some cases it raises - # ActiveRecord::IrreversibleOrderError error. - class LastItems - # rubocop: disable CodeReuse/ActiveRecord - def self.take_items(scope, count) - if Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) - order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) - items = scope.reorder(order.reversed_order).first(count) - items.is_a?(Array) ? items.reverse : items - else - scope.last(count) - end - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/type_name_deprecations.rb b/lib/gitlab/graphql/type_name_deprecations.rb index c27ad1d54f5..1ec6fd1c09f 100644 --- a/lib/gitlab/graphql/type_name_deprecations.rb +++ b/lib/gitlab/graphql/type_name_deprecations.rb @@ -14,6 +14,9 @@ module Gitlab DEPRECATIONS = [ Gitlab::Graphql::DeprecationsBase::NameDeprecation.new( old_name: 'CiRunnerUpgradeStatusType', new_name: 'CiRunnerUpgradeStatus', milestone: '15.3' + ), + Gitlab::Graphql::DeprecationsBase::NameDeprecation.new( + old_name: 'RunnerMembershipFilter', new_name: 'CiRunnerMembershipFilter', milestone: '15.4' ) ].freeze diff --git a/lib/gitlab/harbor/query.rb b/lib/gitlab/harbor/query.rb index c120810ecf1..fcd984b01ce 100644 --- a/lib/gitlab/harbor/query.rb +++ b/lib/gitlab/harbor/query.rb @@ -17,7 +17,7 @@ module Gitlab message: 'Id invalid' }, allow_blank: true validates :artifact_id, format: { - with: /\A[a-zA-Z0-9\_\.\-$]+\z/, + with: /\A[a-zA-Z0-9\_\.\-$:]+\z/, message: 'Id invalid' }, allow_blank: true validates :sort, format: { diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index f5f142c251f..2bd8ea711b5 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -27,17 +27,35 @@ module Gitlab end def check(storage_name) - serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) - result = serv.check + storage_healthy = healthy(storage_name) + unless storage_healthy[:success] + return HealthChecks::Result.new( + name, + storage_healthy[:success], + storage_healthy[:message], + shard: storage_name + ) + end + storage_ready = ready(storage_name) HealthChecks::Result.new( name, - result[:success], - result[:message], + storage_ready[:success], + storage_ready[:message], shard: storage_name ) end + def healthy(storage_name) + serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) + serv.check + end + + def ready(storage_name) + serv = Gitlab::GitalyClient::ServerService.new(storage_name) + serv.readiness_check + end + private def metric_prefix diff --git a/lib/gitlab/health_checks/redis.rb b/lib/gitlab/health_checks/redis.rb new file mode 100644 index 00000000000..895bce5a5a9 --- /dev/null +++ b/lib/gitlab/health_checks/redis.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + ALL_INSTANCE_CHECKS = + ::Gitlab::Redis::ALL_CLASSES.map do |instance_class| + check_class = Class.new + check_class.extend(RedisAbstractCheck) + const_set("#{instance_class.store_name}Check", check_class) + + check_class + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb deleted file mode 100644 index bd843bdaac4..00000000000 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class CacheCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb deleted file mode 100644 index fb92db937dc..00000000000 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class QueuesCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/rate_limiting_check.rb b/lib/gitlab/health_checks/redis/rate_limiting_check.rb deleted file mode 100644 index 0e9d94f7dff..00000000000 --- a/lib/gitlab/health_checks/redis/rate_limiting_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class RateLimitingCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/redis_abstract_check.rb b/lib/gitlab/health_checks/redis/redis_abstract_check.rb index ecad4b06ea9..9a9a4d1faba 100644 --- a/lib/gitlab/health_checks/redis/redis_abstract_check.rb +++ b/lib/gitlab/health_checks/redis/redis_abstract_check.rb @@ -10,12 +10,12 @@ module Gitlab successful?(check) end - private - def redis_instance_class_name Gitlab::Redis.const_get(redis_instance_name.camelize, false) end + private + def metric_prefix "redis_#{redis_instance_name}_ping" end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb deleted file mode 100644 index c793a939abd..00000000000 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def successful?(result) - result == true - end - - def check - redis_health_checks.all?(&:check_up) - end - - def redis_health_checks - [ - Gitlab::HealthChecks::Redis::CacheCheck, - Gitlab::HealthChecks::Redis::QueuesCheck, - Gitlab::HealthChecks::Redis::SharedStateCheck, - Gitlab::HealthChecks::Redis::TraceChunksCheck, - Gitlab::HealthChecks::Redis::RateLimitingCheck, - Gitlab::HealthChecks::Redis::SessionsCheck - ] - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/sessions_check.rb b/lib/gitlab/health_checks/redis/sessions_check.rb deleted file mode 100644 index 90a4c868f40..00000000000 --- a/lib/gitlab/health_checks/redis/sessions_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class SessionsCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb deleted file mode 100644 index 80f91784b8c..00000000000 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class SharedStateCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb deleted file mode 100644 index 9a89a1ce51d..00000000000 --- a/lib/gitlab/health_checks/redis/trace_chunks_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class TraceChunksCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/hook_data/project_member_builder.rb b/lib/gitlab/hook_data/project_member_builder.rb index 90fc83fdf21..2ee61705ec1 100644 --- a/lib/gitlab/hook_data/project_member_builder.rb +++ b/lib/gitlab/hook_data/project_member_builder.rb @@ -37,16 +37,16 @@ module Gitlab project = project_member.project || Project.unscoped.find(project_member.source_id) { - project_name: project.name, - project_path: project.path, - project_path_with_namespace: project.full_path, - project_id: project.id, - user_username: project_member.user.username, - user_name: project_member.user.name, - user_email: project_member.user.email, - user_id: project_member.user.id, - access_level: project_member.human_access, - project_visibility: project.visibility + project_name: project.name, + project_path: project.path, + project_path_with_namespace: project.full_path, + project_id: project.id, + user_username: project_member.user.username, + user_name: project_member.user.name, + user_email: project_member.user.email, + user_id: project_member.user.id, + access_level: project_member.human_access, + project_visibility: project.visibility } end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 30465ff5f74..5b9216c0914 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 39, + 'da_DK' => 38, 'de' => 17, 'en' => 100, 'eo' => 0, - 'es' => 38, + 'es' => 37, 'fil_PH' => 0, 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 32, - 'ko' => 12, + 'ja' => 31, + 'ko' => 17, 'nb_NO' => 26, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 55, - 'ro_RO' => 100, + 'pt_BR' => 56, + 'ro_RO' => 99, 'ru' => 27, 'si_LK' => 10, - 'tr_TR' => 12, + 'tr_TR' => 11, 'uk' => 50, - 'zh_CN' => 99, + 'zh_CN' => 97, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 4abc3da1190..8843b4f5755 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -12,6 +12,7 @@ module Gitlab @methods = config[:methods] || {} @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} + @include_if_exportable = config[:include_if_exportable] || {} end def find_root(model_key) @@ -35,7 +36,8 @@ module Gitlab methods: @methods[model_key], include: resolve_model_tree(model_tree), preload: resolve_preloads(model_key, model_tree), - export_reorder: @export_reorders[model_key] + export_reorder: @export_reorders[model_key], + include_if_exportable: @include_if_exportable[model_key] }.compact end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 1cbfcbdb595..bbec473d29d 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -31,6 +31,8 @@ module Gitlab TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze + attr_reader :relation_name, :importable + def self.create(*args, **kwargs) new(*args, **kwargs).create end diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index ea989487ebd..3c473449ec0 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -58,8 +58,19 @@ module Gitlab records.each_slice(BATCH_SIZE) do |batch| valid_records, invalid_records = batch.partition { |record| record.valid? } - invalid_subrelations << invalid_records relation_object.public_send(relation_name) << valid_records + + # Attempt to save some of the invalid subrelations, as they might be valid after all. + # For example, a merge request `Approval` validates presence of merge_request_id. + # It is not present at a time of calling `#valid?` above, since it's indeed missing. + # However, when saving such subrelation against already persisted merge request + # such validation won't fail (e.g. `merge_request.approvals << Approval.new(user_id: 1)`), + # as we're operating on a merge request that has `id` present. + invalid_records.each do |invalid_record| + relation_object.public_send(relation_name) << invalid_record + + invalid_subrelations << invalid_record unless invalid_record.persisted? + end end end end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index 8df5d52bf77..a08efdf400b 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -27,6 +27,26 @@ included_attributes: - :name namespace_settings: - :prevent_sharing_groups_outside_hierarchy + iterations_cadence: &iterations_cadence_definition + - :group_id + - :created_at + - :updated_at + - :start_date + - :active + - :roll_over + - :title + - :description + - :sequence + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_definition + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations: *iteration_definition excluded_attributes: group: @@ -44,6 +64,23 @@ excluded_attributes: - :max_pages_size epics: - :state_id + iterations_cadence: &iterations_cadence_definition + - :id + - :next_run_date + - :duration_in_weeks + - :iterations_in_advance + - :automatic + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_excluded_definition + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + - :sequence + iterations: *iteration_excluded_definition methods: labels: @@ -83,6 +120,7 @@ ee: - events: - :push_event_payload - :system_note_metadata + - :resource_state_events - boards: - :board_assignee - :milestone @@ -92,3 +130,5 @@ ee: - milestone: - events: - :push_event_payload + - iterations_cadences: + - :iterations diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 082d2346f3d..6507def7d01 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -24,6 +24,29 @@ included_attributes: - :username author: - :name + iterations_cadence: &iterations_cadence_definition + - :group_id + - :created_at + - :updated_at + - :start_date + - :next_run_date + - :duration_in_weeks + - :iterations_in_advance + - :active + - :automatic + - :roll_over + - :title + - :description + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_definition + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations: *iteration_definition excluded_attributes: group: @@ -41,6 +64,18 @@ excluded_attributes: - :extra_shared_runners_minutes_limit epics: - :state_id + iterations_cadence: &iterations_cadence_definition + - :id + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_excluded_definition + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + iterations: *iteration_excluded_definition methods: labels: @@ -79,6 +114,7 @@ ee: - :award_emoji - events: - :push_event_payload + - :resource_state_events - boards: - :board_assignee - :milestone @@ -88,3 +124,5 @@ ee: - milestone: - events: - :push_event_payload + - iterations_cadences: + - :iterations diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 8b39362b6bb..fa9e765b33a 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -68,23 +68,23 @@ module Gitlab def restorer @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - relation_reader: relation_reader, - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader, - importable: @group, + user: @user, + shared: @shared, + relation_reader: relation_reader, + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader, + importable: @group, importable_attributes: @group_attributes, - importable_path: nil + importable_path: nil ) end def create_group(group_hash:, parent_group:) group_params = { - name: group_hash['name'], - path: group_hash['path'], + name: group_hash['name'], + path: group_hash['path'], parent_id: parent_group&.id, visibility_level: sub_group_visibility_level(group_hash, parent_group) } diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 258078d595b..1b8436c4ed9 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -5,10 +5,11 @@ module Gitlab module Group class RelationFactory < Base::RelationFactory OVERRIDES = { - labels: :group_labels, + labels: :group_labels, priorities: :label_priorities, - label: :group_label, - parent: :epic + label: :group_label, + parent: :epic, + iterations_cadences: 'Iterations::Cadence' }.freeze EXISTING_OBJECT_RELATIONS = %i[ @@ -25,7 +26,10 @@ module Gitlab private def setup_models - setup_note if @relation_name == :notes + case @relation_name + when :notes then setup_note + when :'Iterations::Cadence' then setup_iterations_cadence + end update_group_references end @@ -44,6 +48,10 @@ module Gitlab def use_attributes_permitter? false end + + def setup_iterations_cadence + @relation_hash['automatic'] = false + end end end end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index fab677bd772..5a78f2fb531 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -136,9 +136,9 @@ module Gitlab attributes_permitter.permit(importable_class_sym, params) else Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: params, + relation_hash: params, relation_class: importable_class, - excluded_keys: excluded_keys_for_relation(importable_class_sym)) + excluded_keys: excluded_keys_for_relation(importable_class_sym)) end end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 796b9258e57..b4c86c3fc7f 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -46,7 +46,8 @@ module Gitlab group, group_tree, json_writer, - exportable_path: "groups/#{group.id}" + exportable_path: "groups/#{group.id}", + current_user: @current_user ).execute end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 78f43f79072..99396d64779 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -6,11 +6,7 @@ module Gitlab class StreamingSerializer include Gitlab::ImportExport::CommandLineUtil - BATCH_SIZE = 2 - - def self.batch_size(exportable) - BATCH_SIZE - end + BATCH_SIZE = 100 class Raw < String def to_json(*_args) @@ -18,8 +14,9 @@ module Gitlab end end - def initialize(exportable, relations_schema, json_writer, exportable_path:, logger: Gitlab::Export::Logger) + def initialize(exportable, relations_schema, json_writer, current_user:, exportable_path:, logger: Gitlab::Export::Logger) @exportable = exportable + @current_user = current_user @exportable_path = exportable_path @relations_schema = relations_schema @json_writer = json_writer @@ -63,7 +60,7 @@ module Gitlab private - attr_reader :json_writer, :relations_schema, :exportable, :logger + attr_reader :json_writer, :relations_schema, :exportable, :logger, :current_user def serialize_many_relations(key, records, options) log_relation_export(key, records.size) @@ -77,7 +74,7 @@ module Gitlab batch.each do |record| before_read_callback(record) - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) after_read_callback(record) end @@ -87,8 +84,29 @@ module Gitlab json_writer.write_relation_array(@exportable_path, key, enumerator) end + def exportable_json_record(record, options, key) + associations = relations_schema[:include_if_exportable]&.dig(key) + return Raw.new(record.to_json(options)) unless associations && options[:include] + + filtered_options = options.deep_dup + associations.each do |association| + filtered_options[:include].delete_if do |option| + !exportable_json_association?(option, record, association.to_sym) + end + end + + Raw.new(record.to_json(filtered_options)) + end + + def exportable_json_association?(option, record, association) + return true unless option.has_key?(association) + return false unless record.respond_to?(:exportable_association?) + + record.exportable_association?(association, current_user: current_user) + end + def batch(relation, key) - opts = { of: batch_size } + opts = { of: BATCH_SIZE } order_by = reorders(relation, key) # we need to sort issues by non primary key column(relative_position) @@ -115,7 +133,7 @@ module Gitlab enumerator = Enumerator.new do |items| records.each do |record| - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) end end @@ -125,7 +143,7 @@ module Gitlab def serialize_single_relation(key, record, options) log_relation_export(key) - json = Raw.new(record.to_json(options)) + json = exportable_json_record(record, options, key) json_writer.write_relation(@exportable_path, key, json) end @@ -138,10 +156,6 @@ module Gitlab relations_schema[:preload] end - def batch_size - @batch_size ||= self.class.batch_size(@exportable) - end - def reorders(relation, key) export_reorder = relations_schema[:export_reorder]&.dig(key) return unless export_reorder diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index c6b961ea210..cf75a2c7fa8 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -7,7 +7,7 @@ module Gitlab def serialize(exportable, relations_tree) Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree, batch_size: batch_size(exportable)) + .new(exportable, relations_tree) .execute end @@ -18,12 +18,6 @@ module Gitlab File.write(File.join(dir_path, filename), tree_json) end - - private - - def batch_size(exportable) - Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable) - end end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index b1f2a17d4b7..c94549a2b3f 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -139,7 +139,7 @@ module Gitlab end def parsed_hash(member) - Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, relation_class: relation_class) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index c5b8f3fd35b..33e4823f192 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -29,6 +29,9 @@ tree: - resource_label_events: - label: - :priorities + - resource_milestone_events: + - :milestone + - :resource_state_events - designs: - notes: - :author @@ -82,6 +85,9 @@ tree: - resource_label_events: - label: - :priorities + - resource_milestone_events: + - :milestone + - :resource_state_events - :external_pull_requests - ci_pipelines: - notes: @@ -287,6 +293,7 @@ included_attributes: - :forking_access_level - :metrics_dashboard_access_level - :operations_access_level + - :monitor_access_level - :analytics_access_level - :security_and_compliance_access_level - :container_registry_access_level @@ -551,6 +558,7 @@ included_attributes: - :failure_reason - :scheduled_at - :scheduling_type + - :ci_stage ci_pipelines: - :ref - :sha @@ -599,7 +607,6 @@ included_attributes: merge_request_assignees: - :user_id - :created_at - - :state merge_request_reviewers: - :user_id - :created_at @@ -699,6 +706,7 @@ included_attributes: - :metrics_dashboard_access_level - :analytics_access_level - :operations_access_level + - :monitor_access_level - :security_and_compliance_access_level - :container_registry_access_level - :package_registry_access_level @@ -721,6 +729,18 @@ included_attributes: - :build_git_strategy - :build_enabled - :security_and_compliance_enabled + resource_milestone_events: + - :user_id + - :action + - :created_at + - :state + resource_state_events: + - :user_id + - :state + - :created_at + - :source_commit + - :close_after_error_tracking_resolve + - :close_auto_resolve_prometheus_alert # Do not include the following attributes for the models specified. excluded_attributes: @@ -989,6 +1009,46 @@ excluded_attributes: milestone_releases: - :milestone_id - :release_id + resource_milestone_events: + - :id + - :issue_id + - :merge_request_id + - :milestone_id + resource_state_events: + - :id + - :issue_id + - :merge_request_id + - :epic_id + - :source_merge_request_id + iteration: + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + - :sequence + resource_iteration_events: + - :id + - :issue_id + - :merge_request_id + - :iteration_id + iterations_cadence: + - :id + - :last_run_date + - :duration_in_weeks + - :iterations_in_advance + - :automatic + - :group_id + - :created_at + - :updated_at + - :start_date + - :active + - :roll_over + - :description + - :sequence + methods: notes: - :type @@ -1062,6 +1122,11 @@ ee: - epic_issue: - :epic - :issuable_sla + - iteration: + - :iterations_cadence + - resource_iteration_events: + - iteration: + - :iterations_cadence - protected_branches: - :unprotect_access_levels - protected_environments: @@ -1120,5 +1185,44 @@ ee: - :auto_fix_dependency_scanning - :auto_fix_sast project: - - :requirements_enabled - - :requirements_access_level + - :requirements_enabled + - :requirements_access_level + resource_iteration_events: + - :user_id + - :action + - :created_at + iteration: + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations_cadence: + - :title + + preloads: + issues: + epic: + + # When associated resources are from outside the project, you might need to + # validate that a user who is exporting the project or group can access these + # associations. `include_if_exportable` accepts an array of associations for a + # resource. During export, the `exportable_association?` method on the + # resource is called with the association's name and user to validate if + # associated resource can be included in the export. + # + # This definition will call issue's `exportable_association?(:epic_issue, + # current_user: current_user)` method and include issue's epic_issue association + # for each issue only if the method returns true: + # + # Example: + # include_if_exportable: + # project: + # issues: + # - epic_issue + include_if_exportable: + project: + issues: + - :epic_issue diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index 59bb8af750e..89f2b36ea58 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -80,8 +80,8 @@ module Gitlab def import_params { namespace_id: namespace.id, - path: project_path, - file: File.open(file_path) + path: project_path, + file: File.open(file_path) } end diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index bf60d115a25..50a67a746f8 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -21,7 +21,7 @@ module Gitlab end def find - return if epic? && group.nil? + return if group_relation_without_group? return find_diff_commit_user if diff_commit_user? return find_diff_commit if diff_commit? @@ -60,7 +60,7 @@ module Gitlab def prepare_attributes attributes.dup.tap do |atts| - atts.delete('group') unless epic? + atts.delete('group') unless epic? || iteration? if label? atts['type'] = 'ProjectLabel' # Always create project labels @@ -141,6 +141,10 @@ module Gitlab klass == MergeRequestDiffCommit end + def iteration? + klass == Iteration + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -157,7 +161,13 @@ module Gitlab milestone.ensure_project_iid! milestone.save! end + + def group_relation_without_group? + (epic? || iteration?) && group.nil? + end end end end end + +Gitlab::ImportExport::Project::ObjectBuilder.prepend_mod diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index b40827e36f8..8e91adac196 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,8 @@ module Gitlab project, reader.project_tree, json_writer, - exportable_path: 'project' + exportable_path: 'project', + current_user: nil ) end diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb index 6e9548f393a..47196db6f8a 100644 --- a/lib/gitlab/import_export/project/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -5,7 +5,7 @@ module Gitlab module Project class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze + GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze private diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 1b54e4b975e..bd34cd3ff6e 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -50,7 +50,8 @@ module Gitlab reader.project_tree, json_writer, exportable_path: "project", - logger: @logger + logger: @logger, + current_user: @current_user ) Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 4fee779c767..a371930621d 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -4,15 +4,20 @@ module Gitlab module Instrumentation # Aggregates Redis measurements from different request storage sources. class Redis + # Actioncable has it's separate instrumentation, but isn't configurable + # in the same way as all the other instances using a class. ActionCable = Class.new(RedisBase) - Cache = Class.new(RedisBase).enable_redis_cluster_validation - Queues = Class.new(RedisBase) - SharedState = Class.new(RedisBase).enable_redis_cluster_validation - TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation - RateLimiting = Class.new(RedisBase).enable_redis_cluster_validation - Sessions = Class.new(RedisBase).enable_redis_cluster_validation - - STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks, RateLimiting, Sessions].freeze + + STORAGES = ( + Gitlab::Redis::ALL_CLASSES.map do |redis_instance_class| + instrumentation_class = Class.new(RedisBase) + + instrumentation_class.enable_redis_cluster_validation unless redis_instance_class == Gitlab::Redis::Queues + + const_set(redis_instance_class.store_name, instrumentation_class) + instrumentation_class + end << ActionCable + ).freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 0beab008f73..0bd10597f24 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -20,21 +20,19 @@ module Gitlab ::RequestStore[call_duration_key] += duration end - def add_call_details(duration, args) + def add_call_details(duration, commands) return unless Gitlab::PerformanceBar.enabled_for_request? - # redis-rb passes an array (e.g. [[:get, key]]) - return unless args.length == 1 detail_store << { - cmd: args.first, + commands: commands, duration: duration, backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) } end - def increment_request_count + def increment_request_count(amount = 1) ::RequestStore[request_count_key] ||= 0 - ::RequestStore[request_count_key] += 1 + ::RequestStore[request_count_key] += amount end def increment_read_bytes(num_bytes) @@ -78,9 +76,9 @@ module Gitlab self end - def instance_count_request + def instance_count_request(amount = 1) @request_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_requests_total, 'Client side Redis request count, per Redis server') - @request_counter.increment({ storage: storage_key }) + @request_counter.increment({ storage: storage_key }, amount) end def instance_count_exception(ex) diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 14474693ddf..7e2acb91b94 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -13,27 +13,15 @@ module Gitlab end end - def call(*args, &block) - start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined - instrumentation_class.instance_count_request - instrumentation_class.redis_cluster_validate!(args.first) - - super(*args, &block) - rescue ::Redis::BaseError => ex - instrumentation_class.instance_count_exception(ex) - raise ex - ensure - duration = Gitlab::Metrics::System.monotonic_time - start - - unless APDEX_EXCLUDE.include?(command_from_args(args)) - instrumentation_class.instance_observe_duration(duration) + def call(command) + instrument_call([command]) do + super end + end - if ::RequestStore.active? - # These metrics measure total Redis usage per Rails request / job. - instrumentation_class.increment_request_count - instrumentation_class.add_duration(duration) - instrumentation_class.add_call_details(duration, args) + def call_pipeline(pipeline) + instrument_call(pipeline.commands) do + super end end @@ -50,6 +38,31 @@ module Gitlab private + def instrument_call(commands) + start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + instrumentation_class.instance_count_request(commands.size) + + commands.each { |c| instrumentation_class.redis_cluster_validate!(c) } + + yield + rescue ::Redis::BaseError => ex + instrumentation_class.instance_count_exception(ex) + raise ex + ensure + duration = Gitlab::Metrics::System.monotonic_time - start + + unless exclude_from_apdex?(commands) + commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } + end + + if ::RequestStore.active? + # These metrics measure total Redis usage per Rails request / job. + instrumentation_class.increment_request_count(commands.size) + instrumentation_class.add_duration(duration) + instrumentation_class.add_call_details(duration, commands) + end + end + def measure_write_size(command) size = 0 @@ -97,10 +110,8 @@ module Gitlab @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables end - def command_from_args(args) - command = args[0] - command = command[0] if command.is_a?(Array) - command.to_s.downcase + def exclude_from_apdex?(commands) + commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } end end end diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb index 563805fcb01..448ac4c2ae0 100644 --- a/lib/gitlab/issuable/clone/copy_resource_events_service.rb +++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb @@ -49,7 +49,7 @@ module Gitlab event.attributes .except(*blocked_state_event_attributes) .merge(entity_key => new_entity.id, - 'state' => ResourceStateEvent.states[event.state]) + 'state' => ResourceStateEvent.states[event.state]) end end @@ -62,9 +62,9 @@ module Gitlab event.attributes .except('id') .merge(entity_key => new_entity.id, - 'milestone_id' => milestone&.id, - 'action' => ResourceMilestoneEvent.actions[event.action], - 'state' => ResourceMilestoneEvent.states[event.state]) + 'milestone_id' => milestone&.id, + 'action' => ResourceMilestoneEvent.actions[event.action], + 'state' => ResourceMilestoneEvent.states[event.state]) end def copy_events(table_name, events_to_copy) diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb index 60344e4be68..fd41d9eeb5a 100644 --- a/lib/gitlab/jira_import.rb +++ b/lib/gitlab/jira_import.rb @@ -66,11 +66,6 @@ module Gitlab cache_class.write(cache_key, value) end - def self.cache_issue_mapping(issue_id, jira_issue_id, project_id) - cache_key = JiraImport.jira_item_cache_key(project_id, jira_issue_id, :issues) - cache_class.write(cache_key, issue_id) - end - def self.get_import_label_id(project_id) cache_class.read(JiraImport.import_label_cache_key(project_id)) end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 15163bd4a57..0a0a1defd11 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -71,11 +71,11 @@ module Gitlab containers.map do |container| { - selectors: { pod: pod_name, container: container["name"] }, - url: container_exec_url(api_url, namespace, pod_name, container["name"]), + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], - headers: ::Gitlab::Kubernetes.build_header_hash, - created_at: created_at + headers: ::Gitlab::Kubernetes.build_header_hash, + created_at: created_at } end end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index 7d78c8dee25..dd1502bbbcd 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -79,6 +79,20 @@ module Gitlab @users[login] = api.user(login) end + def repository(id) + request(:repository, id).to_h + end + + def repos + repositories = request(:repos, nil) + + if repositories.is_a?(Array) + repositories.map(&:to_h) + else + repositories + end + end + private def api_endpoint diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index c54325bcdf5..01e04fa9c81 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -18,11 +18,11 @@ module Gitlab attrs = { name: name, path: name, - description: repo.description, + description: repo[:description], namespace_id: namespace.id, visibility_level: visibility_level, import_type: type, - import_source: repo.full_name, + import_source: repo[:full_name], import_url: import_url, skip_wiki: skip_wiki }.merge!(extra_attrs) @@ -33,11 +33,11 @@ module Gitlab private def import_url - repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") + repo[:clone_url].sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level - visibility_level = repo.private ? Gitlab::VisibilityLevel::PRIVATE : @namespace.visibility_level + visibility_level = repo[:private] ? Gitlab::VisibilityLevel::PRIVATE : @namespace.visibility_level visibility_level = Gitlab::CurrentSettings.default_project_visibility if Gitlab::CurrentSettings.restricted_visibility_levels.include?(visibility_level) visibility_level @@ -49,7 +49,7 @@ module Gitlab # repository already exist. # def skip_wiki - repo.has_wiki? + repo[:has_wiki] end end end diff --git a/lib/gitlab/mailgun/webhook_processors/failure_logger.rb b/lib/gitlab/mailgun/webhook_processors/failure_logger.rb index a7a85bd1672..fa72abf1311 100644 --- a/lib/gitlab/mailgun/webhook_processors/failure_logger.rb +++ b/lib/gitlab/mailgun/webhook_processors/failure_logger.rb @@ -5,11 +5,12 @@ module Gitlab module WebhookProcessors class FailureLogger < Base def execute - log_failure if permanent_failure? || temporary_failure_over_threshold? + log_failure if permanent_failure_over_threshold? || temporary_failure_over_threshold? end - def permanent_failure? - payload['event'] == 'failed' && payload['severity'] == 'permanent' + def permanent_failure_over_threshold? + payload['event'] == 'failed' && payload['severity'] == 'permanent' && + Gitlab::ApplicationRateLimiter.throttled?(:permanent_email_failure, scope: payload['recipient']) end def temporary_failure_over_threshold? diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 80dff075391..6fe9bb10cdf 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -14,9 +14,9 @@ module Gitlab def save(repositories, group_id) Gitlab::Redis::SharedState.with do |redis| - redis.multi do - redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) - redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + redis.multi do |multi| + multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) end end end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index f635f41ec39..aab58bfa211 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -31,7 +31,7 @@ module Gitlab if job.is_a?(ActionMailer::MailDeliveryJob) { "class" => job.arguments.first, - "jid" => job.job_id + "jid" => job.job_id } else job diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 5a8efa34097..752ab153f37 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -10,9 +10,9 @@ module Gitlab results = {} Gitlab::Redis::Cache.with do |r| - r.pipelined do + r.pipelined do |pipeline| subjects.each do |subject| - results[subject.cache_key] = new(subject).read + results[subject.cache_key] = new(subject).read(pipeline) end end end @@ -34,11 +34,15 @@ module Gitlab end end - def read + def read(pipeline = nil) @loaded = true - Gitlab::Redis::Cache.with do |r| - r.mapped_hmget(markdown_cache_key, *fields) + if pipeline + pipeline.mapped_hmget(markdown_cache_key, *fields) + else + Gitlab::Redis::Cache.with do |r| + r.mapped_hmget(markdown_cache_key, *fields) + end end end diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb index 7163a70a5cb..e20e186cab9 100644 --- a/lib/gitlab/memory/jemalloc.rb +++ b/lib/gitlab/memory/jemalloc.rb @@ -27,21 +27,27 @@ module Gitlab # Write jemalloc stats to the given directory # @param [String] path Directory path the dump will be put into + # @param [String] tmp_dir Directory path the dump will be streaming to. It is moved to `path` when finished. # @param [String] format `json` or `txt` # @param [String] filename_label Optional custom string that will be injected into the file name, e.g. `worker_0` # @return [String] Full path to the resulting dump file - def dump_stats(path:, format: STATS_DEFAULT_FORMAT, filename_label: nil) + def dump_stats(path:, tmp_dir: Dir.tmpdir, format: STATS_DEFAULT_FORMAT, filename_label: nil) verify_format!(format) format_settings = STATS_FORMATS[format] + tmp_file_path = File.join(tmp_dir, file_name(format_settings[:extension], filename_label)) file_path = File.join(path, file_name(format_settings[:extension], filename_label)) with_malloc_stats_print do |stats_print| - File.open(file_path, 'wb') do |io| + File.open(tmp_file_path, 'wb') do |io| write_stats(stats_print, io, format_settings) end end + # On OSX, `with_malloc_stats_print` is no-op, and, as result, no file will be written + return unless File.exist?(tmp_file_path) + + FileUtils.mv(tmp_file_path, file_path) file_path end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index b99bec4ac3e..05f0717d7c3 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -18,12 +18,19 @@ module Gitlab def initialize(reports_path:) @reports_path = reports_path + + # Store report in tmp subdir while it is still streaming. + # This will clearly separate finished reports from the files we are still writing to. + @tmp_dir = File.join(@reports_path, 'tmp') + FileUtils.mkdir_p(@tmp_dir) end def run return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, filename_label: worker_id).tap { cleanup } + Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do + cleanup + end end def active? diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 91edb68ad66..38231fa933b 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -16,8 +16,9 @@ module Gitlab # The duration for which a process may be above a given fragmentation # threshold is computed as `max_strikes * sleep_time_seconds`. class Watchdog - DEFAULT_SLEEP_TIME_SECONDS = 60 - DEFAULT_HEAP_FRAG_THRESHOLD = 0.5 + DEFAULT_SLEEP_TIME_SECONDS = 60 * 5 + DEFAULT_MAX_HEAP_FRAG = 0.5 + DEFAULT_MAX_MEM_GROWTH = 3.0 DEFAULT_MAX_STRIKES = 5 # This handler does nothing. It returns `false` to indicate to the @@ -29,7 +30,7 @@ module Gitlab class NullHandler include Singleton - def on_high_heap_fragmentation(value) + def call # NOP false end @@ -41,7 +42,7 @@ module Gitlab @pid = pid end - def on_high_heap_fragmentation(value) + def call Process.kill(:TERM, @pid) true end @@ -55,7 +56,7 @@ module Gitlab @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) end - def on_high_heap_fragmentation(value) + def call @worker.term true end @@ -63,6 +64,9 @@ module Gitlab # max_heap_fragmentation: # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + # max_mem_growth: + # A multiplier for how much excess private memory a worker can map compared to a reference process + # (itself or the primary in a pre-fork server.) # max_strikes: # How many times the process is allowed to be above max_heap_fragmentation before # a handler is invoked. @@ -71,7 +75,8 @@ module Gitlab def initialize( handler: NullHandler.instance, logger: Logger.new($stdout), - max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_HEAP_FRAG_THRESHOLD, + max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_MAX_HEAP_FRAG, + max_mem_growth: ENV['GITLAB_MEMWD_MAX_MEM_GROWTH']&.to_f || DEFAULT_MAX_MEM_GROWTH, max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, **options) @@ -79,17 +84,37 @@ module Gitlab @handler = handler @logger = logger - @max_heap_fragmentation = max_heap_fragmentation @sleep_time_seconds = sleep_time_seconds @max_strikes = max_strikes + @stats = { + heap_frag: { + max: max_heap_fragmentation, + strikes: 0 + }, + mem_growth: { + max: max_mem_growth, + strikes: 0 + } + } @alive = true - @strikes = 0 init_prometheus_metrics(max_heap_fragmentation) end - attr_reader :strikes, :max_heap_fragmentation, :max_strikes, :sleep_time_seconds + attr_reader :max_strikes, :sleep_time_seconds + + def max_heap_fragmentation + @stats[:heap_frag][:max] + end + + def max_mem_growth + @stats[:mem_growth][:max] + end + + def strikes(stat) + @stats[stat][:strikes] + end def call @logger.info(log_labels.merge(message: 'started')) @@ -97,7 +122,10 @@ module Gitlab while @alive sleep(@sleep_time_seconds) - monitor_heap_fragmentation if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + next unless Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + + monitor_heap_fragmentation + monitor_memory_growth end @logger.info(log_labels.merge(message: 'stopped')) @@ -109,32 +137,73 @@ module Gitlab private - def monitor_heap_fragmentation - heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + def monitor_memory_condition(stat_key) + return unless @alive + + stat = @stats[stat_key] + + ok, labels = yield(stat) - if heap_fragmentation > @max_heap_fragmentation - @strikes += 1 - @heap_frag_violations.increment + if ok + stat[:strikes] = 0 else - @strikes = 0 + stat[:strikes] += 1 + @counter_violations.increment(reason: stat_key.to_s) end - if @strikes > @max_strikes - # If the handler returns true, it means the event is handled and we can shut down. - @alive = !handle_heap_fragmentation_limit_exceeded(heap_fragmentation) - @strikes = 0 + if stat[:strikes] > @max_strikes + @alive = !memory_limit_exceeded_callback(stat_key, labels) + stat[:strikes] = 0 end end - def handle_heap_fragmentation_limit_exceeded(value) - @logger.warn( - log_labels.merge( - message: 'heap fragmentation limit exceeded', - memwd_cur_heap_frag: value - )) - @heap_frag_violations_handled.increment + def monitor_heap_fragmentation + monitor_memory_condition(:heap_frag) do |stat| + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + [ + heap_fragmentation <= stat[:max], + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: heap_fragmentation, + memwd_max_heap_frag: stat[:max] + } + ] + end + end + + def monitor_memory_growth + monitor_memory_condition(:mem_growth) do |stat| + worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] + reference_uss = reference_mem[:uss] + memory_limit = stat[:max] * reference_uss + [ + worker_uss <= memory_limit, + { + message: 'memory limit exceeded', + memwd_uss_bytes: worker_uss, + memwd_ref_uss_bytes: reference_uss, + memwd_max_uss_bytes: memory_limit + } + ] + end + end + + # On pre-fork systems this would be the primary process memory from which workers fork. + # Otherwise it is the current process' memory. + # + # We initialize this lazily because in the initializer the application may not have + # finished booting yet, which would yield an incorrect baseline. + def reference_mem + @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) + end + + def memory_limit_exceeded_callback(stat_key, handler_labels) + all_labels = log_labels.merge(handler_labels) + .merge(memwd_cur_strikes: strikes(stat_key)) + @logger.warn(all_labels) + @counter_violations_handled.increment(reason: stat_key.to_s) - handler.on_high_heap_fragmentation(value) + handler.call end def handler @@ -151,9 +220,7 @@ module Gitlab worker_id: worker_id, memwd_handler_class: handler.class.name, memwd_sleep_time_s: @sleep_time_seconds, - memwd_max_heap_frag: @max_heap_fragmentation, memwd_max_strikes: @max_strikes, - memwd_cur_strikes: @strikes, memwd_rss_bytes: process_rss_bytes } end @@ -174,14 +241,14 @@ module Gitlab @heap_frag_limit.set({}, max_heap_fragmentation) default_labels = { pid: worker_id } - @heap_frag_violations = Gitlab::Metrics.counter( - :gitlab_memwd_heap_frag_violations_total, - 'Total number of times heap fragmentation in a Ruby process exceeded its allowed maximum', + @counter_violations = Gitlab::Metrics.counter( + :gitlab_memwd_violations_total, + 'Total number of times a Ruby process violated a memory threshold', default_labels ) - @heap_frag_violations_handled = Gitlab::Metrics.counter( - :gitlab_memwd_heap_frag_violations_handled_total, - 'Total number of times heap fragmentation violations in a Ruby process were handled', + @counter_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_violations_handled_total, + 'Total number of times Ruby process memory violations were handled', default_labels ) end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index 55d14d6f94a..622b6adec7e 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -40,8 +40,8 @@ module Gitlab def formatted_panel { - title: panel[:title], - type: CHART_TYPE, + title: panel[:title], + type: CHART_TYPE, y_label: '', # Grafana panels do not include a Y-Axis label metrics: panel[:targets].map.with_index do |target, idx| formatted_metric(target, idx) @@ -51,9 +51,9 @@ module Gitlab def formatted_metric(metric, idx) { - id: "#{metric[:legendFormat]}_#{idx}", - query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]), + id: "#{metric[:legendFormat]}_#{idx}", + query_range: format_query(metric), + label: replace_variables(metric[:legendFormat]), prometheus_endpoint_path: prometheus_endpoint_for_metric(metric) }.compact end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb index 4e46eec17d6..3650ddf698a 100644 --- a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb @@ -24,15 +24,15 @@ module Gitlab panel_group[:panels].each do |panel| panel[:metrics].each do |metric| prometheus_metrics << { - project: project, - title: panel[:title], - y_label: panel[:y_label], - query: metric[:query_range] || metric[:query], - unit: metric[:unit], - legend: metric[:label], - identifier: metric[:id], - group: Enums::PrometheusMetric.groups[:custom], - common: false, + project: project, + title: panel[:title], + y_label: panel[:y_label], + query: metric[:query_range] || metric[:query], + unit: metric[:unit], + legend: metric[:label], + identifier: metric[:id], + group: Enums::PrometheusMetric.groups[:custom], + common: false, dashboard_path: dashboard_path }.compact end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb index 588c677ca28..29f1274a097 100644 --- a/lib/gitlab/metrics/dashboard/validator/client.rb +++ b/lib/gitlab/metrics/dashboard/validator/client.rb @@ -34,8 +34,8 @@ module Gitlab def post_schema_validator PostSchemaValidator.new( - project: project, - metric_ids: custom_formats.metric_ids_cache, + project: project, + metric_ids: custom_formats.metric_ids_cache, dashboard_path: dashboard_path ) end diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb index e17f1c13cf0..258b655229e 100644 --- a/lib/gitlab/metrics/exporter/metrics_middleware.rb +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -27,8 +27,8 @@ module Gitlab labels = { method: env['REQUEST_METHOD'].downcase, - path: env['PATH_INFO'].to_s, - code: response.first.to_s + path: env['PATH_INFO'].to_s, + code: response.first.to_s } @requests_total.increment(labels) diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb new file mode 100644 index 00000000000..e37129fed38 --- /dev/null +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module GlobalSearchSlis + class << self + # The following targets are the 99.95th percentile of code searches + # gathered on 24-08-2022 + # from https://log.gprd.gitlab.net/goto/0c89cd80-23af-11ed-8656-f5f2137823ba (internal only) + BASIC_CONTENT_TARGET_S = 7.031 + BASIC_CODE_TARGET_S = 21.903 + ADVANCED_CONTENT_TARGET_S = 4.865 + ADVANCED_CODE_TARGET_S = 13.546 + + def initialize_slis! + if Feature.enabled?(:global_search_custom_slis) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) + end + + return unless Feature.enabled?(:global_search_error_rate_sli) + + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:global_search, possible_labels) + end + + def record_apdex(elapsed:, search_type:, search_level:, search_scope:) + return unless Feature.enabled?(:global_search_custom_slis) + + Gitlab::Metrics::Sli::Apdex[:global_search].increment( + labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), + success: elapsed < duration_target(search_type, search_scope) + ) + end + + def record_error_rate(error:, search_type:, search_level:, search_scope:) + return unless Feature.enabled?(:global_search_error_rate_sli) + + Gitlab::Metrics::Sli::ErrorRate[:global_search].increment( + labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), + error: error + ) + end + + private + + def duration_target(search_type, search_scope) + if search_type == 'basic' && content_search?(search_scope) + BASIC_CONTENT_TARGET_S + elsif search_type == 'basic' && code_search?(search_scope) + BASIC_CODE_TARGET_S + elsif search_type == 'advanced' && content_search?(search_scope) + ADVANCED_CONTENT_TARGET_S + elsif search_type == 'advanced' && code_search?(search_scope) + ADVANCED_CODE_TARGET_S + end + end + + def search_types + %w[basic advanced] + end + + def search_levels + %w[project group global] + end + + def search_scopes + Gitlab::Search::AbuseDetection::ALLOWED_SCOPES + end + + def endpoint_ids + ['SearchController#show', 'GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', + 'GET /api/:version/groups/:id/(-/)search'] + end + + def possible_labels + search_types.flat_map do |search_type| + search_levels.flat_map do |search_level| + search_scopes.flat_map do |search_scope| + endpoint_ids.flat_map do |endpoint_id| + { + search_type: search_type, + search_level: search_level, + search_scope: search_scope, + endpoint_id: endpoint_id + } + end + end + end + end + end + + def labels(search_type:, search_level:, search_scope:) + { + search_type: search_type, + search_level: search_level, + search_scope: search_scope, + endpoint_id: endpoint_id + } + end + + def endpoint_id + ::Gitlab::ApplicationContext.current_context_attribute(:caller_id) + end + + def code_search?(search_scope) + search_scope == 'blobs' + end + + def content_search?(search_scope) + !code_search?(search_scope) + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 848a55e59ff..d818aa43853 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -12,15 +12,15 @@ module Gitlab def init_metrics { - puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), - puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), - puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), - puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), + puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), + puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), + puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), + puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), - puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), - puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), - puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') + puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), + puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), + puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8e002293347..4fe338ffc7f 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -31,16 +31,16 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(metric_name(:file, :descriptors), 'File descriptors used', labels), - process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), - process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), - process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), + file_descriptors: ::Gitlab::Metrics.gauge(metric_name(:file, :descriptors), 'File descriptors used', labels), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), + process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), - process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), - sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), - gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), - heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) + process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), + gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), + heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) } GC.stat.keys.each do |key| diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index e646846face..d7eef722d6e 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -10,8 +10,8 @@ module Gitlab extend self PROC_STAT_PATH = '/proc/self/stat' - PROC_STATUS_PATH = '/proc/self/status' - PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup' + PROC_STATUS_PATH = '/proc/%s/status' + PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' PROC_LIMITS_PATH = '/proc/self/limits' PROC_FD_GLOB = '/proc/self/fd/*' @@ -34,14 +34,14 @@ module Gitlab } end - # Returns the current process' RSS (resident set size) in bytes. - def memory_usage_rss - sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes + # Returns the given process' RSS (resident set size) in bytes. + def memory_usage_rss(pid: 'self') + sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes end - # Returns the current process' USS/PSS (unique/proportional set size) in bytes. - def memory_usage_uss_pss - sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) + # Returns the given process' USS/PSS (unique/proportional set size) in bytes. + def memory_usage_uss_pss(pid: 'self') + sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) .transform_values(&:kilobytes) end diff --git a/lib/gitlab/nav/top_nav_menu_builder.rb b/lib/gitlab/nav/top_nav_menu_builder.rb index 721ae1889b8..dca3432a6a1 100644 --- a/lib/gitlab/nav/top_nav_menu_builder.rb +++ b/lib/gitlab/nav/top_nav_menu_builder.rb @@ -6,9 +6,15 @@ module Gitlab def initialize @primary = [] @secondary = [] + @last_header_added = nil end - def add_primary_menu_item(**args) + def add_primary_menu_item(header: nil, **args) + if header && (header != @last_header_added) + add_menu_header(dest: @primary, title: header) + @last_header_added = header + end + add_menu_item(dest: @primary, **args) end @@ -30,6 +36,12 @@ module Gitlab dest.push(item) end + + def add_menu_header(dest:, **args) + header = ::Gitlab::Nav::TopNavMenuHeader.build(**args) + + dest.push(header) + end end end end diff --git a/lib/gitlab/nav/top_nav_menu_header.rb b/lib/gitlab/nav/top_nav_menu_header.rb new file mode 100644 index 00000000000..520091dbd97 --- /dev/null +++ b/lib/gitlab/nav/top_nav_menu_header.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavMenuHeader + def self.build(title:) + { + type: :header, + title: title + } + end + end + end +end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index 4cb38e6bb9b..75eb0e8a264 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -11,6 +11,7 @@ module Gitlab def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil) { id: id, + type: :item, title: title, active: active, icon: icon, diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index 11ca6a3a3ba..a8e25708107 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -42,11 +42,14 @@ module Gitlab def build menu = @menu_builder.build + hide_menu_text = Feature.enabled?(:new_navbar_layout) + menu.merge({ views: @views, shortcuts: @shortcuts, - activeTitle: _('Menu') - }) + menuTitle: (_('Menu') unless hide_menu_text), + menuTooltip: (_('Main menu') if hide_menu_text) + }.compact) end end end diff --git a/lib/gitlab/no_cache_headers.rb b/lib/gitlab/no_cache_headers.rb index f80ca2c1369..2d03741480d 100644 --- a/lib/gitlab/no_cache_headers.rb +++ b/lib/gitlab/no_cache_headers.rb @@ -4,8 +4,8 @@ module Gitlab module NoCacheHeaders DEFAULT_GITLAB_NO_CACHE_HEADERS = { 'Cache-Control' => "#{ActionDispatch::Http::Cache::Response::DEFAULT_CACHE_CONTROL}, no-store, no-cache", - 'Pragma' => 'no-cache', # HTTP 1.0 compatibility - 'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT' + 'Pragma' => 'no-cache', # HTTP 1.0 compatibility + 'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT' }.freeze def no_cache_headers diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 1f1061fe4f1..d4de2791195 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -38,7 +38,7 @@ module Gitlab if finder.is_a?(BranchesFinder) Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project) + true elsif finder.is_a?(::Repositories::TreeFinder) Feature.enabled?(:repository_tree_gitaly_pagination, project) else @@ -52,7 +52,7 @@ module Gitlab if finder.is_a?(BranchesFinder) Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project) + true elsif finder.is_a?(::Repositories::TreeFinder) Feature.enabled?(:repository_tree_gitaly_pagination, project) else diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 302e7b406b1..d1fe1d2dfc1 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -213,7 +213,7 @@ module Gitlab attr_reader :reversed_order_expression, :nullable, :distinct def calculate_reversed_order(order_expression) - unless AREL_ORDER_CLASSES.has_key?(order_expression.class) # Arel can reverse simple orders + unless order_expression.is_a?(Arel::Nodes::Ordering) raise "Couldn't determine reversed order for `#{order_expression}`, please provide the `reversed_order_expression` parameter." end @@ -229,10 +229,10 @@ module Gitlab end def parse_order_direction(order_expression, order_direction) - transformed_order_direction = if order_direction.nil? && AREL_ORDER_CLASSES[order_expression.class] - AREL_ORDER_CLASSES[order_expression.class] - elsif order_direction.present? + transformed_order_direction = if order_direction.present? order_direction.to_s.downcase.to_sym + elsif order_expression.is_a?(Arel::Nodes::Ordering) + AREL_ORDER_CLASSES[order_expression.class] || AREL_ORDER_CLASSES[order_expression.value.class] end unless REVERSED_ORDER_DIRECTIONS.has_key?(transformed_order_direction) diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb new file mode 100644 index 00000000000..630c364d455 --- /dev/null +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Patch to address https://github.com/ondrejbartas/sidekiq-cron/issues/361 +# This restores the poll interval to v1.2.0 behavior +# https://github.com/ondrejbartas/sidekiq-cron/blob/v1.2.0/lib/sidekiq/cron/poller.rb#L36-L38 +# This patch only applies to v1.4.0 +require 'sidekiq/cron/version' + +if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.4.0') + raise 'New version of sidekiq-cron detected, please remove or update this patch' +end + +module Gitlab + module Patch + module SidekiqCronPoller + def poll_interval_average + Sidekiq.options[:poll_interval] || Sidekiq::Cron::POLL_INTERVAL + end + end + end +end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 189627506f3..4883c649a62 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -207,19 +207,22 @@ module Gitlab desc { _('Add Zoom meeting') } explanation { _('Adds a Zoom meeting.') } - params '<Zoom URL>' + params do + zoom_link_params + end types Issue condition do @zoom_service = zoom_link_service + @zoom_service.can_add_link? end - parse_params do |link| - @zoom_service.parse_link(link) + parse_params do |link_params| + @zoom_service.parse_link(link_params) end - command :zoom do |link| - result = @zoom_service.add_link(link) + command :zoom do |link, link_text = nil| + result = add_zoom_link(link, link_text) @execution_message[:zoom] = result.message - @updates.merge!(result.payload) if result.payload + merge_updates(result, @updates) end desc { _('Remove Zoom meeting') } @@ -315,12 +318,52 @@ module Gitlab @updates[:remove_contacts] = contact_emails.split(' ') end - private - - def zoom_link_service - ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + desc { _('Add a timeline event to incident') } + explanation { _('Adds a timeline event to incident.') } + params '<timeline comment> | <date(YYYY-MM-DD)> <time(HH:MM)>' + types Issue + condition do + quick_action_target.incident? && + current_user.can?(:admin_incident_management_timeline_event, quick_action_target) + end + parse_params do |event_params| + Gitlab::QuickActions::TimelineTextAndDateTimeSeparator.new(event_params).execute + end + command :timeline do |event_text, date_time| + if event_text && date_time + timeline_event = timeline_event_create_service(event_text, date_time).execute + + @execution_message[:timeline] = + if timeline_event.success? + _('Timeline event added successfully.') + else + _('Something went wrong while adding timeline event.') + end + end end end + + private + + def zoom_link_service + ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + end + + def zoom_link_params + '<Zoom URL>' + end + + def add_zoom_link(link, _link_text) + zoom_link_service.add_link(link) + end + + def merge_updates(result, update_hash) + update_hash.merge!(result.payload) if result.payload + end + + def timeline_event_create_service(event_text, event_date_time) + ::IncidentManagement::TimelineEvents::CreateService.new(quick_action_target, current_user, { note: event_text, occurred_at: event_date_time, editable: true }) + end end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 3cb01db1491..d38b81bff0b 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -88,33 +88,21 @@ module Gitlab @execution_message[:rebase] = _('Scheduled a rebase of branch %{branch}.') % { branch: branch } end - desc { _('Toggle the Draft status') } + desc { _('Set the Draft status') } explanation do - noun = quick_action_target.to_ability_name.humanize(capitalize: false) - if quick_action_target.draft? - _("Marks this %{noun} as ready.") - else - _("Marks this %{noun} as a draft.") - end % { noun: noun } + draft_action_message(_("Marks")) end execution_message do - noun = quick_action_target.to_ability_name.humanize(capitalize: false) - if quick_action_target.draft? - _("Marked this %{noun} as ready.") - else - _("Marked this %{noun} as a draft.") - end % { noun: noun } + draft_action_message(_("Marked")) end types MergeRequest condition do quick_action_target.respond_to?(:draft?) && - # Allow it to mark as draft on MR creation page or through MR notes - # (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end command :draft do - @updates[:wip_event] = quick_action_target.draft? ? 'ready' : 'draft' + @updates[:wip_event] = draft_action_command end desc { _('Set the Ready status') } @@ -317,6 +305,25 @@ module Gitlab end end + def draft_action_message(verb) + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if !quick_action_target.draft? + _("%{verb} this %{noun} as a draft.") + elsif Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project) + _("%{verb} this %{noun} as ready.") + else + _("No change to this %{noun}'s draft status.") + end % { verb: verb, noun: noun } + end + + def draft_action_command + if Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project) + quick_action_target.draft? ? 'ready' : 'draft' + else + 'draft' + end + end + def merge_orchestration_service @merge_orchestration_service ||= ::MergeRequests::MergeOrchestrationService.new(project, current_user) end diff --git a/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb new file mode 100644 index 00000000000..e8002656ff5 --- /dev/null +++ b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + class TimelineTextAndDateTimeSeparator + DATETIME_REGEX = %r{(\d{2,4}[\-.]\d{1,2}[\-.]\d{1,2} \d{1,2}:\d{2})}.freeze + MIXED_DELIMITER = %r{([/.])}.freeze + TIME_REGEX = %r{(\d{1,2}:\d{2})}.freeze + + def initialize(timeline_event_arg) + @timeline_event_arg = timeline_event_arg + @timeline_text = get_text + @timeline_date_string = get_raw_date_string + end + + def execute + return if @timeline_event_arg.blank? + return if date_contains_mixed_delimiters? + return [@timeline_text, get_current_date_time] unless date_time_present? + return unless valid_date? + + [@timeline_text, get_actual_date_time] + end + + private + + def get_text + @timeline_event_arg.split('|')[0]&.strip + end + + def get_raw_date_string + @timeline_event_arg.split('|')[1]&.strip + end + + def get_current_date_time + DateTime.current.strftime("%Y-%m-%d %H:%M:00 UTC") + end + + def get_actual_date_time + DateTime.parse(@timeline_date_string) + end + + def date_time_present? + DATETIME_REGEX =~ @timeline_date_string || TIME_REGEX =~ @timeline_date_string + end + + def date_contains_mixed_delimiters? + MIXED_DELIMITER =~ @timeline_date_string + end + + def valid_date? + get_actual_date_time + rescue Date::Error + nil + end + end + end +end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 7ccbeadfd8a..2de3c07712f 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -15,8 +15,10 @@ module Gitlab keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) - redis.pipelined do - keys.each_slice(1000) { |subset| redis.unlink(*subset) } + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.pipelined do |pipeline| + keys.each_slice(1000) { |subset| pipeline.unlink(*subset) } + end end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..8857b544364 --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + # List all Gitlab::Redis::Wrapper descendants that are backed by an actual + # separate redis instance here. + # + # This will make sure the connection pool is initialized on application boot in + # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. + ALL_CLASSES = [ + Gitlab::Redis::Cache, + Gitlab::Redis::Queues, + Gitlab::Redis::RateLimiting, + Gitlab::Redis::Sessions, + Gitlab::Redis::SharedState, + Gitlab::Redis::TraceChunks + ].freeze + end +end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 4ab1024d528..043f14630d5 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -12,7 +12,7 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, - expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 2.weeks).to_i # Cache should not grow forever + expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i # Cache should not grow forever } end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index cdd2ac6100e..a7c36786d2d 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -267,7 +267,7 @@ module Gitlab def same_redis_store? strong_memoize(:same_redis_store) do - # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>" + # <Redis client v4.7.1 for unix:///path_to/redis/redis.socket/5>" primary_store.inspect == secondary_store.inspect end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index 430f3e8d162..1ecdf506208 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -83,14 +83,14 @@ module Gitlab full_key = cache_key(key) with do |redis| - results = redis.pipelined do + results = redis.pipelined do |pipeline| # Set each hash key to the provided value hash.each do |h_key, h_value| - redis.hset(full_key, h_key, h_value) + pipeline.hset(full_key, h_key, h_value) end # Update the expiry time for this hset - redis.expire(full_key, expires_in) + pipeline.expire(full_key, expires_in) end results.all? diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 3061fb96190..33c7d96c45b 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -21,14 +21,14 @@ module Gitlab full_key = cache_key(key) with do |redis| - redis.multi do - redis.unlink(full_key) + redis.multi do |multi| + multi.unlink(full_key) # Splitting into groups of 1000 prevents us from creating a too-long # Redis command - value.each_slice(1000) { |subset| redis.sadd(full_key, subset) } + value.each_slice(1000) { |subset| multi.sadd(full_key, subset) } - redis.expire(full_key, expires_in) + multi.expire(full_key, expires_in) end end @@ -39,9 +39,9 @@ module Gitlab full_key = cache_key(key) smembers, exists = with do |redis| - redis.multi do - redis.smembers(full_key) - redis.exists(full_key) + redis.multi do |multi| + multi.smembers(full_key) + multi.exists(full_key) end end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index a84a6ac2d14..258c904290d 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,6 +6,7 @@ module Gitlab module RequestForgeryProtection + # rubocop:disable Rails/ApplicationController class Controller < ActionController::Base protect_from_forgery with: :exception, prepend: true @@ -31,5 +32,6 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end + # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 2450ad88bbb..ec514adafc8 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -151,48 +151,6 @@ module Gitlab model.logger = old_loggers[connection_name] end end - - module Ci - class DailyBuildGroupReportResult - DEFAULT_BRANCH = 'master' - COUNT_OF_DAYS = 5 - - def initialize(project) - @project = project - @last_pipeline = project.last_pipeline - end - - def seed - COUNT_OF_DAYS.times do |count| - date = Time.now.utc - count.day - create_report(date) - end - end - - private - - attr_reader :project, :last_pipeline - - def create_report(date) - last_pipeline.builds.uniq(&:group_name).each do |build| - ::Ci::DailyBuildGroupReportResult.create( - project: project, - last_pipeline: last_pipeline, - date: date, - ref_path: last_pipeline.source_ref_path, - group_name: build.group_name, - data: { - 'coverage' => rand(20..99) - }, - group: project.group, - default_branch: last_pipeline.default_branch? - ) - rescue ActiveRecord::RecordNotUnique - return false - end - end - end - end end end # :nocov: diff --git a/lib/gitlab/seeders/ci/daily_build_group_report_result.rb b/lib/gitlab/seeders/ci/daily_build_group_report_result.rb new file mode 100644 index 00000000000..10ec65f6bf4 --- /dev/null +++ b/lib/gitlab/seeders/ci/daily_build_group_report_result.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + class DailyBuildGroupReportResult + DEFAULT_BRANCH = 'master' + COUNT_OF_DAYS = 5 + + def initialize(project) + @project = project + @last_pipeline = project.last_pipeline + end + + def seed + COUNT_OF_DAYS.times do |count| + date = Time.now.utc - count.day + create_report(date) + end + end + + private + + attr_reader :project, :last_pipeline + + def create_report(date) + last_pipeline.builds.uniq(&:group_name).each do |build| + ::Ci::DailyBuildGroupReportResult.create( + project: project, + last_pipeline: last_pipeline, + date: date, + ref_path: last_pipeline.source_ref_path, + group_name: build.group_name, + data: { + 'coverage' => rand(20..99) + }, + group: project.group, + default_branch: last_pipeline.default_branch? + ) + rescue ActiveRecord::RecordNotUnique + return false + end + end + end + end + end +end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 896e7e3f65e..23c23393bc8 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -33,10 +33,10 @@ module Gitlab def write(key, value) with do |redis| - redis.pipelined do - redis.sadd(cache_key(key), value) + redis.pipelined do |pipeline| + pipeline.sadd(cache_key(key), value) - redis.expire(cache_key(key), expires_in) + pipeline.expire(cache_key(key), expires_in) end end @@ -57,9 +57,9 @@ module Gitlab full_key = cache_key(key) with do |redis| - redis.multi do - redis.sismember(full_key, value) - redis.exists(full_key) + redis.multi do |multi| + multi.sismember(full_key, value) + multi.exists(full_key) end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index d26e1a34a9f..b167afe589a 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -70,7 +70,9 @@ module Gitlab link_path = File.join(shell_path, '.gitlab_shell_secret') if File.exist?(shell_path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) + # It could happen that link_path is a broken symbolic link. + # In that case !File.exist?(link_path) is true, but we still want to overwrite the (broken) symbolic link. + FileUtils.ln_sf(secret_file, link_path) end end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index ca92fed9c40..24e2eca420e 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -41,11 +41,11 @@ module Gitlab def init_metrics { - sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), + sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'), sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'), - sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker'), - sidekiq_memory_killer_running_jobs: ::Gitlab::Metrics.counter(:sidekiq_memory_killer_running_jobs_total, 'Current running jobs when limit was reached') + sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker'), + sidekiq_memory_killer_running_jobs: ::Gitlab::Metrics.counter(:sidekiq_memory_killer_running_jobs_total, 'Current running jobs when limit was reached') } end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 7533770e254..ab126ea4749 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -112,10 +112,12 @@ module Gitlab end def delete! - with_redis do |redis| - redis.multi do |multi| - multi.del(idempotency_key, deduplicated_flag_key) - delete_wal_locations!(multi) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + with_redis do |redis| + redis.multi do |multi| + multi.del(idempotency_key, deduplicated_flag_key) + delete_wal_locations!(multi) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 180cdad916b..3dd5355d3a3 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -22,21 +22,21 @@ module Gitlab def metrics { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), + sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), - sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), - sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all), - sidekiq_mem_total_bytes: ::Gitlab::Metrics.gauge(:sidekiq_mem_total_bytes, 'Number of bytes allocated for both objects consuming an object slot and objects that required a malloc', {}, :all) + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), + sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), + sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all), + sidekiq_mem_total_bytes: ::Gitlab::Metrics.gauge(:sidekiq_mem_total_bytes, 'Number of bytes allocated for both objects consuming an object slot and objects that required a malloc', {}, :all) } end diff --git a/lib/gitlab/sidekiq_versioning.rb b/lib/gitlab/sidekiq_versioning.rb index 80c0b7650f3..28c9714f82f 100644 --- a/lib/gitlab/sidekiq_versioning.rb +++ b/lib/gitlab/sidekiq_versioning.rb @@ -10,11 +10,7 @@ module Gitlab if queues.any? Sidekiq.redis do |conn| - conn.pipelined do - queues.each do |queue| - conn.sadd('queues', queue) - end - end + conn.sadd('queues', queues) end end rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index d28b5fb509a..55497c5e365 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -87,16 +87,16 @@ module Gitlab { attachments: [ { - title: "#{issue.title} · #{issue.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url(only_path: false), - fallback: fallback_message, - pretext: custom_pretext, - text: text, - color: color(resource), - fields: fields, - mrkdwn_in: fields_with_markdown + title: "#{issue.title} · #{issue.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url(only_path: false), + fallback: fallback_message, + pretext: custom_pretext, + text: text, + color: color(resource), + fields: fields, + mrkdwn_in: fields_with_markdown } ] } diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 40b01552244..0b9f3baa4de 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -33,33 +33,50 @@ module Gitlab @endpoint_url = @endpoint_url.sub(URL_SCHEME_REGEX, '') end - def issue_spam?(spam_issue:, user:, context: {}) - issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) + def spam?(spammable:, user:, context: {}, extra_features: {}) + metadata = { 'authorization' => Gitlab::CurrentSettings.spam_check_api_key || '' } + protobuf_args = { spammable: spammable, user: user, context: context, extra_features: extra_features } + + pb, grpc_method = build_protobuf(**protobuf_args) + response = grpc_method.call(pb, metadata: metadata) - response = grpc_client.check_for_spam_issue(issue, - metadata: { 'authorization' => - Gitlab::CurrentSettings.spam_check_api_key }) verdict = convert_verdict_to_gitlab_constant(response.verdict) [verdict, response.extra_attributes.to_h, response.error] end private + def get_spammable_mappings(spammable) + case spammable + when Issue + [::Spamcheck::Issue, grpc_client.method(:check_for_spam_issue)] + when Snippet + [::Spamcheck::Snippet, grpc_client.method(:check_for_spam_snippet)] + else + raise ArgumentError, "Not a spammable type: #{spammable.class.name}" + end + end + def convert_verdict_to_gitlab_constant(verdict) VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(verdict), verdict) end - def build_issue_protobuf(issue:, user:, context:) - issue_pb = ::Spamcheck::Issue.new - issue_pb.title = issue.spam_title || '' - issue_pb.description = issue.spam_description || '' - issue_pb.created_at = convert_to_pb_timestamp(issue.created_at) if issue.created_at - issue_pb.updated_at = convert_to_pb_timestamp(issue.updated_at) if issue.updated_at - issue_pb.user_in_project = user.authorized_project?(issue.project) - issue_pb.project = build_project_protobuf(issue) - issue_pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) - issue_pb.user = build_user_protobuf(user) - issue_pb + def build_protobuf(spammable:, user:, context:, extra_features:) + protobuf_class, grpc_method = get_spammable_mappings(spammable) + pb = protobuf_class.new(**extra_features) + pb.title = spammable.spam_title || '' + pb.description = spammable.spam_description || '' + pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at + pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at + pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) + pb.user = build_user_protobuf(user) + + unless spammable.project.nil? + pb.user_in_project = user.authorized_project?(spammable.project) + pb.project = build_project_protobuf(spammable) + end + + [pb, grpc_method] end def build_user_protobuf(user) diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 7ef1be6ff44..7494f0584d0 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -22,6 +22,10 @@ module Gitlab "payment_method_validation" end + def self.registration_validation_form_id + "cc_registration_validation" + end + def self.registration_validation_form_url "#{self.subscriptions_url}/payment_forms/cc_registration_validation" end @@ -90,3 +94,4 @@ Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.paymen Gitlab::SubscriptionPortal::PAYMENT_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.payment_validation_form_id.freeze Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze +Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.registration_validation_form_id.freeze diff --git a/lib/gitlab/template/gitignore_template.rb b/lib/gitlab/template/gitignore_template.rb index 72a1b7460c2..d8e0ec82410 100644 --- a/lib/gitlab/template/gitignore_template.rb +++ b/lib/gitlab/template/gitignore_template.rb @@ -11,7 +11,7 @@ module Gitlab def categories { "Languages" => '', - "Global" => 'Global' + "Global" => 'Global' } end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 3b46b4c5498..45f836f10d3 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -10,6 +10,8 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] + action = action.to_s + tracker.event(category, action, label: label, property: property, value: value, context: contexts) rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 72df8b423df..ba3176ca6e7 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -6,7 +6,7 @@ module Gitlab include ::MarkupHelper CACHE_EXPIRE_IN = 1.hour - MAX_OFFSET = 2**31 + MAX_OFFSET = 2**31 - 1 attr_reader :commit, :project, :path, :offset, :limit, :user, :resolved_commits @@ -35,6 +35,8 @@ module Gitlab # - commit_path: URI of the commit in the web interface # - commit_title_html: Rendered commit title def summarize + return [] if offset < 0 + commits_hsh = fetch_last_cached_commits_list prerender_commit_full_titles!(commits_hsh.values) diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index deab2cd43a6..712512d0e02 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -5,27 +5,10 @@ module Gitlab class MigrationHelper attr_reader :logger - CATEGORIES = [%w(AvatarUploader Project :avatar), - %w(AvatarUploader Group :avatar), - %w(AvatarUploader User :avatar), - %w(AttachmentUploader Note :attachment), - %w(AttachmentUploader Appearance :logo), - %w(AttachmentUploader Appearance :header_logo), - %w(FaviconUploader Appearance :favicon), - %w(FileUploader Project), - %w(PersonalFileUploader Snippet), - %w(NamespaceFileUploader Snippet), - %w(DesignManagement::DesignV432x230Uploader DesignManagement::Action :image_v432x230), - %w(FileUploader MergeRequest)].freeze - def initialize(args, logger) prepare_variables(args, logger) end - def self.categories - CATEGORIES - end - def migrate_to_remote_storage @to_store = ObjectStorage::Store::REMOTE @@ -45,17 +28,14 @@ module Gitlab end def prepare_variables(args, logger) - @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym - @uploader_class = args.uploader_class.constantize - @model_class = args.model_class.constantize + @mounted_as = args.mounted_as&.gsub(':', '') + @uploader_class = args.uploader_class + @model_class = args.model_class&.constantize @logger = logger end def enqueue_batch(batch, index) - job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, - @model_class, - @mounted_as, - @to_store) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, @to_store) logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job) rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e # continue for the next batch @@ -66,10 +46,12 @@ module Gitlab def uploads(store_type = [nil, ObjectStorage::Store::LOCAL]) Upload.class_eval { include EachBatch } unless Upload < EachBatch - Upload - .where(store: store_type, - uploader: @uploader_class.to_s, - model_type: @model_class.base_class.sti_name) + uploads = Upload.where(store: store_type) + uploads = uploads.where(uploader: @uploader_class) if @uploader_class.present? + uploads = uploads.where(model_type: @model_class.base_class.sti_name) if @model_class.present? + uploads = uploads.where(mount_point: @mounted_as) if @mounted_as.present? + + uploads end # rubocop:enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb index c0d53b1b21a..67dc1455b23 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -20,15 +20,20 @@ module Gitlab private def relation - return super.where(source_type: source_type) if source_type.present? # rubocop: disable CodeReuse/ActiveRecord - - super + scope = super + scope = scope.where(source_type: source_type) if source_type.present? + scope = scope.where(status: status) if status.present? + scope end def source_type options[:source_type].to_s end + def status + options[:status] + end + def allowed_source_types BulkImports::Entity.source_types.keys.map(&:to_s) end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_user_auth_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_user_auth_metric.rb new file mode 100644 index 00000000000..1de93ce6dfa --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_user_auth_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUserAuthMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + AuthenticationEvent.success + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb index a25bad2436b..26d963e2407 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -11,37 +11,49 @@ module Gitlab # instrumentation_class: RedisMetric # options: # event: pushes - # counter_class: SourceCodeCounter + # prefix: source_code # class RedisMetric < BaseMetric + include Gitlab::UsageDataCounters::RedisCounter + + USAGE_PREFIX = "USAGE_" + def initialize(time_frame:, options: {}) super raise ArgumentError, "'event' option is required" unless metric_event.present? - raise ArgumentError, "'counter class' option is required" unless counter_class.present? + raise ArgumentError, "'prefix' option is required" unless prefix.present? end def metric_event options[:event] end - def counter_class_name - options[:counter_class] + def prefix + options[:prefix] end - def counter_class - "Gitlab::UsageDataCounters::#{counter_class_name}".constantize + def include_usage_prefix? + options.fetch(:include_usage_prefix, true) end def value redis_usage_data do - counter_class.read(metric_event) + total_count(redis_key) end end def suggested_name Gitlab::Usage::Metrics::NameSuggestion.for(:redis) end + + private + + def redis_key + key = "#{prefix}_#{metric_event}".upcase + key.prepend(USAGE_PREFIX) if include_usage_prefix? + key + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6f36a09fe48..e2232dc5e2a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -137,7 +137,7 @@ module Gitlab projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), - projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), + projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.of_report_type(:terraform), :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), protected_branches: count(ProtectedBranch), protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])), @@ -146,7 +146,7 @@ module Gitlab personal_snippets: count(PersonalSnippet), project_snippets: count(ProjectSnippet), suggestions: count(Suggestion), - terraform_reports: count(::Ci::JobArtifact.terraform_reports), + terraform_reports: count(::Ci::JobArtifact.of_report_type(:terraform)), terraform_states: count(::Terraform::State), todos: count(Todo), uploads: count(Upload), @@ -268,7 +268,7 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters - Gitlab::UsageDataCounters.counters + Gitlab::UsageDataCounters.unmigrated_counters end def components_usage_data diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index 224897ed758..eae1c593a8f 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -3,29 +3,38 @@ module Gitlab module UsageDataCounters COUNTERS = [ - PackageEventCounter, WikiPageCounter, - WebIdeCounter, NoteCounter, SnippetCounter, SearchCounter, CycleAnalyticsCounter, ProductivityAnalyticsCounter, SourceCodeCounter, + KubernetesAgentCounter, + MergeRequestWidgetExtensionCounter + ].freeze + + COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES = [ + PackageEventCounter, MergeRequestCounter, DesignsCounter, - KubernetesAgentCounter, DiffsCounter, ServiceUsageDataCounter, - MergeRequestWidgetExtensionCounter + WebIdeCounter ].freeze UsageDataCounterError = Class.new(StandardError) UnknownEvent = Class.new(UsageDataCounterError) class << self + def unmigrated_counters + # we are using the #counters method instead of the COUNTERS const + # to make sure it's working correctly for `ee` version of UsageDataCounters + counters - self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + end + def counters - self::COUNTERS + self::COUNTERS + self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES end def count(event_name) diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 4ab310a2519..5d2ab5eaf74 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -10,7 +10,9 @@ module Gitlab::UsageDataCounters def redis_key(event) require_known_event(event) - "USAGE_#{prefix}_#{event}".upcase + usage_prefix = Gitlab::Usage::Metrics::Instrumentations::RedisMetric::USAGE_PREFIX + + "#{usage_prefix}#{prefix}_#{event}".upcase end def count(event) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index a5db8ba4dcc..f0cb9bcbe94 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -20,11 +20,7 @@ module Gitlab CATEGORIES_FOR_TOTALS = %w[ analytics - code_review compliance - deploy_token_packages - ecosystem - epic_boards_usage epics_usage error_tracking ide_edit @@ -32,11 +28,13 @@ module Gitlab issues_edit pipeline_authoring quickactions - user_packages ].freeze CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ ci_users + deploy_token_packages + code_review + ecosystem error_tracking ide_edit importer @@ -49,6 +47,7 @@ module Gitlab source_code terraform testing + user_packages work_items ].freeze diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 316d9bb3dc1..dda72f7fa3b 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -36,95 +36,118 @@ module Gitlab ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self - def track_issue_created_action(author:) + def track_issue_created_action(author:, project:) + track_snowplow_action(ISSUE_CREATED, author, project) track_unique_action(ISSUE_CREATED, author) end - def track_issue_title_changed_action(author:) + def track_issue_title_changed_action(author:, project:) + track_snowplow_action(ISSUE_TITLE_CHANGED, author, project) track_unique_action(ISSUE_TITLE_CHANGED, author) end - def track_issue_description_changed_action(author:) + def track_issue_description_changed_action(author:, project:) + track_snowplow_action(ISSUE_DESCRIPTION_CHANGED, author, project) track_unique_action(ISSUE_DESCRIPTION_CHANGED, author) end - def track_issue_assignee_changed_action(author:) + def track_issue_assignee_changed_action(author:, project:) + track_snowplow_action(ISSUE_ASSIGNEE_CHANGED, author, project) track_unique_action(ISSUE_ASSIGNEE_CHANGED, author) end - def track_issue_made_confidential_action(author:) + def track_issue_made_confidential_action(author:, project:) + track_snowplow_action(ISSUE_MADE_CONFIDENTIAL, author, project) track_unique_action(ISSUE_MADE_CONFIDENTIAL, author) end - def track_issue_made_visible_action(author:) + def track_issue_made_visible_action(author:, project:) + track_snowplow_action(ISSUE_MADE_VISIBLE, author, project) track_unique_action(ISSUE_MADE_VISIBLE, author) end - def track_issue_closed_action(author:) + def track_issue_closed_action(author:, project:) + track_snowplow_action(ISSUE_CLOSED, author, project) track_unique_action(ISSUE_CLOSED, author) end - def track_issue_reopened_action(author:) + def track_issue_reopened_action(author:, project:) + track_snowplow_action(ISSUE_REOPENED, author, project) track_unique_action(ISSUE_REOPENED, author) end - def track_issue_label_changed_action(author:) + def track_issue_label_changed_action(author:, project:) + track_snowplow_action(ISSUE_LABEL_CHANGED, author, project) track_unique_action(ISSUE_LABEL_CHANGED, author) end - def track_issue_milestone_changed_action(author:) + def track_issue_milestone_changed_action(author:, project:) + track_snowplow_action(ISSUE_MILESTONE_CHANGED, author, project) track_unique_action(ISSUE_MILESTONE_CHANGED, author) end - def track_issue_cross_referenced_action(author:) + def track_issue_cross_referenced_action(author:, project:) + track_snowplow_action(ISSUE_CROSS_REFERENCED, author, project) track_unique_action(ISSUE_CROSS_REFERENCED, author) end - def track_issue_moved_action(author:) + def track_issue_moved_action(author:, project:) + track_snowplow_action(ISSUE_MOVED, author, project) track_unique_action(ISSUE_MOVED, author) end - def track_issue_related_action(author:) + def track_issue_related_action(author:, project:) + track_snowplow_action(ISSUE_RELATED, author, project) track_unique_action(ISSUE_RELATED, author) end - def track_issue_unrelated_action(author:) + def track_issue_unrelated_action(author:, project:) + track_snowplow_action(ISSUE_UNRELATED, author, project) track_unique_action(ISSUE_UNRELATED, author) end - def track_issue_marked_as_duplicate_action(author:) + def track_issue_marked_as_duplicate_action(author:, project:) + track_snowplow_action(ISSUE_MARKED_AS_DUPLICATE, author, project) track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author) end - def track_issue_locked_action(author:) + def track_issue_locked_action(author:, project:) + track_snowplow_action(ISSUE_LOCKED, author, project) track_unique_action(ISSUE_LOCKED, author) end - def track_issue_unlocked_action(author:) + def track_issue_unlocked_action(author:, project:) + track_snowplow_action(ISSUE_UNLOCKED, author, project) track_unique_action(ISSUE_UNLOCKED, author) end - def track_issue_designs_added_action(author:) + def track_issue_designs_added_action(author:, project:) + track_snowplow_action(ISSUE_DESIGNS_ADDED, author, project) track_unique_action(ISSUE_DESIGNS_ADDED, author) end - def track_issue_designs_modified_action(author:) + def track_issue_designs_modified_action(author:, project:) + track_snowplow_action(ISSUE_DESIGNS_MODIFIED, author, project) track_unique_action(ISSUE_DESIGNS_MODIFIED, author) end - def track_issue_designs_removed_action(author:) + def track_issue_designs_removed_action(author:, project:) + track_snowplow_action(ISSUE_DESIGNS_REMOVED, author, project) track_unique_action(ISSUE_DESIGNS_REMOVED, author) end - def track_issue_due_date_changed_action(author:) + def track_issue_due_date_changed_action(author:, project:) + track_snowplow_action(ISSUE_DUE_DATE_CHANGED, author, project) track_unique_action(ISSUE_DUE_DATE_CHANGED, author) end - def track_issue_time_estimate_changed_action(author:) + def track_issue_time_estimate_changed_action(author:, project:) + track_snowplow_action(ISSUE_TIME_ESTIMATE_CHANGED, author, project) track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author) end - def track_issue_time_spent_changed_action(author:) + def track_issue_time_spent_changed_action(author:, project:) + track_snowplow_action(ISSUE_TIME_SPENT_CHANGED, author, project) track_unique_action(ISSUE_TIME_SPENT_CHANGED, author) end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index a8f1bab1f20..10e36a75a3a 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -139,6 +139,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_container_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_api_fuzzing category: ci_templates redis_slot: ci_templates @@ -231,6 +235,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_katalon + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_mono category: ci_templates redis_slot: ci_templates @@ -319,6 +327,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_license_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_deploy category: ci_templates redis_slot: ci_templates @@ -331,6 +343,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_dependency_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_test category: ci_templates redis_slot: ci_templates @@ -523,6 +539,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_license_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy category: ci_templates redis_slot: ci_templates @@ -535,6 +555,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_dependency_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_test category: ci_templates redis_slot: ci_templates @@ -635,6 +659,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_container_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_api_fuzzing category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index c21b99ba834..0bd809f8aa5 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -1,9 +1,29 @@ --- -- name: i_code_review_mr_diffs +- name: i_code_review_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_commit + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_commit redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_mr_with_invalid_approvers +- name: i_code_review_mr_diffs redis_slot: code_review category: code_review aggregation: weekly @@ -135,12 +155,10 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_jetbrains_api_request - name: i_code_review_user_gitlab_cli_api_request redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_gitlab_cli_api_request - name: i_code_review_user_create_mr_from_issue redis_slot: code_review category: code_review @@ -177,30 +195,6 @@ redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review - aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting redis_slot: code_review @@ -400,53 +394,36 @@ redis_slot: code_review category: code_review aggregation: weekly -## Metrics -- name: i_code_review_merge_request_widget_metrics_view - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_full_report_clicked - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand_success - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand_warning +- name: i_code_review_submit_review_approve redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand_failed +- name: i_code_review_submit_review_comment redis_slot: code_review category: code_review aggregation: weekly -## Status Checks -- name: i_code_review_merge_request_widget_status_checks_view +## License Compliance +- name: i_code_review_merge_request_widget_license_compliance_view redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_full_report_clicked +- name: i_code_review_merge_request_widget_license_compliance_full_report_clicked redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand +- name: i_code_review_merge_request_widget_license_compliance_expand redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand_success +- name: i_code_review_merge_request_widget_license_compliance_expand_success redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand_warning +- name: i_code_review_merge_request_widget_license_compliance_expand_warning redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand_failed +- name: i_code_review_merge_request_widget_license_compliance_expand_failed redis_slot: code_review category: code_review aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 6c4754ae19f..29b231f88f8 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -146,6 +146,11 @@ category: testing redis_slot: testing aggregation: weekly +- name: i_testing_test_report_uploaded + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_ci_i_testing_test_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -332,11 +337,6 @@ redis_slot: testing category: testing aggregation: weekly -# Container Security - Network Policies -- name: clusters_using_network_policies_ui - redis_slot: network_policies - category: network_policies - aggregation: weekly # Geo group - name: g_geo_proxied_requests category: geo @@ -352,3 +352,8 @@ category: manage aggregation: weekly expiry: 42 +# Environments page +- name: users_visiting_environments_pages + category: environments + redis_slot: users + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index f594c6a1b7c..7f7c9166086 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -8,14 +8,6 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly -- name: i_ecosystem_jira_service_list_issues - category: ecosystem - redis_slot: ecosystem - aggregation: weekly -- name: i_ecosystem_jira_service_create_issue - category: ecosystem - redis_slot: ecosystem - aggregation: weekly - name: i_ecosystem_slack_service_issue_notification category: ecosystem redis_slot: ecosystem diff --git a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml deleted file mode 100644 index 3879c561cc4..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Epic board events -# -# We are using the same slot of issue events 'project_management' for -# epic events to allow data aggregation. -# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 -- name: g_project_management_users_creating_epic_boards - category: epic_boards_usage - redis_slot: project_management - aggregation: daily - -- name: g_project_management_users_viewing_epic_boards - category: epic_boards_usage - redis_slot: project_management - aggregation: daily - -- name: g_project_management_users_updating_epic_board_names - category: epic_boards_usage - redis_slot: project_management - aggregation: daily diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index e1de74a3d07..966e6c584c7 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -2,4 +2,3 @@ category: kubernetes_agent redis_slot: agent aggregation: weekly - feature_flag: track_agent_users_using_ci_tunnel diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index f980503b4bf..58a0c0695af 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -127,6 +127,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_timeline + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_page category: quickactions redis_slot: quickactions @@ -303,11 +307,3 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_attention - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_remove_attention - category: quickactions - redis_slot: quickactions - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index fbb03a31a6f..93137b762ec 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -49,6 +49,8 @@ module Gitlab MR_LOAD_CONFLICT_UI_ACTION = 'i_code_review_user_load_conflict_ui' MR_RESOLVE_CONFLICT_ACTION = 'i_code_review_user_resolve_conflict' MR_RESOLVE_THREAD_IN_ISSUE_ACTION = 'i_code_review_user_resolve_thread_in_issue' + MR_SUBMIT_REVIEW_APPROVE = 'i_code_review_submit_review_approve' + MR_SUBMIT_REVIEW_COMMENT = 'i_code_review_submit_review_comment' class << self def track_mr_diffs_action(merge_request:) @@ -230,6 +232,14 @@ module Gitlab track_unique_action_by_user(MR_RESOLVE_THREAD_IN_ISSUE_ACTION, user) end + def track_submit_review_approve(user:) + track_unique_action_by_user(MR_SUBMIT_REVIEW_APPROVE, user) + end + + def track_submit_review_comment(user:) + track_unique_action_by_user(MR_SUBMIT_REVIEW_COMMENT, user) + end + private def track_unique_action_by_merge_request(action, merge_request) diff --git a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb index dafc36ab7ce..f88bbc41c70 100644 --- a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb @@ -5,7 +5,7 @@ module Gitlab class MergeRequestWidgetExtensionCounter < BaseCounter KNOWN_EVENTS = %w[view full_report_clicked expand expand_success expand_warning expand_failed].freeze PREFIX = 'i_code_review_merge_request_widget' - WIDGETS = %w[accessibility code_quality status_checks terraform test_summary metrics].freeze + WIDGETS = %w[accessibility code_quality license_compliance status_checks terraform test_summary metrics].freeze class << self private diff --git a/lib/gitlab/utils/deep_size.rb b/lib/gitlab/utils/deep_size.rb index e185786e638..20f2d699e2b 100644 --- a/lib/gitlab/utils/deep_size.rb +++ b/lib/gitlab/utils/deep_size.rb @@ -25,10 +25,6 @@ module Gitlab !too_big? && !too_deep? end - def self.human_default_max_size - ActiveSupport::NumberHelper.number_to_human_size(DEFAULT_MAX_SIZE) - end - private def evaluate diff --git a/lib/gitlab/utils/execution_tracker.rb b/lib/gitlab/utils/execution_tracker.rb new file mode 100644 index 00000000000..6d48658853c --- /dev/null +++ b/lib/gitlab/utils/execution_tracker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + class ExecutionTracker + MAX_RUNTIME = 30.seconds + + ExecutionTimeOutError = Class.new(StandardError) + + delegate :monotonic_time, to: :'Gitlab::Metrics::System' + + def initialize + @start_time = monotonic_time + end + + def over_limit? + monotonic_time - start_time >= MAX_RUNTIME + end + + private + + attr_reader :start_time + end + end +end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index a2d217fb42f..2a57ca9ae02 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -46,6 +46,13 @@ module Gitlab url_builder.build(__subject__, only_path: true) end + def path_with_line_numbers(path, start_line, end_line) + path.tap do |complete_path| + complete_path << "#L#{start_line}" + complete_path << "-#{end_line}" if end_line && end_line != start_line + end + end + class_methods do def presenter? true diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 049e3befe64..7360585df43 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -47,17 +47,17 @@ module Gitlab def options { - s_('VisibilityLevel|Private') => PRIVATE, + s_('VisibilityLevel|Private') => PRIVATE, s_('VisibilityLevel|Internal') => INTERNAL, - s_('VisibilityLevel|Public') => PUBLIC + s_('VisibilityLevel|Public') => PUBLIC } end def string_options { - 'private' => PRIVATE, + 'private' => PRIVATE, 'internal' => INTERNAL, - 'public' => PUBLIC + 'public' => PUBLIC } end diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb index 1b5350d4a4e..031d9ec6ec4 100644 --- a/lib/gitlab/web_hooks/recursion_detection.rb +++ b/lib/gitlab/web_hooks/recursion_detection.rb @@ -40,9 +40,9 @@ module Gitlab cache_key = cache_key_for_hook(hook) ::Gitlab::Redis::SharedState.with do |redis| - redis.multi do - redis.sadd(cache_key, hook.id) - redis.expire(cache_key, TOUCH_CACHE_TTL) + redis.multi do |multi| + multi.sadd(cache_key, hook.id) + multi.expire(cache_key, TOUCH_CACHE_TTL) end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e81670ce89a..906439d5e71 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -12,7 +12,7 @@ module Gitlab VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' - NOTIFICATION_CHANNEL = 'workhorse:notifications' + NOTIFICATION_PREFIX = 'workhorse:notifications:' ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type' ARCHIVE_FORMATS = %w(zip tar.gz tar.bz2 tar).freeze @@ -217,7 +217,8 @@ module Gitlab Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result - redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + redis.publish(NOTIFICATION_PREFIX + key, value) + value else redis.get(key) diff --git a/lib/gitlab_edition.rb b/lib/gitlab_edition.rb index 02006148a34..5e3ed35ace4 100644 --- a/lib/gitlab_edition.rb +++ b/lib/gitlab_edition.rb @@ -7,6 +7,21 @@ module GitlabEdition Pathname.new(File.expand_path('..', __dir__)) end + def self.path_glob(path) + "#{root}/#{extension_path_prefixes}#{path}" + end + + def self.extension_path_prefixes + path_prefixes = extensions + return '' if path_prefixes.empty? + + path_prefixes.map! { "#{_1}/" } + path_prefixes.unshift '' + + # For example `{,ee/,jh/}` + "{#{path_prefixes.join(',')}}" + end + def self.extensions if jh? %w[ee jh] diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 39cf994ca3f..38a1a968aec 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -22,7 +22,7 @@ module GoogleApi "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/monitoring" ].freeze - ROLES_LIST = %w[roles/iam.serviceAccountUser roles/artifactregistry.admin roles/cloudbuild.builds.builder roles/run.admin roles/storage.admin roles/cloudsql.admin roles/browser].freeze + ROLES_LIST = %w[roles/iam.serviceAccountUser roles/artifactregistry.admin roles/cloudbuild.builds.builder roles/run.admin roles/storage.admin roles/cloudsql.client roles/browser].freeze REVOKE_URL = 'https://oauth2.googleapis.com/revoke' class << self diff --git a/lib/learn_gitlab/onboarding.rb b/lib/learn_gitlab/onboarding.rb deleted file mode 100644 index 54af01a21fe..00000000000 --- a/lib/learn_gitlab/onboarding.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -module LearnGitlab - class Onboarding - include Gitlab::Utils::StrongMemoize - include Gitlab::Experiment::Dsl - - ACTION_ISSUE_IDS = { - pipeline_created: 7, - trial_started: 2, - required_mr_approvals_enabled: 11, - code_owners_enabled: 10 - }.freeze - - ACTION_PATHS = [ - :issue_created, - :git_write, - :merge_request_created, - :user_added - ].freeze - - def initialize(namespace, current_user = nil) - @namespace = namespace - @current_user = current_user - end - - def completed_percentage - return 0 unless onboarding_progress - - attributes = onboarding_progress.attributes.symbolize_keys - - total_actions = action_columns.count - completed_actions = action_columns.count { |column| attributes[column].present? } - - (completed_actions.to_f / total_actions.to_f * 100).round - end - - private - - def onboarding_progress - strong_memoize(:onboarding_progress) do - OnboardingProgress.find_by(namespace: namespace) # rubocop: disable CodeReuse/ActiveRecord - end - end - - def action_columns - strong_memoize(:action_columns) do - tracked_actions.map { |action_key| OnboardingProgress.column_name(action_key) } - end - end - - def tracked_actions - ACTION_ISSUE_IDS.keys + ACTION_PATHS + deploy_section_tracked_actions - end - - def deploy_section_tracked_actions - experiment(:security_actions_continuous_onboarding, - namespace: namespace, - user: current_user, - sticky_to: current_user - ) do |e| - e.control { [:security_scan_enabled] } - e.candidate { [:license_scanning_run, :secure_dependency_scanning_run, :secure_dast_run] } - end.run - end - - attr_reader :namespace, :current_user - end -end diff --git a/lib/learn_gitlab/project.rb b/lib/learn_gitlab/project.rb deleted file mode 100644 index 64f91dcf1a8..00000000000 --- a/lib/learn_gitlab/project.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module LearnGitlab - class Project - PROJECT_NAME = 'Learn GitLab' - PROJECT_NAME_ULTIMATE_TRIAL = 'Learn GitLab - Ultimate trial' - BOARD_NAME = 'GitLab onboarding' - LABEL_NAME = 'Novice' - - def initialize(current_user) - @current_user = current_user - end - - def available? - project && board && label - end - - def project - @project ||= current_user.projects.find_by_name([PROJECT_NAME, PROJECT_NAME_ULTIMATE_TRIAL]) - end - - def board - return unless project - - @board ||= project.boards.find_by_name(BOARD_NAME) - end - - def label - return unless project - - @label ||= project.labels.find_by_name(LABEL_NAME) - end - - private - - attr_reader :current_user - end -end diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index a8b51a95e59..d092cd56e46 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -206,7 +206,7 @@ module ObjectStorage def requires_multipart_upload? return false unless config.aws? - return false if use_workhorse_s3_client? && Feature.enabled?(:s3_omit_multipart_urls) + return false if use_workhorse_s3_client? !has_length end diff --git a/lib/omni_auth/strategies/bitbucket.rb b/lib/omni_auth/strategies/bitbucket.rb index 6c914b4222a..d64f3dd987d 100644 --- a/lib/omni_auth/strategies/bitbucket.rb +++ b/lib/omni_auth/strategies/bitbucket.rb @@ -40,7 +40,7 @@ module OmniAuth end def callback_url - options[:redirect_uri] || (full_host + script_name + callback_path) + options[:redirect_uri] || (full_host + callback_path) end end end diff --git a/lib/peek/views/redis_detailed.rb b/lib/peek/views/redis_detailed.rb index 44ec0ec0f68..76c283bf802 100644 --- a/lib/peek/views/redis_detailed.rb +++ b/lib/peek/views/redis_detailed.rb @@ -16,7 +16,11 @@ module Peek private def format_call_details(call) - super.merge(cmd: format_command(call[:cmd]), + cmd = call[:commands].map do |command| + format_command(command) + end.join(', ') + + super.merge(cmd: cmd, instance: call[:storage]) end diff --git a/lib/product_analytics/event_params.rb b/lib/product_analytics/event_params.rb index 07e0bc8b43a..6cb3d462384 100644 --- a/lib/product_analytics/event_params.rb +++ b/lib/product_analytics/event_params.rb @@ -11,41 +11,41 @@ module ProductAnalytics class EventParams def self.parse_event_params(params) { - project_id: params['aid'], - platform: params['p'], - collector_tstamp: Time.zone.now, - event_id: params['eid'], - v_tracker: params['tv'], - v_collector: Gitlab::VERSION, - v_etl: Gitlab::VERSION, - os_timezone: params['tz'], - name_tracker: params['tna'], - br_lang: params['lang'], - doc_charset: params['cs'], - br_features_pdf: Gitlab::Utils.to_boolean(params['f_pdf']), - br_features_flash: Gitlab::Utils.to_boolean(params['f_fla']), - br_features_java: Gitlab::Utils.to_boolean(params['f_java']), - br_features_director: Gitlab::Utils.to_boolean(params['f_dir']), - br_features_quicktime: Gitlab::Utils.to_boolean(params['f_qt']), - br_features_realplayer: Gitlab::Utils.to_boolean(params['f_realp']), + project_id: params['aid'], + platform: params['p'], + collector_tstamp: Time.zone.now, + event_id: params['eid'], + v_tracker: params['tv'], + v_collector: Gitlab::VERSION, + v_etl: Gitlab::VERSION, + os_timezone: params['tz'], + name_tracker: params['tna'], + br_lang: params['lang'], + doc_charset: params['cs'], + br_features_pdf: Gitlab::Utils.to_boolean(params['f_pdf']), + br_features_flash: Gitlab::Utils.to_boolean(params['f_fla']), + br_features_java: Gitlab::Utils.to_boolean(params['f_java']), + br_features_director: Gitlab::Utils.to_boolean(params['f_dir']), + br_features_quicktime: Gitlab::Utils.to_boolean(params['f_qt']), + br_features_realplayer: Gitlab::Utils.to_boolean(params['f_realp']), br_features_windowsmedia: Gitlab::Utils.to_boolean(params['f_wma']), - br_features_gears: Gitlab::Utils.to_boolean(params['f_gears']), - br_features_silverlight: Gitlab::Utils.to_boolean(params['f_ag']), - br_colordepth: params['cd'], - br_cookies: Gitlab::Utils.to_boolean(params['cookie']), - dvce_created_tstamp: params['dtm'], - br_viewheight: params['vp'], - domain_sessionidx: params['vid'], - domain_sessionid: params['sid'], - domain_userid: params['duid'], - user_fingerprint: params['fp'], - page_referrer: params['refr'], - page_url: params['url'], - se_category: params['se_ca'], - se_action: params['se_ac'], - se_label: params['se_la'], - se_property: params['se_pr'], - se_value: params['se_va'] + br_features_gears: Gitlab::Utils.to_boolean(params['f_gears']), + br_features_silverlight: Gitlab::Utils.to_boolean(params['f_ag']), + br_colordepth: params['cd'], + br_cookies: Gitlab::Utils.to_boolean(params['cookie']), + dvce_created_tstamp: params['dtm'], + br_viewheight: params['vp'], + domain_sessionidx: params['vid'], + domain_sessionid: params['sid'], + domain_userid: params['duid'], + user_fingerprint: params['fp'], + page_referrer: params['refr'], + page_url: params['url'], + se_category: params['se_ca'], + se_action: params['se_ac'], + se_label: params['se_la'], + se_property: params['se_pr'], + se_value: params['se_va'] } end diff --git a/lib/security/weak_passwords.rb b/lib/security/weak_passwords.rb new file mode 100644 index 00000000000..42b02132933 --- /dev/null +++ b/lib/security/weak_passwords.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true +module Security + module WeakPasswords + # These words are predictable in GitLab's specific context, and + # therefore cannot occur anywhere within a password. + FORBIDDEN_WORDS = Set['gitlab', 'devops'].freeze + + # Substrings shorter than this may appear legitimately in a truly + # random password. + MINIMUM_SUBSTRING_SIZE = 4 + + class << self + # Returns true when the password is on a list of weak passwords, + # or contains predictable substrings derived from user attributes. + # Case insensitive. + def weak_for_user?(password, user) + forbidden_word_appears_in_password?(password) || + name_appears_in_password?(password, user) || + username_appears_in_password?(password, user) || + email_appears_in_password?(password, user) || + password_on_weak_list?(password) + end + + private + + def forbidden_word_appears_in_password?(password) + contains_predicatable_substring?(password, FORBIDDEN_WORDS) + end + + def name_appears_in_password?(password, user) + return false if user.name.blank? + + # Check for the full name + substrings = [user.name] + # Also check parts of their name + substrings += user.name.split(/[^\p{Alnum}]/) + + contains_predicatable_substring?(password, substrings) + end + + def username_appears_in_password?(password, user) + return false if user.username.blank? + + # Check for the full username + substrings = [user.username] + # Also check sub-strings in the username + substrings += user.username.split(/[^\p{Alnum}]/) + + contains_predicatable_substring?(password, substrings) + end + + def email_appears_in_password?(password, user) + return false if user.email.blank? + + # Check for the full email + substrings = [user.email] + # Also check full first part and full domain name + substrings += user.email.split("@") + # And any parts of non-word characters (e.g. firstname.lastname+tag@...) + substrings += user.email.split(/[^\p{Alnum}]/) + + contains_predicatable_substring?(password, substrings) + end + + def password_on_weak_list?(password) + # Our weak list stores SHA2 hashes of passwords, not the weak + # passwords themselves. + digest = Digest::SHA256.base64digest(password.downcase) + Settings.gitlab.weak_passwords_digest_set.include?(digest) + end + + # Case-insensitively checks whether a password includes a dynamic + # list of substrings. Substrings which are too short are not + # predictable and may occur randomly, and therefore not checked. + def contains_predicatable_substring?(password, substrings) + substrings = substrings.filter_map do |substring| + substring.downcase if substring.length >= MINIMUM_SUBSTRING_SIZE + end + + password = password.downcase + + # Returns true when a predictable substring occurs anywhere + # in the password. + substrings.any? { |word| password.include?(word) } + end + end + end +end diff --git a/lib/sidebars/groups/menus/observability_menu.rb b/lib/sidebars/groups/menus/observability_menu.rb new file mode 100644 index 00000000000..b479ff3c492 --- /dev/null +++ b/lib/sidebars/groups/menus/observability_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class ObservabilityMenu < ::Sidebars::Menu + override :link + def link + group_observability_index_path(context.group) + end + + override :title + def title + _('Observability') + end + + override :sprite_icon + def sprite_icon + 'monitor' + end + + override :render? + def render? + can?(context.current_user, :read_observability, context.group) + end + + override :active_routes + def active_routes + { controller: :observability, path: 'groups#observability' } + end + end + end + end +end diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb index fda90406e0a..61cd81711f8 100644 --- a/lib/sidebars/groups/menus/packages_registries_menu.rb +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -15,7 +15,7 @@ module Sidebars override :title def title - _('Packages & Registries') + _('Packages and registries') end override :sprite_icon @@ -50,7 +50,9 @@ module Sidebars end def harbor_registry__menu_item - return nil_menu_item(:harbor_registry) if Feature.disabled?(:harbor_registry_integration) + if Feature.disabled?(:harbor_registry_integration) || context.group.harbor_integration.nil? + return nil_menu_item(:harbor_registry) + end ::Sidebars::MenuItem.new( title: _('Harbor Registry'), diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index 18ff3ebc714..df170670aab 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -6,18 +6,23 @@ module Sidebars class SettingsMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false unless can?(context.current_user, :admin_group, context.group) - - add_item(general_menu_item) - add_item(integrations_menu_item) - add_item(access_tokens_menu_item) - add_item(group_projects_menu_item) - add_item(repository_menu_item) - add_item(ci_cd_menu_item) - add_item(applications_menu_item) - add_item(packages_and_registries_menu_item) - - true + if can?(context.current_user, :admin_group, context.group) + add_item(general_menu_item) + add_item(integrations_menu_item) + add_item(access_tokens_menu_item) + add_item(group_projects_menu_item) + add_item(repository_menu_item) + add_item(ci_cd_menu_item) + add_item(applications_menu_item) + add_item(packages_and_registries_menu_item) + return true + elsif Gitlab.ee? && can?(context.current_user, :change_push_rules, context.group) + # Push Rules are the only group setting that can also be edited by maintainers. + # Create an empty sub-menu here and EE adds Repository menu item (with only Push Rules). + return true + end + + false end override :title @@ -112,7 +117,7 @@ module Sidebars end ::Sidebars::MenuItem.new( - title: _('Packages & Registries'), + title: _('Packages and registries'), link: group_settings_packages_and_registries_path(context.group), active_routes: { controller: :packages_and_registries }, item_id: :packages_and_registries diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb index 463c2571b14..e8b815bdce7 100644 --- a/lib/sidebars/groups/panel.rb +++ b/lib/sidebars/groups/panel.rb @@ -12,6 +12,7 @@ module Sidebars add_menu(Sidebars::Groups::Menus::MergeRequestsMenu.new(context)) add_menu(Sidebars::Groups::Menus::CiCdMenu.new(context)) add_menu(Sidebars::Groups::Menus::KubernetesMenu.new(context)) + add_menu(Sidebars::Groups::Menus::ObservabilityMenu.new(context)) add_menu(Sidebars::Groups::Menus::PackagesRegistriesMenu.new(context)) add_menu(Sidebars::Groups::Menus::CustomerRelationsMenu.new(context)) add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context)) diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 1c04a7b117d..63eea0ea500 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -54,12 +54,12 @@ module Sidebars { disabled: true, data: { trigger: 'manual', - container: 'body', - placement: 'right', - highlight: Users::CalloutsHelper::GKE_CLUSTER_INTEGRATION, - highlight_priority: Users::Callout.feature_names[:GKE_CLUSTER_INTEGRATION], - dismiss_endpoint: callouts_path, - auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } + container: 'body', + placement: 'right', + highlight: Users::CalloutsHelper::GKE_CLUSTER_INTEGRATION, + highlight_priority: Users::Callout.feature_names[:GKE_CLUSTER_INTEGRATION], + dismiss_endpoint: callouts_path, + auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } end def terraform_menu_item diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb index d2bc2fa0681..b6fae2af93d 100644 --- a/lib/sidebars/projects/menus/learn_gitlab_menu.rb +++ b/lib/sidebars/projects/menus/learn_gitlab_menu.rb @@ -29,10 +29,10 @@ module Sidebars override :pill_count def pill_count strong_memoize(:pill_count) do - percentage = LearnGitlab::Onboarding.new( + percentage = Onboarding::Completion.new( context.project.namespace, context.current_user - ).completed_percentage + ).percentage "#{percentage}%" end diff --git a/lib/sidebars/projects/menus/merge_requests_menu.rb b/lib/sidebars/projects/menus/merge_requests_menu.rb index fe501667d37..3e543872d36 100644 --- a/lib/sidebars/projects/menus/merge_requests_menu.rb +++ b/lib/sidebars/projects/menus/merge_requests_menu.rb @@ -59,9 +59,9 @@ module Sidebars override :active_routes def active_routes if context.project.issues_enabled? - { controller: :merge_requests } + { controller: 'projects/merge_requests' } else - { controller: [:merge_requests, :milestones] } + { controller: ['projects/merge_requests', :milestones] } end end end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index 23e1a95c401..ecd062f333e 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -6,7 +6,7 @@ module Sidebars class MonitorMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false unless context.project.feature_available?(:operations, context.current_user) + return false unless feature_enabled? add_item(metrics_dashboard_menu_item) add_item(error_tracking_menu_item) @@ -41,6 +41,14 @@ module Sidebars private + def feature_enabled? + if ::Feature.enabled?(:split_operations_visibility_permissions, context.project) + context.project.feature_available?(:monitor, context.current_user) + else + context.project.feature_available?(:operations, context.current_user) + end + end + def metrics_dashboard_menu_item unless can?(context.current_user, :metrics_dashboard, context.project) return ::Sidebars::NilMenuItem.new(item_id: :metrics) diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 914368e6fec..2ddffe42899 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -15,7 +15,7 @@ module Sidebars override :title def title - _('Packages & Registries') + _('Packages and registries') end override :sprite_icon @@ -66,7 +66,9 @@ module Sidebars end def harbor_registry__menu_item - return ::Sidebars::NilMenuItem.new(item_id: :harbor_registry) if Feature.disabled?(:harbor_registry_integration) + if Feature.disabled?(:harbor_registry_integration, context.project) || context.project.harbor_integration.nil? + return ::Sidebars::NilMenuItem.new(item_id: :harbor_registry) + end ::Sidebars::MenuItem.new( title: _('Harbor Registry'), @@ -77,7 +79,8 @@ module Sidebars end def packages_registry_disabled? - !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project) + !::Gitlab.config.packages.enabled || + !can?(context.current_user, :read_package, context.project&.packages_policy_subject) end end end diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 85931e63ebc..11d5f4d59c7 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -13,6 +13,7 @@ module Sidebars add_item(webhooks_menu_item) add_item(access_tokens_menu_item) add_item(repository_menu_item) + add_item(merge_requests_menu_item) add_item(ci_cd_menu_item) add_item(packages_and_registries_menu_item) add_item(pages_menu_item) @@ -109,9 +110,9 @@ module Sidebars end ::Sidebars::MenuItem.new( - title: _('Packages & Registries'), + title: _('Packages and registries'), link: project_settings_packages_and_registries_path(context.project), - active_routes: { path: 'packages_and_registries#show' }, + active_routes: { controller: :packages_and_registries }, item_id: :packages_and_registries ) end @@ -150,6 +151,17 @@ module Sidebars item_id: :usage_quotas ) end + + def merge_requests_menu_item + return unless context.project.merge_requests_enabled? + + ::Sidebars::MenuItem.new( + title: _('Merge requests'), + link: project_settings_merge_requests_path(context.project), + active_routes: { path: 'projects/settings/merge_requests#show' }, + item_id: :merge_requests + ) + end end end end diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb index 1af8e14f034..8ae8f931aab 100644 --- a/lib/sidebars/projects/panel.rb +++ b/lib/sidebars/projects/panel.rb @@ -51,8 +51,7 @@ module Sidebars end def third_party_wiki_menu - wiki_menu_list = [::Sidebars::Projects::Menus::ConfluenceMenu] - wiki_menu_list << ::Sidebars::Projects::Menus::ShimoMenu if Feature.enabled?(:shimo_integration, context.project) + wiki_menu_list = [::Sidebars::Projects::Menus::ConfluenceMenu, ::Sidebars::Projects::Menus::ShimoMenu] wiki_menu_list.find { |wiki_menu| wiki_menu.new(context).render? } end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 0b70dba5c05..76ee5379213 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -40,6 +40,7 @@ module Tasks asset_files end + private_class_method :assets_impacting_webpack_compilation end end @@ -84,9 +85,17 @@ namespace :gitlab do if head_assets_sha256 != master_assets_sha256 || !public_assets_webpack_dir_exists FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists - unless system('yarn webpack') + log_path = ENV['WEBPACK_COMPILE_LOG_PATH'] + + cmd = 'yarn webpack' + cmd += " > #{log_path}" if log_path + + unless system(cmd) abort 'Error: Unable to compile webpack production bundle.'.color(:red) end + + puts "Written webpack stdout log to #{log_path}" if log_path + puts "You can inspect the webpack log here: #{ENV['CI_JOB_URL']}/artifacts/file/#{log_path}" if log_path && ENV['CI_JOB_URL'] end end diff --git a/lib/tasks/gitlab/db/truncate_legacy_tables.rake b/lib/tasks/gitlab/db/truncate_legacy_tables.rake new file mode 100644 index 00000000000..9c3d7c3876d --- /dev/null +++ b/lib/tasks/gitlab/db/truncate_legacy_tables.rake @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + namespace :truncate_legacy_tables do + desc "GitLab | DB | Truncate CI Tables on Main" + task :main, [:min_batch_size] => [:environment, 'gitlab:db:validate_config'] do |_t, args| + args.with_defaults(min_batch_size: 5) + Gitlab::Database::TablesTruncate.new( + database_name: 'main', + min_batch_size: args.min_batch_size.to_i, + logger: Logger.new($stdout), + dry_run: ENV['DRY_RUN'] == 'true', + until_table: ENV['UNTIL_TABLE'] + ).execute + end + + desc "GitLab | DB | Truncate Main Tables on CI" + task :ci, [:min_batch_size] => [:environment, 'gitlab:db:validate_config'] do |_t, args| + args.with_defaults(min_batch_size: 5) + Gitlab::Database::TablesTruncate.new( + database_name: 'ci', + min_batch_size: args.min_batch_size.to_i, + logger: Logger.new($stdout), + dry_run: ENV['DRY_RUN'] == 'true', + until_table: ENV['UNTIL_TABLE'] + ).execute + end + end + end +end diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index 2a3a54b5351..bf9ebc56486 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -144,7 +144,7 @@ namespace :gitlab do rescue ActiveRecord::StatementInvalid => err raise unless err.cause.is_a?(PG::ReadOnlySqlTransaction) - warn "WARNING: Could not write to the database #{db_config.name}: #{err.message}" + warn "WARNING: Could not write to the database #{db_config.name}: cannot execute UPSERT in a read-only transaction" end def get_db_identifier(db_config) diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake index 4bdc62c9319..3cefdcc1aaf 100644 --- a/lib/tasks/gitlab/import_export/export.rake +++ b/lib/tasks/gitlab/import_export/export.rake @@ -27,9 +27,9 @@ namespace :gitlab do task = Gitlab::ImportExport::Project::ExportTask.new( namespace_path: args.namespace_path, - project_path: args.project_path, - username: args.username, - file_path: args.archive_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, logger: logger ) diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index 2702b530334..fc727eda380 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -31,9 +31,9 @@ namespace :gitlab do task = Gitlab::ImportExport::Project::ImportTask.new( namespace_path: args.namespace_path, - project_path: args.project_path, - username: args.username, - file_path: args.archive_path, + project_path: args.project_path, + username: args.username, + file_path: args.archive_path, logger: logger ) diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index f6c518784a9..148801254bf 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -19,16 +19,15 @@ namespace :tw do end CODE_OWNER_RULES = [ - CodeOwnerRule.new('Activation', '@kpaizee'), - CodeOwnerRule.new("Adoption", '@kpaizee'), - CodeOwnerRule.new('Activation', '@kpaizee'), - CodeOwnerRule.new('Adoption', '@kpaizee'), + CodeOwnerRule.new('Activation', '@phillipwells'), + CodeOwnerRule.new('Acquisition', '@phillipwells'), + CodeOwnerRule.new('Anti-Abuse', '@phillipwells'), CodeOwnerRule.new('Authentication and Authorization', '@eread'), CodeOwnerRule.new('Certify', '@msedlakjakubowski'), CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), - CodeOwnerRule.new('Configure', '@sselhorn'), + CodeOwnerRule.new('Configure', '@phillipwells'), CodeOwnerRule.new('Container Security', '@claytoncornell'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Conversion', '@kpaizee'), @@ -41,7 +40,6 @@ namespace :tw do CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), CodeOwnerRule.new('Ecosystem', '@kpaizee'), CodeOwnerRule.new('Editor', '@aqualls'), - CodeOwnerRule.new('Expansion', '@kpaizee'), CodeOwnerRule.new('Foundations', '@rdickenson'), CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), CodeOwnerRule.new('Geo', '@axil'), diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 80290f95e8e..2a91fd1646c 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -2,15 +2,8 @@ namespace :gitlab do namespace :uploads do - namespace :migrate do - desc "GitLab | Uploads | Migrate all uploaded files to object storage" - task all: :environment do - Gitlab::Uploads::MigrationHelper.categories.each do |args| - Rake::Task["gitlab:uploads:migrate"].invoke(*args) - Rake::Task["gitlab:uploads:migrate"].reenable - end - end - end + desc "GitLab | Uploads | Migrate all uploaded files to object storage" + task 'migrate:all' => :migrate # The following is the actual rake task that migrates uploads of specified # category to object storage @@ -19,15 +12,8 @@ namespace :gitlab do Gitlab::Uploads::MigrationHelper.new(args, Logger.new($stdout)).migrate_to_remote_storage end - namespace :migrate_to_local do - desc "GitLab | Uploads | Migrate all uploaded files to local storage" - task all: :environment do - Gitlab::Uploads::MigrationHelper.categories.each do |args| - Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) - Rake::Task["gitlab:uploads:migrate_to_local"].reenable - end - end - end + desc "GitLab | Uploads | Migrate all uploaded files to local storage" + task 'migrate_to_local:all' => :migrate_to_local desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage' task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 9f064ef4c0c..73a79427da3 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -17,11 +17,16 @@ namespace :gitlab do puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :all_metrics_values)) end + desc 'GitLab | UsageData | Generate non SQL data for usage ping in JSON' + task dump_non_sql_in_json: :environment do + puts Gitlab::Json.pretty_generate(Gitlab::Usage::ServicePingReport.for(output: :non_sql_metrics_values)) + end + desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application' task generate_and_send: :environment do - result = ServicePing::SubmitService.new.execute + response = GitlabServicePingWorker.new.perform('triggered_from_cron' => false) - puts Gitlab::Json.pretty_generate(result.attributes) + puts response.body, response.code, response.message, response.headers.inspect end desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON' @@ -51,6 +56,19 @@ namespace :gitlab do File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, banner + YAML.dump(all_includes).gsub(/ *$/m, '')) end + desc 'GitLab | UsageDataMetrics | Generate raw SQL metrics queries for RSpec' + task generate_sql_metrics_queries: :environment do + path = Rails.root.join('tmp', 'test') + + queries = Timecop.freeze(2021, 1, 1) do + Gitlab::Usage::ServicePingReport.for(output: :metrics_queries) + end + + FileUtils.mkdir_p(path) + FileUtils.chdir(path) + File.write('sql_metrics_queries.json', Gitlab::Json.pretty_generate(queries)) + end + def ci_template_includes_hash(source, template_directory = nil) Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/#{template_directory}").map do |template| expanded_template_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.expand_template_name("#{template_directory}/#{template}") diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake deleted file mode 100644 index 29589571344..00000000000 --- a/lib/tasks/haml-lint.rake +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -unless Rails.env.production? - require 'haml_lint/rake_task' - - HamlLint::RakeTask.new -end diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index e993035aa65..0c257585bd0 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -6,6 +6,19 @@ unless Rails.env.production? RuboCop::RakeTask.new namespace :rubocop do + namespace :check do + desc 'Run RuboCop check gracefully' + task :graceful do |_task, args| + require_relative '../../rubocop/check_graceful_task' + + # Don't reveal TODOs in this run. + ENV.delete('REVEAL_RUBOCOP_TODO') + + result = RuboCop::CheckGracefulTask.new($stdout).run(args.extras) + exit result if result.nonzero? + end + end + namespace :todo do desc 'Generate RuboCop todos' task :generate do |_task, args| diff --git a/lib/tasks/tanuki_emoji.rake b/lib/tasks/tanuki_emoji.rake index 0dc7dd4e701..b02d7a532c4 100644 --- a/lib/tasks/tanuki_emoji.rake +++ b/lib/tasks/tanuki_emoji.rake @@ -148,11 +148,11 @@ namespace :tanuki_emoji do SpriteFactory.run!(tmpdir, { output_style: style_path, output_image: "app/assets/images/emoji.png", - selector: '.emoji-', - style: :scss, - nocomments: true, - pngcrush: true, - layout: :packed + selector: '.emoji-', + style: :scss, + nocomments: true, + pngcrush: true, + layout: :packed }) # SpriteFactory's SCSS is a bit too verbose for our purposes here, so @@ -215,10 +215,10 @@ namespace :tanuki_emoji do # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.run!(tmpdir, { output_image: "app/assets/images/emoji@2x.png", - style: false, - nocomments: true, - pngcrush: true, - layout: :packed + style: false, + nocomments: true, + pngcrush: true, + layout: :packed }) end |