diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 09:55:51 +0000 |
| commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
| tree | c42be41678c2586d49a75cabce89322082698334 /lib | |
| parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) | |
| download | gitlab-ce-e8d2c2579383897a1dd7f9debd359abe8ae8373d.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to 'lib')
440 files changed, 4886 insertions, 2400 deletions
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb index 654d3a48162..0462878c90c 100644 --- a/lib/api/admin/ci/variables.rb +++ b/lib/api/admin/ci/variables.rb @@ -8,7 +8,7 @@ module API before { authenticated_as_admin! } - feature_category :continuous_integration + feature_category :pipeline_authoring namespace 'admin' do namespace 'ci' do diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb index 92f7d3dce0d..ab6a4e4a04a 100644 --- a/lib/api/admin/plan_limits.rb +++ b/lib/api/admin/plan_limits.rb @@ -41,6 +41,7 @@ module API optional :npm_max_file_size, type: Integer, desc: 'Maximum NPM package file size in bytes' optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes' optional :pypi_max_file_size, type: Integer, desc: 'Maximum PyPI package file size in bytes' + optional :terraform_module_max_file_size, type: Integer, desc: 'Maximum Terraform Module package file size in bytes' end put "application/plan_limits" do params = declared_params(include_missing: false) diff --git a/lib/api/api.rb b/lib/api/api.rb index 2a3033753f7..f9e89191a36 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -152,6 +152,7 @@ module API mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages + mount ::API::BulkImports mount ::API::Ci::Pipelines mount ::API::Ci::PipelineSchedules mount ::API::Ci::Runner @@ -166,12 +167,14 @@ module API mount ::API::Deployments mount ::API::Environments mount ::API::ErrorTracking + mount ::API::ErrorTrackingCollector mount ::API::Events mount ::API::FeatureFlags mount ::API::FeatureFlagsUserLists mount ::API::Features mount ::API::Files mount ::API::FreezePeriods + mount ::API::Geo mount ::API::GroupAvatar mount ::API::GroupBoards mount ::API::GroupClusters diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb new file mode 100644 index 00000000000..189851cee65 --- /dev/null +++ b/lib/api/bulk_imports.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module API + class BulkImports < ::API::Base + include PaginationParams + + feature_category :importers + + helpers do + def bulk_imports + @bulk_imports ||= ::BulkImports::ImportsFinder.new(user: current_user, status: params[:status]).execute + end + + def bulk_import + @bulk_import ||= bulk_imports.find(params[:import_id]) + end + + def bulk_import_entities + @bulk_import_entities ||= ::BulkImports::EntitiesFinder.new(user: current_user, bulk_import: bulk_import, status: params[:status]).execute + end + + def bulk_import_entity + @bulk_import_entity ||= bulk_import_entities.find(params[:entity_id]) + end + end + + before { authenticate! } + + resource :bulk_imports do + desc 'List all GitLab Migrations' do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + use :pagination + optional :status, type: String, values: BulkImport.all_human_statuses, + desc: 'Return GitLab Migrations with specified status' + end + get do + present paginate(bulk_imports), with: Entities::BulkImport + end + + desc "List all GitLab Migrations' entities" do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + use :pagination + optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses, + desc: "Return all GitLab Migrations' entities with specified status" + end + get :entities do + entities = ::BulkImports::EntitiesFinder.new(user: current_user, status: params[:status]).execute + + present paginate(entities), with: Entities::BulkImports::Entity + end + + desc 'Get GitLab Migration details' do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" + end + get ':import_id' do + present bulk_import, with: Entities::BulkImport + end + + desc "List GitLab Migration entities" do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" + optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses, + desc: 'Return import entities with specified status' + use :pagination + end + get ':import_id/entities' do + present paginate(bulk_import_entities), with: Entities::BulkImports::Entity + end + + desc 'Get GitLab Migration entity details' do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" + requires :entity_id, type: Integer, desc: "The ID of GitLab Migration entity" + end + get ':import_id/entities/:entity_id' do + present bulk_import_entity, with: Entities::BulkImports::Entity + end + end + end +end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index c4e0b699524..0bac6fe2054 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -7,11 +7,11 @@ module API content_type :txt, 'text/plain' - feature_category :continuous_integration + feature_category :runner resource :runners do desc 'Registers a new Runner' do - success Entities::RunnerRegistrationDetails + success Entities::Ci::RunnerRegistrationDetails http_codes [[201, 'Runner was created'], [403, 'Forbidden']] end params do @@ -34,10 +34,10 @@ module API if runner_registration_token_valid? # Create shared runner. Requires admin access attributes.merge(runner_type: :instance_type) - elsif @project = Project.find_by_runners_token(params[:token]) + elsif runner_registrar_valid?('project') && @project = Project.find_by_runners_token(params[:token]) # Create a specific runner for the project attributes.merge(runner_type: :project_type, projects: [@project]) - elsif @group = Group.find_by_runners_token(params[:token]) + elsif runner_registrar_valid?('group') && @group = Group.find_by_runners_token(params[:token]) # Create a specific runner for the group attributes.merge(runner_type: :group_type, groups: [@group]) else @@ -47,7 +47,7 @@ module API @runner = ::Ci::Runner.create(attributes) if @runner.persisted? - present @runner, with: Entities::RunnerRegistrationDetails + present @runner, with: Entities::Ci::RunnerRegistrationDetails else render_validation_error!(@runner) end @@ -82,7 +82,7 @@ module API before { set_application_context } desc 'Request a job' do - success Entities::JobRequest::Response + success Entities::Ci::JobRequest::Response http_codes [[201, 'Job was scheduled'], [204, 'No job for Runner'], [403, 'Forbidden']] @@ -214,6 +214,10 @@ module API .new(job, content_range: content_range) .execute(request.body.read) + if result.status == 403 + break error!('403 Forbidden', 403) + end + if result.status == 416 break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{result.stream_size}" }) end @@ -263,7 +267,7 @@ module API end desc 'Upload artifacts for job' do - success Entities::JobRequest::Response + success Entities::Ci::JobRequest::Response http_codes [[201, 'Artifact uploaded'], [400, 'Bad request'], [403, 'Forbidden'], diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 44ffc941cfa..7f755b1a4d4 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -7,11 +7,11 @@ module API before { authenticate! } - feature_category :continuous_integration + feature_category :runner resource :runners do desc 'Get runners available for user' do - success Entities::Runner + success Entities::Ci::Runner end params do optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_STATUSES, @@ -30,11 +30,11 @@ module API runners = filter_runners(runners, params[:status], allowed_scopes: ::Ci::Runner::AVAILABLE_STATUSES) runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] - present paginate(runners), with: Entities::Runner + present paginate(runners), with: Entities::Ci::Runner end desc 'Get all runners - shared and specific' do - success Entities::Runner + success Entities::Ci::Runner end params do optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_SCOPES, @@ -55,11 +55,11 @@ module API runners = filter_runners(runners, params[:status], allowed_scopes: ::Ci::Runner::AVAILABLE_STATUSES) runners = runners.tagged_with(params[:tag_list]) if params[:tag_list] - present paginate(runners), with: Entities::Runner + present paginate(runners), with: Entities::Ci::Runner end desc "Get runner's details" do - success Entities::RunnerDetails + success Entities::Ci::RunnerDetails end params do requires :id, type: Integer, desc: 'The ID of the runner' @@ -68,11 +68,11 @@ module API runner = get_runner(params[:id]) authenticate_show_runner!(runner) - present runner, with: Entities::RunnerDetails, current_user: current_user + present runner, with: Entities::Ci::RunnerDetails, current_user: current_user end desc "Update runner's details" do - success Entities::RunnerDetails + success Entities::Ci::RunnerDetails end params do requires :id, type: Integer, desc: 'The ID of the runner' @@ -92,14 +92,14 @@ module API update_service = ::Ci::UpdateRunnerService.new(runner) if update_service.update(declared_params(include_missing: false)) - present runner, with: Entities::RunnerDetails, current_user: current_user + present runner, with: Entities::Ci::RunnerDetails, current_user: current_user else render_validation_error!(runner) end end desc 'Remove a runner' do - success Entities::Runner + success Entities::Ci::Runner end params do requires :id, type: Integer, desc: 'The ID of the runner' @@ -139,7 +139,7 @@ module API before { authorize_admin_project } desc 'Get runners available for project' do - success Entities::Runner + success Entities::Ci::Runner end params do optional :scope, type: String, values: ::Ci::Runner::AVAILABLE_SCOPES, @@ -158,11 +158,11 @@ module API runners = filter_runners(runners, params[:scope]) runners = apply_filter(runners, params) - present paginate(runners), with: Entities::Runner + present paginate(runners), with: Entities::Ci::Runner end desc 'Enable a runner for a project' do - success Entities::Runner + success Entities::Ci::Runner end params do requires :runner_id, type: Integer, desc: 'The ID of the runner' @@ -172,14 +172,14 @@ module API authenticate_enable_runner!(runner) if runner.assign_to(user_project) - present runner, with: Entities::Runner + present runner, with: Entities::Ci::Runner else render_validation_error!(runner) end end desc "Disable project's runner" do - success Entities::Runner + success Entities::Ci::Runner end params do requires :runner_id, type: Integer, desc: 'The ID of the runner' @@ -204,7 +204,7 @@ module API before { authorize_admin_group } desc 'Get runners available for group' do - success Entities::Runner + success Entities::Ci::Runner end params do optional :type, type: String, values: ::Ci::Runner::AVAILABLE_TYPES, @@ -218,7 +218,7 @@ module API runners = ::Ci::Runner.belonging_to_group(user_group.id, include_ancestors: true) runners = apply_filter(runners, params) - present paginate(runners), with: Entities::Runner + present paginate(runners), with: Entities::Ci::Runner end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 27fee7fdea2..1785362656e 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -99,40 +99,26 @@ module API updatable_optional_attributes = %w[target_url description coverage] status.assign_attributes(attributes_for_keys(updatable_optional_attributes)) - if status.valid? - status.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, user_project, default_enabled: :yaml) - else - render_validation_error!(status) - end + render_validation_error!(status) unless status.valid? - begin - case params[:state] - when 'pending' - status.enqueue! - when 'running' - status.enqueue - status.run! - when 'success' - status.success! - when 'failed' - status.drop!(:api_failure) - when 'canceled' - status.cancel! - else - render_api_error!('invalid state', 400) - end + response = ::Ci::Pipelines::AddJobService.new(pipeline).execute!(status) do |job| + apply_job_state!(job) + rescue ::StateMachines::InvalidTransition => e + render_api_error!(e.message, 400) + end - if pipeline.latest? - MergeRequest.where(source_project: user_project, source_branch: ref) - .update_all(head_pipeline_id: pipeline.id) - end + render_validation_error!(response.payload[:job]) unless response.success? - present status, with: Entities::CommitStatus - rescue StateMachines::InvalidTransition => e - render_api_error!(e.message, 400) + if pipeline.latest? + MergeRequest + .where(source_project: user_project, source_branch: ref) + .update_all(head_pipeline_id: pipeline.id) end + + present response.payload[:job], with: Entities::CommitStatus end # rubocop: enable CodeReuse/ActiveRecord + helpers do def commit strong_memoize(:commit) do @@ -146,6 +132,24 @@ module API pipelines = pipelines.for_id(params[:pipeline_id]) if params[:pipeline_id] pipelines end + + def apply_job_state!(job) + case params[:state] + when 'pending' + job.enqueue! + when 'running' + job.enqueue + job.run! + when 'success' + job.success! + when 'failed' + job.drop!(:api_failure) + when 'canceled' + job.cancel! + else + render_api_error!('invalid state', 400) + end + end end end end diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index c79ae3068b4..7740ba6bfa6 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -6,20 +6,17 @@ module API module DebianPackageEndpoints extend ActiveSupport::Concern - DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze - COMPONENT_REGEX = %r{[a-z-]+}.freeze - ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX DISTRIBUTION_REQUIREMENTS = { - distribution: DISTRIBUTION_REGEX + distribution: ::Packages::Debian::DISTRIBUTION_REGEX }.freeze COMPONENT_ARCHITECTURE_REQUIREMENTS = { - component: COMPONENT_REGEX, - architecture: ARCHITECTURE_REGEX + component: ::Packages::Debian::COMPONENT_REGEX, + architecture: ::Packages::Debian::ARCHITECTURE_REGEX }.freeze COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { - component: COMPONENT_REGEX, + component: ::Packages::Debian::COMPONENT_REGEX, letter: LETTER_REGEX, source_package: PACKAGE_REGEX }.freeze @@ -40,6 +37,14 @@ module API .sent_through(:http_basic_auth) end + helpers do + def present_release_file + distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename_or_suite: params[:distribution]).execute.last! + + present_carrierwave_file!(distribution.file) + end + end + format :txt content_type :txt, 'text/plain' @@ -65,8 +70,7 @@ module API route_setting :authentication, authenticate_non_public: true get 'Release' do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO Release' + present_release_file end # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease @@ -76,7 +80,8 @@ module API route_setting :authentication, authenticate_non_public: true get 'InRelease' do - not_found! + # Signature to be added in 7.3 of https://gitlab.com/groups/gitlab-org/-/epics/6057#note_582697034 + present_release_file end params do @@ -92,8 +97,20 @@ module API route_setting :authentication, authenticate_non_public: true get 'Packages' do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO Packages' + relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize + + component_file = relation + .preload_distribution + .with_container(project_or_group) + .with_codename_or_suite(params[:distribution]) + .with_component_name(params[:component]) + .with_file_type(:packages) + .with_architecture_name(params[:architecture]) + .with_compression_type(nil) + .order_created_asc + .last! + + present_carrierwave_file!(component_file.file) end end end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index c6116a8b28f..191ed42a5b8 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -24,6 +24,12 @@ module API end namespace ':id/-' do + helpers do + def project_or_group + user_group + end + end + include ::API::Concerns::Packages::DebianPackageEndpoints end end diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index c75b74b4368..0b231906ccd 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -6,7 +6,7 @@ module API include ::API::ProjectsRelationBuilder include Gitlab::Utils::StrongMemoize - expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } + expose :default_branch_or_main, as: :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 expose :topic_names, as: :tag_list diff --git a/lib/api/entities/bulk_import.rb b/lib/api/entities/bulk_import.rb new file mode 100644 index 00000000000..373ae486dcf --- /dev/null +++ b/lib/api/entities/bulk_import.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + class BulkImport < Grape::Entity + expose :id + expose :status_name, as: :status + expose :source_type + expose :created_at + expose :updated_at + end + end +end diff --git a/lib/api/entities/bulk_imports/entity.rb b/lib/api/entities/bulk_imports/entity.rb new file mode 100644 index 00000000000..e8c31256b17 --- /dev/null +++ b/lib/api/entities/bulk_imports/entity.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module API + module Entities + module BulkImports + class Entity < Grape::Entity + expose :id + expose :bulk_import_id + expose :status_name, as: :status + expose :source_full_path + expose :destination_name + expose :destination_namespace + expose :parent_id + expose :namespace_id + expose :project_id + expose :created_at + expose :updated_at + expose :failures, using: EntityFailure + end + end + end +end diff --git a/lib/api/entities/bulk_imports/entity_failure.rb b/lib/api/entities/bulk_imports/entity_failure.rb new file mode 100644 index 00000000000..a3dbe3280ee --- /dev/null +++ b/lib/api/entities/bulk_imports/entity_failure.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Entities + module BulkImports + class EntityFailure < Grape::Entity + expose :pipeline_class + expose :pipeline_step + expose :exception_class + expose :correlation_id_value + expose :created_at + end + end + end +end diff --git a/lib/api/entities/ci/job.rb b/lib/api/entities/ci/job.rb index 76487ed01dc..cf87684ce55 100644 --- a/lib/api/entities/ci/job.rb +++ b/lib/api/entities/ci/job.rb @@ -7,7 +7,7 @@ module API # artifacts_file is included in job_artifacts, but kept for backward compatibility (remove in api/v5) expose :artifacts_file, using: ::API::Entities::Ci::JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :job_artifacts, as: :artifacts, using: ::API::Entities::Ci::JobArtifact - expose :runner, with: ::API::Entities::Runner + expose :runner, with: ::API::Entities::Ci::Runner expose :artifacts_expire_at expose :tag_list do |job| job.tags.map(&:name).sort diff --git a/lib/api/entities/ci/job_request/artifacts.rb b/lib/api/entities/ci/job_request/artifacts.rb new file mode 100644 index 00000000000..4b09db40504 --- /dev/null +++ b/lib/api/entities/ci/job_request/artifacts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Artifacts < Grape::Entity + expose :name + expose :untracked + expose :paths + expose :exclude, expose_nil: false + expose :when + expose :expire_in + expose :artifact_type + expose :artifact_format + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/cache.rb b/lib/api/entities/ci/job_request/cache.rb new file mode 100644 index 00000000000..9820719b4f0 --- /dev/null +++ b/lib/api/entities/ci/job_request/cache.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Cache < Grape::Entity + expose :key, :untracked, :paths, :policy, :when + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/credentials.rb b/lib/api/entities/ci/job_request/credentials.rb new file mode 100644 index 00000000000..57cdd9c9b19 --- /dev/null +++ b/lib/api/entities/ci/job_request/credentials.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Credentials < Grape::Entity + expose :type, :url, :username, :password + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/dependency.rb b/lib/api/entities/ci/job_request/dependency.rb new file mode 100644 index 00000000000..2c6ed417714 --- /dev/null +++ b/lib/api/entities/ci/job_request/dependency.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Dependency < Grape::Entity + expose :id, :name, :token + expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.artifacts? } + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/git_info.rb b/lib/api/entities/ci/job_request/git_info.rb new file mode 100644 index 00000000000..872c896b870 --- /dev/null +++ b/lib/api/entities/ci/job_request/git_info.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class GitInfo < Grape::Entity + expose :repo_url, :ref, :sha, :before_sha + expose :ref_type + expose :refspecs + expose :git_depth, as: :depth + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/image.rb b/lib/api/entities/ci/job_request/image.rb new file mode 100644 index 00000000000..8e404a8fa02 --- /dev/null +++ b/lib/api/entities/ci/job_request/image.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Image < Grape::Entity + expose :name, :entrypoint + expose :ports, using: Entities::Ci::JobRequest::Port + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/job_info.rb b/lib/api/entities/ci/job_request/job_info.rb new file mode 100644 index 00000000000..5c3f4b08af2 --- /dev/null +++ b/lib/api/entities/ci/job_request/job_info.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class JobInfo < Grape::Entity + expose :id, :name, :stage + expose :project_id, :project_name + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/port.rb b/lib/api/entities/ci/job_request/port.rb new file mode 100644 index 00000000000..31aa06ff843 --- /dev/null +++ b/lib/api/entities/ci/job_request/port.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Port < Grape::Entity + expose :number, :protocol, :name + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/response.rb b/lib/api/entities/ci/job_request/response.rb new file mode 100644 index 00000000000..86c945cb236 --- /dev/null +++ b/lib/api/entities/ci/job_request/response.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Response < Grape::Entity + expose :id + expose :token + expose :allow_git_fetch + + expose :job_info, using: Entities::Ci::JobRequest::JobInfo do |model| + model + end + + expose :git_info, using: Entities::Ci::JobRequest::GitInfo do |model| + model + end + + expose :runner_info, using: Entities::Ci::JobRequest::RunnerInfo do |model| + model + end + + expose :runner_variables, as: :variables + expose :steps, using: Entities::Ci::JobRequest::Step + expose :image, using: Entities::Ci::JobRequest::Image + expose :services, using: Entities::Ci::JobRequest::Service + expose :artifacts, using: Entities::Ci::JobRequest::Artifacts + expose :cache, using: Entities::Ci::JobRequest::Cache + expose :credentials, using: Entities::Ci::JobRequest::Credentials + expose :all_dependencies, as: :dependencies, using: Entities::Ci::JobRequest::Dependency + expose :features + end + end + end + end +end + +API::Entities::Ci::JobRequest::Response.prepend_mod_with('API::Entities::Ci::JobRequest::Response') diff --git a/lib/api/entities/ci/job_request/runner_info.rb b/lib/api/entities/ci/job_request/runner_info.rb new file mode 100644 index 00000000000..96336a1080e --- /dev/null +++ b/lib/api/entities/ci/job_request/runner_info.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class RunnerInfo < Grape::Entity + expose :metadata_timeout, as: :timeout + expose :runner_session_url + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb new file mode 100644 index 00000000000..f89b95c1d5c --- /dev/null +++ b/lib/api/entities/ci/job_request/service.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Service < Entities::Ci::JobRequest::Image + expose :alias, :command + end + end + end + end +end diff --git a/lib/api/entities/ci/job_request/step.rb b/lib/api/entities/ci/job_request/step.rb new file mode 100644 index 00000000000..2a0c4cd032e --- /dev/null +++ b/lib/api/entities/ci/job_request/step.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module JobRequest + class Step < Grape::Entity + expose :name, :script, :timeout, :when, :allow_failure + end + end + end + end +end diff --git a/lib/api/entities/ci/runner.rb b/lib/api/entities/ci/runner.rb new file mode 100644 index 00000000000..ede698696de --- /dev/null +++ b/lib/api/entities/ci/runner.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + class Runner < Grape::Entity + expose :id + expose :description + expose :ip_address + expose :active + expose :instance_type?, as: :is_shared + expose :runner_type + expose :name + expose :online?, as: :online + expose :status + end + end + end +end diff --git a/lib/api/entities/ci/runner_details.rb b/lib/api/entities/ci/runner_details.rb new file mode 100644 index 00000000000..9d44da7e5b3 --- /dev/null +++ b/lib/api/entities/ci/runner_details.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + class RunnerDetails < Runner + expose :tag_list + expose :run_untagged + expose :locked + expose :maximum_timeout + expose :access_level + expose :version, :revision, :platform, :architecture + expose :contacted_at + + # rubocop: disable CodeReuse/ActiveRecord + expose :projects, with: Entities::BasicProjectDetails do |runner, options| + if options[:current_user].admin? # rubocop: disable Cop/UserAdmin + runner.projects + else + options[:current_user].authorized_projects.where(id: runner.projects) + end + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord + expose :groups, with: Entities::BasicGroupDetails do |runner, options| + if options[:current_user].admin? # rubocop: disable Cop/UserAdmin + runner.groups + else + options[:current_user].authorized_groups.where(id: runner.groups) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/api/entities/ci/runner_registration_details.rb b/lib/api/entities/ci/runner_registration_details.rb new file mode 100644 index 00000000000..fa7e44c9e40 --- /dev/null +++ b/lib/api/entities/ci/runner_registration_details.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + class RunnerRegistrationDetails < Grape::Entity + expose :id, :token + end + end + end +end diff --git a/lib/api/entities/group_detail.rb b/lib/api/entities/group_detail.rb index 408254a89be..61f35d0f784 100644 --- a/lib/api/entities/group_detail.rb +++ b/lib/api/entities/group_detail.rb @@ -7,6 +7,7 @@ module API SharedGroupWithGroup.represent(group.shared_with_group_links.public_or_visible_to_user(group, options[:current_user])) end expose :runners_token, if: lambda { |group, options| options[:user_can_admin_group] } + expose :prevent_sharing_groups_outside_hierarchy, if: ->(group) { group.root? } expose :projects, using: Entities::Project do |group, options| projects = GroupProjectsFinder.new( diff --git a/lib/api/entities/helm/index.rb b/lib/api/entities/helm/index.rb new file mode 100644 index 00000000000..168298f24b6 --- /dev/null +++ b/lib/api/entities/helm/index.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Helm + class Index < Grape::Entity + expose :api_version, as: :apiVersion + expose :entries + expose :generated + expose :server_info, as: :serverInfo + end + end + end +end diff --git a/lib/api/entities/job_request/artifacts.rb b/lib/api/entities/job_request/artifacts.rb deleted file mode 100644 index 0d27f5a9189..00000000000 --- a/lib/api/entities/job_request/artifacts.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Artifacts < Grape::Entity - expose :name - expose :untracked - expose :paths - expose :exclude, expose_nil: false - expose :when - expose :expire_in - expose :artifact_type - expose :artifact_format - end - end - end -end diff --git a/lib/api/entities/job_request/cache.rb b/lib/api/entities/job_request/cache.rb deleted file mode 100644 index cd533d7e5b3..00000000000 --- a/lib/api/entities/job_request/cache.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Cache < Grape::Entity - expose :key, :untracked, :paths, :policy, :when - end - end - end -end diff --git a/lib/api/entities/job_request/credentials.rb b/lib/api/entities/job_request/credentials.rb deleted file mode 100644 index cdac5566cbd..00000000000 --- a/lib/api/entities/job_request/credentials.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Credentials < Grape::Entity - expose :type, :url, :username, :password - end - end - end -end diff --git a/lib/api/entities/job_request/dependency.rb b/lib/api/entities/job_request/dependency.rb deleted file mode 100644 index 7d6ec832ba1..00000000000 --- a/lib/api/entities/job_request/dependency.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Dependency < Grape::Entity - expose :id, :name, :token - expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.artifacts? } - end - end - end -end diff --git a/lib/api/entities/job_request/git_info.rb b/lib/api/entities/job_request/git_info.rb deleted file mode 100644 index e07099263b5..00000000000 --- a/lib/api/entities/job_request/git_info.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class GitInfo < Grape::Entity - expose :repo_url, :ref, :sha, :before_sha - expose :ref_type - expose :refspecs - expose :git_depth, as: :depth - end - end - end -end diff --git a/lib/api/entities/job_request/image.rb b/lib/api/entities/job_request/image.rb deleted file mode 100644 index 47f4542d2d5..00000000000 --- a/lib/api/entities/job_request/image.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Image < Grape::Entity - expose :name, :entrypoint - expose :ports, using: Entities::JobRequest::Port - end - end - end -end diff --git a/lib/api/entities/job_request/job_info.rb b/lib/api/entities/job_request/job_info.rb deleted file mode 100644 index a4bcc9726d0..00000000000 --- a/lib/api/entities/job_request/job_info.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class JobInfo < Grape::Entity - expose :id, :name, :stage - expose :project_id, :project_name - end - end - end -end diff --git a/lib/api/entities/job_request/port.rb b/lib/api/entities/job_request/port.rb deleted file mode 100644 index ee427da8657..00000000000 --- a/lib/api/entities/job_request/port.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Port < Grape::Entity - expose :number, :protocol, :name - end - end - end -end diff --git a/lib/api/entities/job_request/response.rb b/lib/api/entities/job_request/response.rb deleted file mode 100644 index 2e8dfc5bde0..00000000000 --- a/lib/api/entities/job_request/response.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Response < Grape::Entity - expose :id - expose :token - expose :allow_git_fetch - - expose :job_info, using: Entities::JobRequest::JobInfo do |model| - model - end - - expose :git_info, using: Entities::JobRequest::GitInfo do |model| - model - end - - expose :runner_info, using: Entities::JobRequest::RunnerInfo do |model| - model - end - - expose :runner_variables, as: :variables - expose :steps, using: Entities::JobRequest::Step - expose :image, using: Entities::JobRequest::Image - expose :services, using: Entities::JobRequest::Service - expose :artifacts, using: Entities::JobRequest::Artifacts - expose :cache, using: Entities::JobRequest::Cache - expose :credentials, using: Entities::JobRequest::Credentials - expose :all_dependencies, as: :dependencies, using: Entities::JobRequest::Dependency - expose :features - end - end - end -end - -API::Entities::JobRequest::Response.prepend_mod_with('API::Entities::JobRequest::Response') diff --git a/lib/api/entities/job_request/runner_info.rb b/lib/api/entities/job_request/runner_info.rb deleted file mode 100644 index e6d2e8d9e85..00000000000 --- a/lib/api/entities/job_request/runner_info.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class RunnerInfo < Grape::Entity - expose :metadata_timeout, as: :timeout - expose :runner_session_url - end - end - end -end diff --git a/lib/api/entities/job_request/service.rb b/lib/api/entities/job_request/service.rb deleted file mode 100644 index 9ad5abf4e9e..00000000000 --- a/lib/api/entities/job_request/service.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Service < Entities::JobRequest::Image - expose :alias, :command - end - end - end -end diff --git a/lib/api/entities/job_request/step.rb b/lib/api/entities/job_request/step.rb deleted file mode 100644 index 498dd017fb4..00000000000 --- a/lib/api/entities/job_request/step.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module JobRequest - class Step < Grape::Entity - expose :name, :script, :timeout, :when, :allow_failure - end - end - end -end diff --git a/lib/api/entities/label.rb b/lib/api/entities/label.rb index ca9a0912331..dc147f33671 100644 --- a/lib/api/entities/label.rb +++ b/lib/api/entities/label.rb @@ -18,7 +18,9 @@ module API end expose :subscribed do |label, options| - label.subscribed?(options[:current_user], options[:parent]) + label.subscribed?(options[:current_user]) || ( + options[:parent].is_a?(::Project) && label.subscribed?(options[:current_user], options[:parent]) + ) end end end diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb index 40e8b348c18..04ec44b5167 100644 --- a/lib/api/entities/plan_limit.rb +++ b/lib/api/entities/plan_limit.rb @@ -9,6 +9,7 @@ module API expose :npm_max_file_size expose :nuget_max_file_size expose :pypi_max_file_size + expose :terraform_module_max_file_size end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 68d91fc6970..f5f565e5b07 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -53,13 +53,7 @@ module API expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } - expose(:container_registry_enabled) do |project, options| - if ::Feature.enabled?(:read_container_registry_access_level, project.namespace, default_enabled: :yaml) - project.feature_available?(:container_registry, options[:current_user]) - else - project.read_attribute(:container_registry_enabled) - end - end + expose(:container_registry_enabled) { |project, options| project.feature_available?(:container_registry, options[:current_user]) } expose :service_desk_enabled expose :service_desk_address diff --git a/lib/api/entities/project_integration.rb b/lib/api/entities/project_integration.rb new file mode 100644 index 00000000000..649e4d015b8 --- /dev/null +++ b/lib/api/entities/project_integration.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module API + module Entities + class ProjectIntegration < Entities::ProjectIntegrationBasic + # Expose serialized properties + expose :properties do |integration, options| + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + + attributes = + if integration.data_fields_present? + integration.data_fields.as_json.keys + else + integration.properties.keys + end + + attributes &= integration.api_field_names + + attributes.each_with_object({}) do |attribute, hash| + hash[attribute] = integration.public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/lib/api/entities/project_service_basic.rb b/lib/api/entities/project_integration_basic.rb index eb97ca69a82..2870123b83d 100644 --- a/lib/api/entities/project_service_basic.rb +++ b/lib/api/entities/project_integration_basic.rb @@ -2,10 +2,10 @@ module API module Entities - class ProjectServiceBasic < Grape::Entity + class ProjectIntegrationBasic < Grape::Entity expose :id, :title - expose :slug do |service| - service.to_param.dasherize + expose :slug do |integration| + integration.to_param.dasherize end expose :created_at, :updated_at, :active expose :commit_events, :push_events, :issues_events, :confidential_issues_events diff --git a/lib/api/entities/project_service.rb b/lib/api/entities/project_service.rb deleted file mode 100644 index 947cec1e3cd..00000000000 --- a/lib/api/entities/project_service.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - class ProjectService < Entities::ProjectServiceBasic - # Expose serialized properties - expose :properties do |service, options| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - if service.data_fields_present? - service.data_fields.as_json.slice(*service.api_field_names) - else - service.properties.slice(*service.api_field_names) - end - end - end - end -end diff --git a/lib/api/entities/resource_access_token.rb b/lib/api/entities/resource_access_token.rb new file mode 100644 index 00000000000..a1c7b28af45 --- /dev/null +++ b/lib/api/entities/resource_access_token.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + class ResourceAccessToken < Entities::PersonalAccessToken + expose :access_level do |token, options| + options[:project].project_member(token.user).access_level + end + end + end +end diff --git a/lib/api/entities/resource_access_token_with_token.rb b/lib/api/entities/resource_access_token_with_token.rb new file mode 100644 index 00000000000..edbd9b285de --- /dev/null +++ b/lib/api/entities/resource_access_token_with_token.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class ResourceAccessTokenWithToken < Entities::ResourceAccessToken + expose :token + end + end +end diff --git a/lib/api/entities/runner.rb b/lib/api/entities/runner.rb deleted file mode 100644 index e78f14cf920..00000000000 --- a/lib/api/entities/runner.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - class Runner < Grape::Entity - expose :id - expose :description - expose :ip_address - expose :active - expose :instance_type?, as: :is_shared - expose :runner_type - expose :name - expose :online?, as: :online - expose :status - end - end -end diff --git a/lib/api/entities/runner_details.rb b/lib/api/entities/runner_details.rb deleted file mode 100644 index 0afe298ef64..00000000000 --- a/lib/api/entities/runner_details.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - class RunnerDetails < Runner - expose :tag_list - expose :run_untagged - expose :locked - expose :maximum_timeout - expose :access_level - expose :version, :revision, :platform, :architecture - expose :contacted_at - - # rubocop: disable CodeReuse/ActiveRecord - expose :projects, with: Entities::BasicProjectDetails do |runner, options| - if options[:current_user].admin? - runner.projects - else - options[:current_user].authorized_projects.where(id: runner.projects) - end - end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord - expose :groups, with: Entities::BasicGroupDetails do |runner, options| - if options[:current_user].admin? - runner.groups - else - options[:current_user].authorized_groups.where(id: runner.groups) - end - end - # rubocop: enable CodeReuse/ActiveRecord - end - end -end diff --git a/lib/api/entities/runner_registration_details.rb b/lib/api/entities/runner_registration_details.rb deleted file mode 100644 index c8ed88ba10a..00000000000 --- a/lib/api/entities/runner_registration_details.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - class RunnerRegistrationDetails < Grape::Entity - expose :id, :token - end - end -end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 3ce6d03e236..973e80dd5ef 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -5,7 +5,7 @@ module API class User < UserBasic include UsersHelper expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } - expose :bio, :bio_html, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title + expose :bio, :bio_html, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title, :pronouns expose :bot?, as: :bot expose :work_information do |user| work_information(user) diff --git a/lib/api/error_tracking_collector.rb b/lib/api/error_tracking_collector.rb new file mode 100644 index 00000000000..08ff8d2e4d1 --- /dev/null +++ b/lib/api/error_tracking_collector.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module API + # This API is responsible for collecting error tracking information + # from sentry client. It allows us to use GitLab as an alternative to + # sentry backend. For more details see https://gitlab.com/gitlab-org/gitlab/-/issues/329596. + class ErrorTrackingCollector < ::API::Base + feature_category :error_tracking + + content_type :envelope, 'application/x-sentry-envelope' + default_format :envelope + + before do + not_found!('Project') unless project + not_found! unless feature_enabled? + end + + helpers do + def project + @project ||= find_project(params[:id]) + end + + def feature_enabled? + ::Feature.enabled?(:integrated_error_tracking, project) && + project.error_tracking_setting&.enabled? + end + end + + desc 'Submit error tracking event to the project' do + detail 'This feature was introduced in GitLab 14.1.' + end + params do + requires :id, type: String, desc: 'The ID of a project' + end + post 'error_tracking/collector/api/:id/envelope' do + # There is a reason why we have such uncommon path. + # We depend on a client side error tracking software which + # modifies URL for its own reasons. + # + # When we give user a URL like this + # HOST/api/v4/error_tracking/collector/123 + # + # Then error tracking software will convert it like this: + # HOST/api/v4/error_tracking/collector/api/123/envelope/ + + begin + parsed_request = ::ErrorTracking::Collector::SentryRequestParser.parse(request) + rescue StandardError + render_api_error!('Failed to parse sentry request', 400) + end + + type = parsed_request[:request_type] + + # Sentry sends 2 requests on each exception: transaction and event. + # Everything else is not a desired behavior. + unless type == 'transaction' || type == 'event' + render_api_error!('400 Bad Request', 400) + + break + end + + # We don't have use for transaction request yet, + # so we record only event one. + if type == 'event' + ::ErrorTracking::CollectErrorService + .new(project, nil, event: parsed_request[:event]) + .execute + end + + no_content! + end + end +end diff --git a/lib/api/geo.rb b/lib/api/geo.rb new file mode 100644 index 00000000000..9fc610c9b32 --- /dev/null +++ b/lib/api/geo.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module API + class Geo < ::API::Base + feature_category :geo_replication + + helpers do + # Overridden in EE + def geo_proxy_response + {} + end + end + + resource :geo do + # Workhorse calls this to determine if it is a Geo site that should proxy + # requests. Workhorse doesn't know if it's in a FOSS/EE context. + get '/proxy' do + require_gitlab_workhorse! + + status :ok + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + + geo_proxy_response + end + end + end +end + +API::Geo.prepend_mod diff --git a/lib/api/group_avatar.rb b/lib/api/group_avatar.rb index ddf6787f913..9063040c763 100644 --- a/lib/api/group_avatar.rb +++ b/lib/api/group_avatar.rb @@ -6,15 +6,27 @@ module API feature_category :subgroups - resource :groups do + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Download the group avatar' do detail 'This feature was introduced in GitLab 14.0' end - params do - requires :id, type: String, desc: 'The group id' - end get ':id/avatar' do - present_carrierwave_file!(user_group.avatar) + avatar = user_group.avatar + + not_found!('Avatar') if avatar.blank? + + header( + 'Content-Disposition', + ActionDispatch::Http::ContentDisposition.format( + disposition: 'attachment', + filename: avatar.filename + ) + ) + + present_carrierwave_file!(avatar) end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 0efb8b57885..9b6b28733ff 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -22,7 +22,7 @@ module API optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :order_by, type: String, values: %w[name path id], default: 'name', desc: 'Order by name, path or id' + optional :order_by, type: String, values: %w[name path id similarity], default: 'name', desc: 'Order by name, path, id or similarity if searching' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Minimum access level of authenticated user' optional :top_level_only, type: Boolean, desc: 'Only include top level groups' @@ -50,9 +50,8 @@ module API groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search], include_parents: true) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - order_options = { params[:order_by] => params[:sort] } - order_options["id"] ||= "asc" - groups.reorder(order_options) + + order_groups(groups) end # rubocop: enable CodeReuse/ActiveRecord @@ -123,6 +122,23 @@ module API reorder_projects(projects) end + def order_groups(groups) + return groups.sorted_by_similarity_and_parent_id_desc(params[:search]) if order_by_similarity? + + groups.reorder(group_without_similarity_options) # rubocop: disable CodeReuse/ActiveRecord + end + + def order_by_similarity? + params[:order_by] == 'similarity' && params[:search].present? + end + + def group_without_similarity_options + order_options = { params[:order_by] => params[:sort] } + order_options['name'] = order_options.delete('similarity') if order_options.has_key?('similarity') + order_options["id"] ||= "asc" + order_options + end + # rubocop: disable CodeReuse/ActiveRecord def handle_similarity_order(group, projects) if params[:search].present? && Feature.enabled?(:similarity_search, group, default_enabled: true) @@ -199,6 +215,7 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params + use :optional_update_params use :optional_update_params_ee end put ':id' do diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index dc5630a1395..4280744d8b4 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -10,11 +10,15 @@ module API feature_category :package_registry + PACKAGE_FILENAME = 'package.tgz' FILE_NAME_REQUIREMENTS = { file_name: API::NO_SLASH_URL_PART_REGEX }.freeze content_type :binary, 'application/octet-stream' + content_type :yaml, 'text/yaml' + + formatter :yaml, -> (object, _) { object.serializable_hash.stringify_keys.to_yaml } authenticate_with do |accept| accept.token_types(:personal_access_token, :deploy_token, :job_token) @@ -25,15 +29,33 @@ module API require_packages_enabled! end - after_validation do - not_found! unless Feature.enabled?(:helm_packages, authorized_user_project) - end - params do requires :id, type: String, desc: 'The ID or full path of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/helm' do + desc 'Download a chart index' do + detail 'This feature was introduced in GitLab 14.0' + end + params do + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + end + + get ":channel/index.yaml" do + authorize_read_package!(authorized_user_project) + + package_files = Packages::Helm::PackageFilesFinder.new( + authorized_user_project, + params[:channel], + order_by: 'created_at', + sort: 'desc' + ).execute + + env['api.format'] = :yaml + present ::Packages::Helm::IndexPresenter.new(authorized_user_project, params[:id], package_files), + with: ::API::Entities::Helm::Index + end + desc 'Download a chart' do detail 'This feature was introduced in GitLab 14.0' end @@ -50,6 +72,55 @@ module API present_carrierwave_file!(package_file.file) end + + desc 'Authorize a chart upload from workhorse' do + detail 'This feature was introduced in GitLab 14.0' + end + params do + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + end + post "api/:channel/charts/authorize" do + authorize_workhorse!( + subject: authorized_user_project, + has_length: false, + maximum_size: authorized_user_project.actual_limits.helm_max_file_size + ) + end + + desc 'Upload a chart' do + detail 'This feature was introduced in GitLab 14.0' + end + params do + requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex + requires :chart, type: ::API::Validations::Types::WorkhorseFile, desc: 'The chart file to be published (generated by Multipart middleware)' + end + post "api/:channel/charts" do + authorize_upload!(authorized_user_project) + bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:helm_max_file_size, params[:chart].size) + + package = ::Packages::CreateTemporaryPackageService.new( + authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job) + ).execute(:helm, name: ::Packages::Helm::TEMPORARY_PACKAGE_NAME) + + chart_params = { + file: params[:chart], + file_name: PACKAGE_FILENAME + } + + chart_package_file = ::Packages::CreatePackageFileService.new( + package, chart_params.merge(build: current_authenticated_job) + ).execute + + track_package_event('push_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) + + ::Packages::Helm::ExtractionWorker.perform_async(params[:channel], chart_package_file.id) # rubocop:disable CodeReuse/Worker + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { channel: params[:channel], project_id: authorized_user_project.id }) + + forbidden! + end end end end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6ce04be373f..3398d5da7f5 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -577,10 +577,6 @@ module API Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end - def with_api_params(&block) - yield({ api: true, request: request }) - end - protected def project_finder_params_visibility_ce diff --git a/lib/api/helpers/caching.rb b/lib/api/helpers/caching.rb index f24ac7302c1..f567d85443f 100644 --- a/lib/api/helpers/caching.rb +++ b/lib/api/helpers/caching.rb @@ -8,18 +8,15 @@ module API module Helpers module Caching - # @return [ActiveSupport::Duration] - DEFAULT_EXPIRY = 1.day - + include Gitlab::Cache::Helpers # @return [Hash] DEFAULT_CACHE_OPTIONS = { - race_condition_ttl: 5.seconds + race_condition_ttl: 5.seconds, + version: 1 }.freeze - # @return [ActiveSupport::Cache::Store] - def cache - Rails.cache - end + # @return [Array] + PAGINATION_HEADERS = %w[X-Per-Page X-Page X-Next-Page X-Prev-Page Link X-Total X-Total-Pages].freeze # This is functionally equivalent to the standard `#present` used in # Grape endpoints, but the JSON for the object, or for each object of @@ -45,7 +42,7 @@ module API # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @param presenter_args [Hash] keyword arguments to be passed to the entity # @return [Gitlab::Json::PrecompiledJson] - def present_cached(obj_or_collection, with:, cache_context: -> (_) { current_user&.cache_key }, expires_in: DEFAULT_EXPIRY, **presenter_args) + def present_cached(obj_or_collection, with:, cache_context: -> (_) { current_user&.cache_key }, expires_in: Gitlab::Cache::Helpers::DEFAULT_EXPIRY, **presenter_args) json = if obj_or_collection.is_a?(Enumerable) cached_collection( @@ -79,15 +76,22 @@ module API # @param key [Object] any object that can be converted into a cache key # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @return [Gitlab::Json::PrecompiledJson] - def cache_action(key, **cache_opts) - json = cache.fetch(key, **apply_default_cache_options(cache_opts)) do + def cache_action(key, **custom_cache_opts) + cache_opts = apply_default_cache_options(custom_cache_opts) + + json, cached_headers = cache.fetch(key, **cache_opts) do response = yield - if response.is_a?(Gitlab::Json::PrecompiledJson) - response.to_s - else - Gitlab::Json.dump(response.as_json) - end + cached_body = response.is_a?(Gitlab::Json::PrecompiledJson) ? response.to_s : Gitlab::Json.dump(response.as_json) + cached_headers = header.slice(*PAGINATION_HEADERS) + + [cached_body, cached_headers] + end + + cached_headers.each do |key, value| + next if header.key?(key) + + header key, value end body Gitlab::Json::PrecompiledJson.new(json) @@ -120,77 +124,6 @@ module API def apply_default_cache_options(opts = {}) DEFAULT_CACHE_OPTIONS.merge(opts) end - - # Optionally uses a `Proc` to add context to a cache key - # - # @param object [Object] must respond to #cache_key - # @param context [Proc] a proc that will be called with the object as an argument, and which should return a - # string or array of strings to be combined into the cache key - # @return [String] - def contextual_cache_key(object, context) - return object.cache_key if context.nil? - - [object.cache_key, context.call(object)].flatten.join(":") - end - - # Used for fetching or rendering a single object - # - # @param object [Object] the object to render - # @param presenter [Grape::Entity] - # @param presenter_args [Hash] keyword arguments to be passed to the entity - # @param context [Proc] - # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry - # @return [String] - def cached_object(object, presenter:, presenter_args:, context:, expires_in:) - cache.fetch(contextual_cache_key(object, context), expires_in: expires_in) do - Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) - end - end - - # Used for fetching or rendering multiple objects - # - # @param objects [Enumerable<Object>] the objects to render - # @param presenter [Grape::Entity] - # @param presenter_args [Hash] keyword arguments to be passed to the entity - # @param context [Proc] - # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry - # @return [Array<String>] - def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:) - json = fetch_multi(collection, context: context, expires_in: expires_in) do |obj| - Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) - end - - json.values - end - - # An adapted version of ActiveSupport::Cache::Store#fetch_multi. - # - # The original method only provides the missing key to the block, - # not the missing object, so we have to create a map of cache keys - # to the objects to allow us to pass the object to the missing value - # block. - # - # The result is that this is functionally identical to `#fetch`. - def fetch_multi(*objs, context:, **kwargs) - objs.flatten! - map = multi_key_map(objs, context: context) - - # TODO: `contextual_cache_key` should be constructed based on the guideline https://docs.gitlab.com/ee/development/redis.html#multi-key-commands. - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - cache.fetch_multi(*map.keys, **kwargs) do |key| - yield map[key] - end - end - end - - # @param objects [Enumerable<Object>] objects which _must_ respond to `#cache_key` - # @param context [Proc] a proc that can be called to help generate each cache key - # @return [Hash] - def multi_key_map(objects, context:) - objects.index_by do |object| - contextual_cache_key(object, context) - end - end end end end diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index 5c5109f3d21..e38213532ba 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -30,6 +30,10 @@ module API params :optional_params_ee do end + params :optional_update_params do + optional :prevent_sharing_groups_outside_hierarchy, type: Boolean, desc: 'Prevent sharing groups within this namespace with any groups outside the namespace. Only available on top-level groups.' + end + params :optional_update_params_ee do end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/integrations_helpers.rb index ca13ea0789a..06539772568 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -6,7 +6,7 @@ module API # # The data structures inside this model are returned using class methods, # allowing EE to extend them where necessary. - module ServicesHelpers + module IntegrationsHelpers def self.chat_notification_settings [ { @@ -159,7 +159,7 @@ module API ].freeze end - def self.services + def self.integrations { 'asana' => [ { @@ -772,7 +772,7 @@ module API } end - def self.service_classes + def self.integration_classes [ ::Integrations::Asana, ::Integrations::Assembla, @@ -799,24 +799,24 @@ module API ::Integrations::Packagist, ::Integrations::PipelinesEmail, ::Integrations::Pivotaltracker, + ::Integrations::Prometheus, ::Integrations::Pushover, ::Integrations::Redmine, ::Integrations::Slack, ::Integrations::SlackSlashCommands, ::Integrations::Teamcity, - ::Integrations::Youtrack, - ::PrometheusService + ::Integrations::Youtrack ] end - def self.development_service_classes + def self.development_integration_classes [ ::Integrations::MockCi, - ::MockMonitoringService + ::Integrations::MockMonitoring ] end end end end -API::Helpers::ServicesHelpers.prepend_mod_with('API::Helpers::ServicesHelpers') +API::Helpers::IntegrationsHelpers.prepend_mod_with('API::Helpers::IntegrationsHelpers') diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 69a83043617..272452bd8db 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -66,6 +66,7 @@ module API optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled' optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' optional :packages_enabled, type: Boolean, desc: 'Enable project packages feature' + optional :squash_option, type: String, values: %w(never always default_on default_off), desc: 'Squash default for project. One of `never`, `always`, `default_on`, or `default_off`.' end params :optional_project_params_ee do @@ -145,6 +146,7 @@ module API :request_access_enabled, :resolve_outdated_diff_discussions, :restrict_user_defined_variables, + :squash_option, :shared_runners_enabled, :snippets_access_level, :tag_list, diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 9ec9b5e1e35..a022d1a56ac 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -14,6 +14,10 @@ module API ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) end + def runner_registrar_valid?(type) + Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type) + end + def authenticate_runner! forbidden! unless current_runner diff --git a/lib/api/helpers/snippets_helpers.rb b/lib/api/helpers/snippets_helpers.rb index 42f56680ded..2d8c761101a 100644 --- a/lib/api/helpers/snippets_helpers.rb +++ b/lib/api/helpers/snippets_helpers.rb @@ -72,22 +72,18 @@ module API end def process_create_params(args) - with_api_params do |api_params| - args[:snippet_actions] = args.delete(:files)&.map do |file| - file[:action] = :create - file.symbolize_keys - end - - args.merge(api_params) + args[:snippet_actions] = args.delete(:files)&.map do |file| + file[:action] = :create + file.symbolize_keys end + + args end def process_update_params(args) - with_api_params do |api_params| - args[:snippet_actions] = args.delete(:files)&.map(&:symbolize_keys) + args[:snippet_actions] = args.delete(:files)&.map(&:symbolize_keys) - args.merge(api_params) - end + args end def validate_params_for_multiple_files(snippet) diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index ee0ddccc8d4..a06b052847d 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -124,11 +124,6 @@ module API yield end end - - # Overridden in EE - def geo_proxy - {} - end end namespace 'internal' do @@ -320,12 +315,6 @@ module API two_factor_otp_check end - - # Workhorse calls this to determine if it is a Geo secondary site - # that should proxy requests. FOSS can quickly return empty data. - get '/geo_proxy', feature_category: :geo_replication do - geo_proxy - end end end end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index c28e2181873..7af5c2ad2ee 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -53,8 +53,6 @@ module API def check_agent_token unauthorized! unless agent_token - forbidden! unless Gitlab::Kas.included_in_gitlab_com_rollout?(agent.project) - agent_token.track_usage end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 355b5ed3a1f..54013d0e7b4 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -255,9 +255,11 @@ module API issue_params = convert_parameters_from_legacy_format(issue_params) begin + spam_params = ::Spam::SpamParams.new_from_request(request: request) issue = ::Issues::CreateService.new(project: user_project, current_user: current_user, - params: issue_params.merge(request: request, api: true)).execute + params: issue_params, + spam_params: spam_params).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) @@ -294,13 +296,15 @@ module API issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue - update_params = declared_params(include_missing: false).merge(request: request, api: true) + update_params = declared_params(include_missing: false) update_params = convert_parameters_from_legacy_format(update_params) + spam_params = ::Spam::SpamParams.new_from_request(request: request) issue = ::Issues::UpdateService.new(project: user_project, current_user: current_user, - params: update_params).execute(issue) + params: update_params, + spam_params: spam_params).execute(issue) render_spam_error! if issue.spam? diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 37199279205..beda4433e4f 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -4,7 +4,7 @@ module API class JobArtifacts < ::API::Base before { authenticate_non_get! } - feature_category :continuous_integration + feature_category :build_artifacts # EE::API::JobArtifacts would override the following helpers helpers do diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 3580a7b5e24..945cdf3edb2 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -11,11 +11,7 @@ module API optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' end post '/lint' do - if Feature.enabled?(:security_ci_lint_authorization) - unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? - else - unauthorized! if Gitlab::CurrentSettings.signup_disabled? && current_user.nil? - end + unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 5bae08d4dae..03d1492908d 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -16,6 +16,7 @@ module API feature_category :package_registry PACKAGE_FILENAME = 'package.nupkg' + SYMBOL_PACKAGE_FILENAME = 'package.snupkg' default_format :json @@ -33,6 +34,10 @@ module API end helpers do + params :file_params do + requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + def project_or_group authorized_user_project end @@ -40,6 +45,49 @@ module API def snowplow_gitlab_standard_context { project: authorized_user_project, namespace: authorized_user_project.namespace } end + + def authorize_nuget_upload + authorize_workhorse!( + subject: project_or_group, + has_length: false, + maximum_size: project_or_group.actual_limits.nuget_max_file_size + ) + end + + def temp_file_name(symbol_package) + return ::Packages::Nuget::TEMPORARY_SYMBOL_PACKAGE_NAME if symbol_package + + ::Packages::Nuget::TEMPORARY_PACKAGE_NAME + end + + def file_name(symbol_package) + return SYMBOL_PACKAGE_FILENAME if symbol_package + + PACKAGE_FILENAME + end + + def upload_nuget_package_file(symbol_package: false) + authorize_upload!(project_or_group) + bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) + + file_params = params.merge( + file: params[:package], + file_name: file_name(symbol_package) + ) + + package = ::Packages::CreateTemporaryPackageService.new( + project_or_group, current_user, declared_params.merge(build: current_authenticated_job) + ).execute(:nuget, name: temp_file_name(symbol_package)) + + package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) + .execute + + yield(package) if block_given? + + ::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker + + created! + end end params do @@ -55,40 +103,54 @@ module API end params do - requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + use :file_params end put do - authorize_upload!(project_or_group) - bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) - - file_params = params.merge( - file: params[:package], - file_name: PACKAGE_FILENAME - ) - - package = ::Packages::CreateTemporaryPackageService.new( - project_or_group, current_user, declared_params.merge(build: current_authenticated_job) - ).execute(:nuget, name: ::Packages::Nuget::TEMPORARY_PACKAGE_NAME) - - package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) - .execute + upload_nuget_package_file do |package| + track_package_event( + 'push_package', + :nuget, + category: 'API::NugetPackages', + user: current_user, + project: package.project, + namespace: package.project.namespace + ) + end + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) - track_package_event('push_package', :nuget, category: 'API::NugetPackages', user: current_user, project: package.project, namespace: package.project.namespace) + forbidden! + end + put 'authorize' do + authorize_nuget_upload + end - ::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker + # https://docs.microsoft.com/en-us/nuget/api/symbol-package-publish-resource + desc 'The NuGet Symbol Package Publish endpoint' do + detail 'This feature was introduced in GitLab 14.1' + end - created! + params do + use :file_params + end + put 'symbolpackage' do + upload_nuget_package_file(symbol_package: true) do |package| + track_package_event( + 'push_symbol_package', + :nuget, + category: 'API::NugetPackages', + user: current_user, + project: package.project, + namespace: package.project.namespace + ) + end rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) forbidden! end - put 'authorize' do - authorize_workhorse!( - subject: project_or_group, - has_length: false, - maximum_size: project_or_group.actual_limits.nuget_max_file_size - ) + put 'symbolpackage/authorize' do + authorize_nuget_upload end # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource @@ -115,14 +177,20 @@ module API requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX end - get '*package_version/*package_filename', format: :nupkg do + get '*package_version/*package_filename', format: [:nupkg, :snupkg] do filename = "#{params[:package_filename]}.#{params[:format]}" package_file = ::Packages::PackageFileFinder.new(find_package(params[:package_name], params[:package_version]), filename, with_file_name_like: true) .execute not_found!('Package') unless package_file - track_package_event('pull_package', :nuget, category: 'API::NugetPackages', project: package_file.project, namespace: package_file.project.namespace) + track_package_event( + params[:format] == 'snupkg' ? 'pull_symbol_package' : 'pull_package', + :nuget, + category: 'API::NugetPackages', + project: package_file.project, + namespace: package_file.project.namespace + ) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false present_carrierwave_file!(package_file.file, supports_direct_download: false) diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 35a68ec4e18..54c0a0628a7 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -71,9 +71,7 @@ module API .new(user_project, params[:package_id]).execute destroy_conditionally!(package) do |package| - if package.destroy - package.sync_maven_metadata(current_user) - end + ::Packages::DestroyPackageService.new(container: package, current_user: current_user).execute end end end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 084492fd503..fdbfdf1f7a9 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -75,7 +75,8 @@ module API snippet_params = process_create_params(declared_params(include_missing: false)) - service_response = ::Snippets::CreateService.new(project: user_project, current_user: current_user, params: snippet_params).execute + spam_params = ::Spam::SpamParams.new_from_request(request: request) + service_response = ::Snippets::CreateService.new(project: user_project, current_user: current_user, params: snippet_params, spam_params: spam_params).execute snippet = service_response.payload[:snippet] if service_response.success? @@ -116,7 +117,8 @@ module API snippet_params = process_update_params(declared_params(include_missing: false)) - service_response = ::Snippets::UpdateService.new(project: user_project, current_user: current_user, params: snippet_params).execute(snippet) + spam_params = ::Spam::SpamParams.new_from_request(request: request) + service_response = ::Snippets::UpdateService.new(project: user_project, current_user: current_user, params: snippet_params, spam_params: spam_params).execute(snippet) snippet = service_response.payload[:snippet] if service_response.success? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 83c335a3248..3b1d239398f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -157,6 +157,8 @@ module API [options[:with].prepare_relation(projects, options), options] end + Preloaders::UserMaxAccessLevelInProjectsPreloader.new(records, current_user).execute if current_user + present records, options end @@ -608,6 +610,10 @@ module API users = users.search(params[:search]) if params[:search].present? users = users.where_not_in(params[:skip_users]) if params[:skip_users].present? + if Feature.enabled?(:sort_by_project_users_by_project_authorizations_user_id, user_project, default_enabled: :yaml) + users = users.order('project_authorizations.user_id' => :asc) # rubocop: disable CodeReuse/ActiveRecord + end + present paginate(users), with: Entities::UserBasic end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 7cd8b442706..3b7e2b4bd27 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -62,6 +62,8 @@ module API get ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do authorize_download_code! + not_found! unless release + present release, with: Entities::Release, current_user: current_user, include_html_description: params[:include_html_description] end @@ -177,7 +179,7 @@ module API end def authorize_download_code! - authorize! :download_code, release + authorize! :download_code, user_project end def authorize_create_evidence! diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index a5234828de3..f274406e225 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -120,24 +120,28 @@ module API optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare' do - if params[:from_project_id].present? - target_project = MergeRequestTargetProjectFinder - .new(current_user: current_user, source_project: user_project, project_feature: :repository) - .execute(include_routes: true).find_by_id(params[:from_project_id]) - - if target_project.blank? - render_api_error!("Target project id:#{params[:from_project_id]} is not a fork of project id:#{params[:id]}", 400) + ff_enabled = Feature.enabled?(:api_caching_rate_limit_repository_compare, user_project, default_enabled: :yaml) + + cache_action_if(ff_enabled, [user_project, :repository_compare, current_user, declared_params], expires_in: 1.minute) do + if params[:from_project_id].present? + target_project = MergeRequestTargetProjectFinder + .new(current_user: current_user, source_project: user_project, project_feature: :repository) + .execute(include_routes: true).find_by_id(params[:from_project_id]) + + if target_project.blank? + render_api_error!("Target project id:#{params[:from_project_id]} is not a fork of project id:#{params[:id]}", 400) + end + else + target_project = user_project end - else - target_project = user_project - end - compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight]) + compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight]) - if compare - present compare, with: Entities::Compare - else - not_found!("Ref") + if compare + present compare, with: Entities::Compare + else + not_found!("Ref") + end end end diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index 705e4778c83..f42acc6b2eb 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -21,9 +21,10 @@ module API next unauthorized! unless current_user.can?(:read_resource_access_tokens, resource) - tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute + tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute.preload_users - present paginate(tokens), with: Entities::PersonalAccessToken + resource.project_members.load + present paginate(tokens), with: Entities::ResourceAccessToken, project: resource end desc 'Revoke a resource access token' do @@ -57,6 +58,7 @@ module API requires :id, type: String, desc: "The #{source_type} ID" requires :name, type: String, desc: "Resource access token name" requires :scopes, type: Array[String], desc: "The permissions of the token" + optional :access_level, type: Integer, desc: "The access level of the token in the project" optional :expires_at, type: Date, desc: "The expiration date of the token" end post ':id/access_tokens' do @@ -69,7 +71,7 @@ module API ).execute if token_response.success? - present token_response.payload[:access_token], with: Entities::PersonalAccessTokenWithToken + present token_response.payload[:access_token], with: Entities::ResourceAccessTokenWithToken, project: resource else bad_request!(token_response.message) end diff --git a/lib/api/services.rb b/lib/api/services.rb index 8a7abe721dd..a37b6f4626a 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -3,11 +3,11 @@ module API class Services < ::API::Base feature_category :integrations - services = Helpers::ServicesHelpers.services - service_classes = Helpers::ServicesHelpers.service_classes + integrations = Helpers::IntegrationsHelpers.integrations + integration_classes = Helpers::IntegrationsHelpers.integration_classes if Rails.env.development? - services['mock-ci'] = [ + integrations['mock-ci'] = [ { required: true, name: :mock_service_url, @@ -15,28 +15,27 @@ module API desc: 'URL to the mock service' } ] - services['mock-deployment'] = [] - services['mock-monitoring'] = [] + integrations['mock-deployment'] = [] + integrations['mock-monitoring'] = [] - service_classes += Helpers::ServicesHelpers.development_service_classes + integration_classes += Helpers::IntegrationsHelpers.development_integration_classes end - SERVICES = services.freeze - SERVICE_CLASSES = service_classes.freeze + INTEGRATIONS = integrations.freeze - SERVICE_CLASSES.each do |service| - event_names = service.try(:event_names) || next + integration_classes.each do |integration| + event_names = integration.try(:event_names) || next event_names.each do |event_name| - SERVICES[service.to_param.tr("_", "-")] << { + INTEGRATIONS[integration.to_param.tr("_", "-")] << { required: false, name: event_name.to_sym, type: String, - desc: service.event_description(event_name) + desc: IntegrationsHelper.integration_event_description(integration, event_name) } end end - TRIGGER_SERVICES = { + TRIGGER_INTEGRATIONS = { 'mattermost-slash-commands' => [ { name: :token, @@ -61,24 +60,24 @@ module API before { authorize_admin_project } helpers do - def service_attributes(service) - service.fields.inject([]) do |arr, hash| + def integration_attributes(integration) + integration.fields.inject([]) do |arr, hash| arr << hash[:name].to_sym end end end - desc 'Get all active project services' do - success Entities::ProjectServiceBasic + desc 'Get all active project integrations' do + success Entities::ProjectIntegrationBasic end get ":id/services" do - services = user_project.integrations.active + integrations = user_project.integrations.active - present services, with: Entities::ProjectServiceBasic + present integrations, with: Entities::ProjectIntegrationBasic end - SERVICES.each do |service_slug, settings| - desc "Set #{service_slug} service for project" + INTEGRATIONS.each do |slug, settings| + desc "Set #{slug} integration for project" params do settings.each do |setting| if setting[:required] @@ -88,56 +87,52 @@ module API end end end - put ":id/services/#{service_slug}" do - service = user_project.find_or_initialize_service(service_slug.underscore) - service_params = declared_params(include_missing: false).merge(active: true) + put ":id/services/#{slug}" do + integration = user_project.find_or_initialize_integration(slug.underscore) + params = declared_params(include_missing: false).merge(active: true) - if service.update(service_params) - present service, with: Entities::ProjectService + if integration.update(params) + present integration, with: Entities::ProjectIntegration else render_api_error!('400 Bad Request', 400) end end end - desc "Delete a service for project" + desc "Delete an integration from a project" params do - requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' + requires :slug, type: String, values: INTEGRATIONS.keys, desc: 'The name of the service' end - delete ":id/services/:service_slug" do - service = user_project.find_or_initialize_service(params[:service_slug].underscore) + delete ":id/services/:slug" do + integration = user_project.find_or_initialize_integration(params[:slug].underscore) - destroy_conditionally!(service) do - attrs = service_attributes(service).inject({}) do |hash, key| - hash.merge!(key => nil) - end + destroy_conditionally!(integration) do + attrs = integration_attributes(integration).index_with { nil }.merge(active: false) - unless service.update(attrs.merge(active: false)) - render_api_error!('400 Bad Request', 400) - end + render_api_error!('400 Bad Request', 400) unless integration.update(attrs) end end - desc 'Get the service settings for project' do - success Entities::ProjectService + desc 'Get the integration settings for a project' do + success Entities::ProjectIntegration end params do - requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' + requires :slug, type: String, values: INTEGRATIONS.keys, desc: 'The name of the service' end - get ":id/services/:service_slug" do - integration = user_project.find_or_initialize_service(params[:service_slug].underscore) + get ":id/services/:slug" do + integration = user_project.find_or_initialize_integration(params[:slug].underscore) not_found!('Service') unless integration&.persisted? - present integration, with: Entities::ProjectService + present integration, with: Entities::ProjectIntegration end end - TRIGGER_SERVICES.each do |service_slug, settings| + TRIGGER_INTEGRATIONS.each do |integration_slug, settings| helpers do - def slash_command_service(project, service_slug, params) - project.integrations.active.find do |service| - service.try(:token) == params[:token] && service.to_param == service_slug.underscore + def slash_command_integration(project, integration_slug, params) + project.integrations.active.find do |integration| + integration.try(:token) == params[:token] && integration.to_param == integration_slug.underscore end end end @@ -146,7 +141,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc "Trigger a slash command for #{service_slug}" do + desc "Trigger a slash command for #{integration_slug}" do detail 'Added in GitLab 8.13' end params do @@ -154,14 +149,14 @@ module API requires setting[:name], type: setting[:type], desc: setting[:desc] end end - post ":id/services/#{service_slug.underscore}/trigger" do + post ":id/services/#{integration_slug.underscore}/trigger" do project = find_project(params[:id]) # This is not accurate, but done to prevent leakage of the project names not_found!('Service') unless project - service = slash_command_service(project, service_slug, params) - result = service.try(:trigger, params) + integration = slash_command_integration(project, integration_slug, params) + result = integration.try(:trigger, params) if result status result[:status] || 200 diff --git a/lib/api/settings.rb b/lib/api/settings.rb index b4f8320cb74..952bf09b1b1 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -160,6 +160,10 @@ module API optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' optional :local_markdown_version, type: Integer, desc: 'Local markdown version, increase this value when any cached markdown should be invalidated' optional :allow_local_requests_from_hooks_and_services, type: Boolean, desc: 'Deprecated: Use :allow_local_requests_from_web_hooks_and_services instead. Allow requests to the local network from hooks and services.' # support legacy names, can be removed in v5 + optional :mailgun_events_enabled, type: Grape::API::Boolean, desc: 'Enable Mailgun event receiver' + given mailgun_events_enabled: ->(val) { val } do + requires :mailgun_signing_key, type: String, desc: 'The Mailgun HTTP webhook signing key for receiving events from webhook' + end optional :snowplow_enabled, type: Grape::API::Boolean, desc: 'Enable Snowplow tracking' given snowplow_enabled: ->(val) { val } do requires :snowplow_collector_hostname, type: String, desc: 'The Snowplow collector hostname' diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index b506192fe1c..f1ec1024492 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -84,7 +84,8 @@ module API attrs = process_create_params(declared_params(include_missing: false)) - service_response = ::Snippets::CreateService.new(project: nil, current_user: current_user, params: attrs).execute + spam_params = ::Spam::SpamParams.new_from_request(request: request) + service_response = ::Snippets::CreateService.new(project: nil, current_user: current_user, params: attrs, spam_params: spam_params).execute snippet = service_response.payload[:snippet] if service_response.success? @@ -126,7 +127,8 @@ module API attrs = process_update_params(declared_params(include_missing: false)) - service_response = ::Snippets::UpdateService.new(project: nil, current_user: current_user, params: attrs).execute(snippet) + spam_params = ::Spam::SpamParams.new_from_request(request: request) + service_response = ::Snippets::UpdateService.new(project: nil, current_user: current_user, params: attrs, spam_params: spam_params).execute(snippet) snippet = service_response.payload[:snippet] diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index fe23a111b7f..e4133713c1f 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -31,6 +31,7 @@ module API optional :push_events, type: Boolean, desc: "Trigger hook on push events" optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" optional :merge_requests_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :repository_update_events, type: Boolean, desc: "Trigger hook on repository update events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" end post do diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 7deec15dcac..43c75206b88 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -4,7 +4,7 @@ module API class UsageData < ::API::Base before { authenticate_non_get! } - feature_category :usage_ping + feature_category :service_ping namespace 'usage_data' do before do diff --git a/lib/api/usage_data_non_sql_metrics.rb b/lib/api/usage_data_non_sql_metrics.rb index 63a14a223f5..d9e0d153e58 100644 --- a/lib/api/usage_data_non_sql_metrics.rb +++ b/lib/api/usage_data_non_sql_metrics.rb @@ -4,7 +4,7 @@ module API class UsageDataNonSqlMetrics < ::API::Base before { authenticated_as_admin! } - feature_category :usage_ping + feature_category :service_ping namespace 'usage_data' do before do diff --git a/lib/api/usage_data_queries.rb b/lib/api/usage_data_queries.rb index 0ad9ad7650c..22e83fe0294 100644 --- a/lib/api/usage_data_queries.rb +++ b/lib/api/usage_data_queries.rb @@ -4,7 +4,7 @@ module API class UsageDataQueries < ::API::Base before { authenticated_as_admin! } - feature_category :usage_ping + feature_category :service_ping namespace 'usage_data' do before do diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 8b0745c6b5b..75df0e050a6 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,7 +7,7 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } - feature_category :continuous_integration + feature_category :pipeline_authoring helpers Helpers::VariablesHelpers diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 0429d9496d6..f07fd786b4b 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -9,6 +9,8 @@ module Backup attr_reader :config, :db_file_name IGNORED_ERRORS = [ + # Ignore warnings + /WARNING:/, # Ignore the DROP errors; recent database dumps will use --if-exists with pg_dump /does not exist$/, # User may not have permissions to drop extensions or schemas @@ -18,7 +20,7 @@ module Backup def initialize(progress, filename: nil) @progress = progress - @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] + @config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash @db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end @@ -30,9 +32,9 @@ module Backup compress_rd.close dump_pid = - case config["adapter"] + case config[:adapter] when "postgresql" then - progress.print "Dumping PostgreSQL database #{config['database']} ... " + progress.print "Dumping PostgreSQL database #{database} ... " pg_env pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. pgsql_args << '--if-exists' @@ -47,7 +49,7 @@ module Backup end end - Process.spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) + Process.spawn('pg_dump', *pgsql_args, database, out: compress_wr) end compress_wr.close @@ -68,9 +70,9 @@ module Backup decompress_wr.close status, errors = - case config["adapter"] + case config[:adapter] when "postgresql" then - progress.print "Restoring PostgreSQL database #{config['database']} ... " + progress.print "Restoring PostgreSQL database #{database} ... " pg_env execute_and_track_errors(pg_restore_cmd, decompress_rd) end @@ -93,6 +95,10 @@ module Backup protected + def database + @config[:database] + end + def ignore_error?(line) IGNORED_ERRORS_REGEXP.match?(line) end @@ -128,17 +134,17 @@ module Backup def pg_env args = { - 'username' => 'PGUSER', - 'host' => 'PGHOST', - 'port' => 'PGPORT', - 'password' => 'PGPASSWORD', + username: 'PGUSER', + host: 'PGHOST', + port: 'PGPORT', + password: 'PGPASSWORD', # SSL - 'sslmode' => 'PGSSLMODE', - 'sslkey' => 'PGSSLKEY', - 'sslcert' => 'PGSSLCERT', - 'sslrootcert' => 'PGSSLROOTCERT', - 'sslcrl' => 'PGSSLCRL', - 'sslcompression' => 'PGSSLCOMPRESSION' + sslmode: 'PGSSLMODE', + sslkey: 'PGSSLKEY', + sslcert: 'PGSSLCERT', + sslrootcert: 'PGSSLROOTCERT', + sslcrl: 'PGSSLCRL', + sslcompression: 'PGSSLCOMPRESSION' } args.each do |opt, arg| # This enables the use of different PostgreSQL settings in @@ -161,7 +167,7 @@ module Backup private def pg_restore_cmd - ['psql', config['database']] + ['psql', database] end end end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index cfd3d463f9e..c15b0ed6a1b 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -3,8 +3,10 @@ module Backup # Backup and restores repositories using gitaly-backup class GitalyBackup - def initialize(progress) + def initialize(progress, parallel: nil, parallel_storage: nil) @progress = progress + @parallel = parallel + @parallel_storage = parallel_storage end def start(type) @@ -19,8 +21,12 @@ module Backup raise Error, "unknown backup type: #{type}" end + args = [] + args += ['-parallel', @parallel.to_s] if type == :create && @parallel + args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage + @read_io, @write_io = IO.pipe - @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, in: @read_io, out: progress) + @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, *args, in: @read_io, out: @progress) end def wait @@ -48,9 +54,11 @@ module Backup }.merge(Gitlab::GitalyClient.connection_data(repository.storage)).to_json) end - private + def parallel_enqueue? + false + end - attr_reader :progress + private def started? @pid.present? @@ -61,7 +69,7 @@ module Backup end def bin_path - File.absolute_path(File.join(Gitlab.config.gitaly.client_path, 'gitaly-backup')) + File.absolute_path(Gitlab.config.backup.gitaly_backup_path) end end end diff --git a/lib/backup/gitaly_rpc_backup.rb b/lib/backup/gitaly_rpc_backup.rb index 53f1de40509..baac4eb26ca 100644 --- a/lib/backup/gitaly_rpc_backup.rb +++ b/lib/backup/gitaly_rpc_backup.rb @@ -44,6 +44,10 @@ module Backup end end + def parallel_enqueue? + true + end + private attr_reader :progress diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 80d23c1eb7f..0b5a62529b4 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -12,7 +12,10 @@ module Backup def dump(max_concurrency:, max_storage_concurrency:) strategy.start(:create) - if max_concurrency <= 1 && max_storage_concurrency <= 1 + # gitaly-backup is designed to handle concurrency on its own. So we want + # to avoid entering the buggy concurrency code here when gitaly-backup + # is enabled. + if (max_concurrency <= 1 && max_storage_concurrency <= 1) || !strategy.parallel_enqueue? return enqueue_consecutive end diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb index 12afece6e53..3ae9c5f8d90 100644 --- a/lib/banzai/filter/references/label_reference_filter.rb +++ b/lib/banzai/filter/references/label_reference_filter.rb @@ -11,13 +11,21 @@ module Banzai def parent_records(parent, ids) return Label.none unless parent.is_a?(Project) || parent.is_a?(Group) - labels = find_labels(parent) - label_ids = ids.map {|y| y[:label_id]}.compact - label_names = ids.map {|y| y[:label_name]}.compact - id_relation = labels.where(id: label_ids) - label_relation = labels.where(title: label_names) + labels = find_labels(parent) + label_ids = ids.map {|y| y[:label_id]}.compact - Label.from_union([id_relation, label_relation]) + unless label_ids.empty? + id_relation = labels.where(id: label_ids) + end + + label_names = ids.map {|y| y[:label_name]}.compact + unless label_names.empty? + label_relation = labels.where(title: label_names) + end + + return Label.none if (relation = [id_relation, label_relation].compact).empty? + + Label.from_union(relation) end def find_object(parent_object, id) diff --git a/lib/banzai/filter/references/milestone_reference_filter.rb b/lib/banzai/filter/references/milestone_reference_filter.rb index 31a961f3e73..d992e667056 100644 --- a/lib/banzai/filter/references/milestone_reference_filter.rb +++ b/lib/banzai/filter/references/milestone_reference_filter.rb @@ -10,19 +10,55 @@ module Banzai self.reference_type = :milestone self.object_class = Milestone - # Links to project milestones contain the IID, but when we're handling - # 'regular' references, we need to use the global ID to disambiguate - # between group and project milestones. - def find_object(parent, id) - return unless valid_context?(parent) + def parent_records(parent, ids) + return Milestone.none unless valid_context?(parent) - find_milestone_with_finder(parent, id: id) + milestone_iids = ids.map {|y| y[:milestone_iid]}.compact + unless milestone_iids.empty? + iid_relation = find_milestones(parent, true).where(iid: milestone_iids) + end + + milestone_names = ids.map {|y| y[:milestone_name]}.compact + unless milestone_names.empty? + milestone_relation = find_milestones(parent, false).where(name: milestone_names) + end + + return Milestone.none if (relation = [iid_relation, milestone_relation].compact).empty? + + Milestone.from_union(relation).includes(:project, :group) + end + + def find_object(parent_object, id) + key = reference_cache.records_per_parent[parent_object].keys.find do |k| + k[:milestone_iid] == id[:milestone_iid] || k[:milestone_name] == id[:milestone_name] + end + + reference_cache.records_per_parent[parent_object][key] if key end - def find_object_from_link(parent, iid) - return unless valid_context?(parent) + # Transform a symbol extracted from the text to a meaningful value + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `parse_symbol(ref) == record_identifier(record)`. + # + # This contract is slightly broken here, as we only have either the milestone_iid + # or the milestone_name, but not both. But below, we have both pieces of information. + # But it's accounted for in `find_object` + def parse_symbol(symbol, match_data) + if symbol + # when parsing links, there is no `match_data[:milestone_iid]`, but `symbol` + # holds the iid + { milestone_iid: symbol.to_i, milestone_name: nil } + else + { milestone_iid: match_data[:milestone_iid]&.to_i, milestone_name: match_data[:milestone_name]&.tr('"', '') } + end + end - find_milestone_with_finder(parent, iid: iid) + # This method has the contract that if a string `ref` refers to a + # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`. + # See note in `parse_symbol` above + def record_identifier(record) + { milestone_iid: record.iid, milestone_name: record.name } end def valid_context?(parent) @@ -50,12 +86,14 @@ module Banzai return super(text, pattern) if pattern != Milestone.reference_pattern milestones = {} - unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| - milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) - if milestone - milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~ - "#{REFERENCE_PLACEHOLDER}#{milestone.id}" + unescaped_html = unescape_html_entities(text).gsub(pattern).with_index do |match, index| + ident = identifier($~) + milestone = yield match, ident, $~[:project], $~[:namespace], $~ + + if milestone != match + milestones[index] = milestone + "#{REFERENCE_PLACEHOLDER}#{index}" else match end @@ -66,31 +104,10 @@ module Banzai escape_with_placeholders(unescaped_html, milestones) end - def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) - project_path = reference_cache.full_project_path(namespace_ref, project_ref) - - # Returns group if project is not found by path - parent = parent_from_ref(project_path) + def find_milestones(parent, find_by_iid = false) + finder_params = milestone_finder_params(parent, find_by_iid) - return unless parent - - milestone_params = milestone_params(milestone_id, milestone_name) - - find_milestone_with_finder(parent, milestone_params) - end - - def milestone_params(iid, name) - if name - { name: name.tr('"', '') } - else - { iid: iid.to_i } - end - end - - def find_milestone_with_finder(parent, params) - finder_params = milestone_finder_params(parent, params[:iid].present?) - - MilestonesFinder.new(finder_params).find_by(params) + MilestonesFinder.new(finder_params).execute end def milestone_finder_params(parent, find_by_iid) @@ -131,6 +148,14 @@ module Banzai def object_link_title(object, matches) nil end + + def parent + project || group + end + + def requires_unescaping? + true + end end end end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 2572481c8fc..b110c59a4f1 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -45,13 +45,15 @@ module Banzai return end - html_attr.value = + path = if context[:only_path] path else Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s end + replace_html_attr_value(html_attr, path) + if html_attr.name == 'href' html_attr.parent.set_attribute('data-link', 'true') end @@ -59,6 +61,21 @@ module Banzai html_attr.parent.add_class('gfm') end + def replace_html_attr_value(html_attr, path) + if path != html_attr.value + preserve_original_link(html_attr, html_attr.parent) + end + + html_attr.value = path + end + + def preserve_original_link(html_attr, node) + return if html_attr.blank? + return if node.value?('data-canonical-src') + + node.set_attribute('data-canonical-src', html_attr.value) + end + def group context[:group] end diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 2b95d87ff8e..0736181d940 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -36,7 +36,7 @@ module Banzai protected def process_link(link_attr, node) - process_link_attr(link_attr) + process_link_attr(link_attr, node) remove_unsafe_links({ node: node }, remove_invalid_links: false) end @@ -44,14 +44,27 @@ module Banzai !context[:wiki].nil? end - def process_link_attr(html_attr) + def process_link_attr(html_attr, node) return if html_attr.blank? - html_attr.value = apply_rewrite_rules(html_attr.value) + rewritten_value = apply_rewrite_rules(html_attr.value) + + if html_attr.value != rewritten_value + preserve_original_link(html_attr, node) + end + + html_attr.value = rewritten_value rescue URI::Error, Addressable::URI::InvalidURIError # noop end + def preserve_original_link(html_attr, node) + return if html_attr.blank? + return if node.value?('data-canonical-src') + + node.set_attribute('data-canonical-src', html_attr.value) + end + def apply_rewrite_rules(link_string) Rewriter.new(link_string, wiki: context[:wiki], slug: context[:page_slug]).apply_rules end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index 3fc3ae02088..af0dcad107e 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -11,11 +11,11 @@ module Banzai @texts_and_contexts << { text: text, context: context } end - def references(type, project, current_user = nil) + def references(type, project, current_user, ids_only: false) context = RenderContext.new(project, current_user) processor = Banzai::ReferenceParser[type].new(context) - processor.process(html_documents) + processor.process(html_documents, ids_only: ids_only) end def reset_memoized_values diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 3dfea8ee895..0c015ba00c7 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -76,9 +76,11 @@ module Banzai end # Returns an Array of objects referenced by any of the given HTML nodes. - def referenced_by(nodes) + def referenced_by(nodes, options = {}) ids = unique_attribute_values(nodes, self.class.data_attribute) + return ids if options.fetch(:ids_only, false) + if ids.empty? references_relation.none else @@ -194,7 +196,7 @@ module Banzai # Processes the list of HTML documents and returns an Array containing all # the references. - def process(documents) + def process(documents, ids_only: false) type = self.class.reference_type reference_options = self.class.reference_options @@ -202,17 +204,17 @@ module Banzai Querying.css(document, "a[data-reference-type='#{type}'].gfm", reference_options).to_a end - gather_references(nodes) + gather_references(nodes, ids_only: ids_only) end # Gathers the references for the given HTML nodes. Returns visible # references and a list of nodes which are not visible to the user - def gather_references(nodes) + def gather_references(nodes, ids_only: false) nodes = nodes_user_can_reference(current_user, nodes) visible = nodes_visible_to_user(current_user, nodes) not_visible = nodes - visible - { visible: referenced_by(visible), not_visible: not_visible } + { visible: referenced_by(visible, ids_only: ids_only), not_visible: not_visible } end # Returns a Hash containing the projects for a given list of HTML nodes. diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 0bfb6a92020..88896970bc6 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -5,7 +5,7 @@ module Banzai class CommitParser < BaseParser self.reference_type = :commit - def referenced_by(nodes) + def referenced_by(nodes, options = {}) commit_ids = commit_ids_per_project(nodes) projects = find_projects_for_hash_keys(commit_ids) diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index 480eefd5c4d..fb4a392105f 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -5,7 +5,7 @@ module Banzai class CommitRangeParser < BaseParser self.reference_type = :commit_range - def referenced_by(nodes) + def referenced_by(nodes, options = {}) range_ids = commit_range_ids_per_project(nodes) projects = find_projects_for_hash_keys(range_ids) diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb index 029b09dcd25..e8ee337064a 100644 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -5,7 +5,7 @@ module Banzai class ExternalIssueParser < BaseParser self.reference_type = :external_issue - def referenced_by(nodes) + def referenced_by(nodes, options = {}) issue_ids = issue_ids_per_project(nodes) projects = find_projects_for_hash_keys(issue_ids) issues = [] diff --git a/lib/banzai/reference_parser/issuable_parser.rb b/lib/banzai/reference_parser/issuable_parser.rb index f8c26288017..efcaa6664b6 100644 --- a/lib/banzai/reference_parser/issuable_parser.rb +++ b/lib/banzai/reference_parser/issuable_parser.rb @@ -13,7 +13,7 @@ module Banzai end end - def referenced_by(nodes) + def referenced_by(nodes, options = {}) records = records_for_nodes(nodes) nodes.map { |node| records[node] }.compact.uniq diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 36c41c6615f..c40ca9dc7cd 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -5,7 +5,7 @@ module Banzai class UserParser < BaseParser self.reference_type = :user - def referenced_by(nodes) + def referenced_by(nodes, options = {}) group_ids = [] user_ids = [] project_ids = [] diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb index ca549c4be14..0adc2b1c57f 100644 --- a/lib/bulk_imports/clients/graphql.rb +++ b/lib/bulk_imports/clients/graphql.rb @@ -23,15 +23,19 @@ module BulkImports attr_reader :client - delegate :query, :parse, :execute, to: :client + delegate :query, :parse, to: :client def initialize(url: Gitlab::Saas.com_url, token: nil) @url = Gitlab::Utils.append_path(url, '/api/graphql') @token = token - @client = Graphlient::Client.new( - @url, - options(http: HTTP) - ) + @client = Graphlient::Client.new(@url, options(http: HTTP)) + @compatible_instance_version = false + end + + def execute(*args) + validate_instance_version! + + client.execute(*args) end def options(extra = {}) @@ -44,6 +48,19 @@ module BulkImports } }.merge(extra) end + + def validate_instance_version! + return if @compatible_instance_version + + response = client.execute('{ metadata { version } }') + version = Gitlab::VersionInfo.parse(response.data.metadata.version) + + if version.major < BulkImport::MINIMUM_GITLAB_MAJOR_VERSION + raise ::BulkImports::Error.unsupported_gitlab_version + else + @compatible_instance_version = true + end + end end end end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index c5f12d8c2ba..6c363a3552f 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -7,14 +7,13 @@ module BulkImports DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 30 - ConnectionError = Class.new(StandardError) - - def initialize(uri:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION) - @uri = URI.parse(uri) + def initialize(url:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION) + @url = url @token = token&.strip @page = page @per_page = per_page @api_version = api_version + @compatible_instance_version = false end def get(resource, query = {}) @@ -53,10 +52,28 @@ module BulkImports Gitlab::Utils.append_path(api_url, resource) end + def validate_instance_version! + return if @compatible_instance_version + + response = with_error_handling do + Gitlab::HTTP.get(resource_url(:version), default_options) + end + + version = Gitlab::VersionInfo.parse(response.parsed_response['version']) + + if version.major < BulkImport::MINIMUM_GITLAB_MAJOR_VERSION + raise ::BulkImports::Error.unsupported_gitlab_version + else + @compatible_instance_version = true + end + end + private # rubocop:disable GitlabSecurity/PublicSend def request(method, resource, options = {}, &block) + validate_instance_version! + with_error_handling do Gitlab::HTTP.public_send( method, @@ -96,19 +113,15 @@ module BulkImports def with_error_handling response = yield - raise ConnectionError, "Error #{response.code}" unless response.success? + raise(::BulkImports::Error, "Error #{response.code}") unless response.success? response rescue *Gitlab::HTTP::HTTP_ERRORS => e - raise ConnectionError, e - end - - def base_uri - @base_uri ||= "#{@uri.scheme}://#{@uri.host}:#{@uri.port}" + raise(::BulkImports::Error, e) end def api_url - Gitlab::Utils.append_path(base_uri, "/api/#{@api_version}") + Gitlab::Utils.append_path(@url, "/api/#{@api_version}") end end end diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb index 79d626001a0..788d10ca364 100644 --- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -7,6 +7,8 @@ module BulkImports include Gitlab::ImportExport::CommandLineUtil include Gitlab::Utils::StrongMemoize + FILE_SIZE_LIMIT = 5.gigabytes + ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze EXPORT_DOWNLOAD_URL_PATH = "/%{resource}/%{full_path}/export_relations/download?relation=%{relation}" def initialize(relation:) @@ -39,7 +41,9 @@ module BulkImports configuration: context.configuration, relative_url: relative_resource_url(context), dir: tmp_dir, - filename: filename + filename: filename, + file_size_limit: FILE_SIZE_LIMIT, + allowed_content_types: ALLOWED_CONTENT_TYPES ) end diff --git a/lib/bulk_imports/common/extractors/rest_extractor.rb b/lib/bulk_imports/common/extractors/rest_extractor.rb index 2179e0575c5..88315305222 100644 --- a/lib/bulk_imports/common/extractors/rest_extractor.rb +++ b/lib/bulk_imports/common/extractors/rest_extractor.rb @@ -25,7 +25,7 @@ module BulkImports def http_client(configuration) @http_client ||= BulkImports::Clients::HTTP.new( - uri: configuration.url, + url: configuration.url, token: configuration.access_token, per_page: 100 ) diff --git a/lib/bulk_imports/error.rb b/lib/bulk_imports/error.rb new file mode 100644 index 00000000000..0464aea642e --- /dev/null +++ b/lib/bulk_imports/error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module BulkImports + class Error < StandardError + def self.unsupported_gitlab_version + self.new("Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MINIMUM_GITLAB_MAJOR_VERSION}.") + end + end +end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index db5882d49a9..1140beef48c 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -18,7 +18,7 @@ module BulkImports def http_client(configuration) @http_client ||= BulkImports::Clients::HTTP.new( - uri: configuration.url, + url: configuration.url, token: configuration.access_token, per_page: 100 ) diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/groups/graphql/get_members_query.rb index e44d3c5aa9b..e76c87cc521 100644 --- a/lib/bulk_imports/groups/graphql/get_members_query.rb +++ b/lib/bulk_imports/groups/graphql/get_members_query.rb @@ -22,6 +22,7 @@ module BulkImports integer_value: integerValue } user { + user_gid: id public_email: publicEmail } } diff --git a/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb new file mode 100644 index 00000000000..6de8bbbc910 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/group_avatar_pipeline.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class GroupAvatarPipeline + include Pipeline + + ALLOWED_AVATAR_DOWNLOAD_TYPES = (AvatarUploader::MIME_WHITELIST + %w(application/octet-stream)).freeze + + GroupAvatarLoadingError = Class.new(StandardError) + + def extract(context) + context.extra[:tmpdir] = Dir.mktmpdir + + filepath = BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: "/groups/#{context.entity.encoded_source_full_path}/avatar", + dir: context.extra[:tmpdir], + file_size_limit: Avatarable::MAXIMUM_FILE_SIZE, + allowed_content_types: ALLOWED_AVATAR_DOWNLOAD_TYPES + ).execute + + BulkImports::Pipeline::ExtractedData.new(data: { filepath: filepath }) + end + + def load(context, data) + return if data.blank? + + File.open(data[:filepath]) do |avatar| + service = ::Groups::UpdateService.new( + portable, + current_user, + avatar: avatar + ) + + unless service.execute + raise GroupAvatarLoadingError, portable.errors.full_messages.first + end + end + end + + def after_run(_) + FileUtils.remove_entry(context.extra[:tmpdir]) if context.extra[:tmpdir].present? + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb index 5e4293d2c06..265abd5e3a7 100644 --- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb @@ -15,6 +15,9 @@ module BulkImports def load(context, data) return unless data + # Current user is already a member + return if data['user_id'].to_i == context.current_user.id + context.group.members.create!(data) end end diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb index e92c898171a..b9de375d0e9 100644 --- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb @@ -6,18 +6,20 @@ module BulkImports class MemberAttributesTransformer def transform(context, data) data - .then { |data| add_user(data) } + .then { |data| add_user(data, context) } .then { |data| add_access_level(data) } .then { |data| add_author(data, context) } end private - def add_user(data) + def add_user(data, context) user = find_user(data&.dig('user', 'public_email')) return unless user + cache_source_user_id(data, user, context) + data .except('user') .merge('user_id' => user.id) @@ -48,6 +50,16 @@ module BulkImports data.merge('created_by_id' => context.current_user.id) end + + def cache_source_user_id(data, user, context) + gid = data&.dig('user', 'user_gid') + + return unless gid + + source_user_id = GlobalID.parse(gid).model_id + + ::BulkImports::UsersMapper.new(context: context).cache_source_user_id(source_user_id, user.id) + end end end end diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index 2de06bbcb88..93fd6986173 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -88,11 +88,7 @@ module BulkImports end def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new( - exported_members: [], - user: current_user, - importable: portable - ) + @members_mapper ||= BulkImports::UsersMapper.new(context: context) end end end diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index bc7fc14b5a0..b1bceecbaea 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -9,6 +9,10 @@ module BulkImports pipeline: BulkImports::Groups::Pipelines::GroupPipeline, stage: 0 }, + avatar: { + pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, + stage: 1 + }, subgroups: { pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, stage: 1 diff --git a/lib/bulk_imports/users_mapper.rb b/lib/bulk_imports/users_mapper.rb new file mode 100644 index 00000000000..74412bc3831 --- /dev/null +++ b/lib/bulk_imports/users_mapper.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module BulkImports + class UsersMapper + include Gitlab::Utils::StrongMemoize + + SOURCE_USER_IDS_CACHE_KEY = 'bulk_imports/%{bulk_import}/%{entity}/source_user_ids' + + def initialize(context:) + @context = context + @cache_key = SOURCE_USER_IDS_CACHE_KEY % { + bulk_import: @context.bulk_import.id, + entity: @context.entity.id + } + end + + def map + strong_memoize(:map) do + map = hash_with_default + + cached_source_user_ids.each_pair do |source_id, destination_id| + map[source_id.to_i] = destination_id.to_i + end + + map + end + end + + def include?(source_user_id) + map.has_key?(source_user_id) + end + + def default_user_id + @context.current_user.id + end + + def cache_source_user_id(source_id, destination_id) + ::Gitlab::Cache::Import::Caching.hash_add(@cache_key, source_id, destination_id) + end + + private + + def hash_with_default + Hash.new { default_user_id } + end + + def cached_source_user_ids + ::Gitlab::Cache::Import::Caching.values_from_hash(@cache_key) + end + end +end diff --git a/lib/error_tracking/collector/sentry_request_parser.rb b/lib/error_tracking/collector/sentry_request_parser.rb new file mode 100644 index 00000000000..29e4cc8976f --- /dev/null +++ b/lib/error_tracking/collector/sentry_request_parser.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module ErrorTracking + module Collector + class SentryRequestParser + def self.parse(request) + # Request body can be "" or "gzip". + # If later then body was compressed with Zlib.gzip + encoding = request.headers['Content-Encoding'] + + body = if encoding == 'gzip' + Zlib.gunzip(request.body.read) + else + request.body.read + end + + # Request body contains 3 json objects merged together in one StringIO. + # We need to separate and parse them into array of hash objects. + json_objects = [] + parser = Yajl::Parser.new + + parser.parse(body) do |json_object| + json_objects << json_object + end + + # The request contains 3 objects: sentry metadata, type data and event data. + # We need only last two. Type to decide what to do with the request. + # And event data as it contains all information about the exception. + _, type, event = json_objects + + { + request_type: type['type'], + event: event + } + end + end + end +end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 4c537eeaa89..055a3a771c2 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -26,17 +26,17 @@ module ExtractsPath # Automatically renders `not_found!` if a valid tree path could not be # resolved (e.g., when a user inserts an invalid path or ref). # + # Automatically redirects to the current default branch if the ref matches a + # previous default branch that has subsequently been deleted. + # # rubocop:disable Gitlab/ModuleWithInstanceVariables override :assign_ref_vars def assign_ref_vars super - if @path.empty? && !@commit && @id.ends_with?('.atom') - @id = @ref = extract_ref_without_atom(@id) - @commit = @repo.commit(@ref) + rectify_atom! - request.format = :atom if @commit - end + rectify_renamed_default_branch! && return raise InvalidPathError unless @commit @@ -59,6 +59,42 @@ module ExtractsPath private + # Override in controllers to determine which actions are subject to the redirect + def redirect_renamed_default_branch? + false + end + + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def rectify_atom! + return if @commit + return unless @id.ends_with?('.atom') + return unless @path.empty? + + @id = @ref = extract_ref_without_atom(@id) + @commit = @repo.commit(@ref) + + request.format = :atom if @commit + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + + # For GET/HEAD requests, if the ref doesn't exist in the repository, check + # whether we're trying to access a renamed default branch. If we are, we can + # redirect to the current default branch instead of rendering a 404. + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def rectify_renamed_default_branch! + return unless redirect_renamed_default_branch? + return if @commit + return unless @id && @ref && repository_container.respond_to?(:previous_default_branch) + return unless repository_container.previous_default_branch == @ref + return unless request.get? || request.head? + + flash[:notice] = _('The default branch for this project has been changed. Please update your bookmarks.') + redirect_to url_for(id: @id.sub(/\A#{Regexp.escape(@ref)}/, repository_container.default_branch)) + + true + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + override :repository_container def repository_container @project diff --git a/lib/gitlab.rb b/lib/gitlab.rb index d93d7acbaad..f10168623e9 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -136,6 +136,7 @@ module Gitlab def self.process_name return 'sidekiq' if Gitlab::Runtime.sidekiq? + return 'action_cable' if Gitlab::Runtime.action_cable? return 'console' if Gitlab::Runtime.console? return 'test' if Rails.env.test? diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 530e53f9d10..8eb067ed0ec 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -31,6 +31,10 @@ module Gitlab raise NotImplementedError end + def hash_code + Digest::SHA256.hexdigest(self.class.identifier.to_s) + end + # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query. # Example: get me all the Issue records between start event end end event def timestamp_projection diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index 723486231b1..3546a7e3ddb 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -3,10 +3,6 @@ module Gitlab module Analytics class UniqueVisits - def track_visit(*args, **kwargs) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(*args, **kwargs) - end - # Returns number of unique visitors for given targets in given time frame # # @param [String, Array[<String>]] targets ids of targets to count visits on. Special case for :any diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 580c7042f1e..8cab2f65726 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -156,10 +156,10 @@ module Gitlab underscored_service = matched_login['service'].underscore - return unless Integration.available_services_names.include?(underscored_service) + return unless Integration.available_integration_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included - # in the Integration.available_services_names allowlist. + # in the Integration.available_integration_names allowlist. accessor = Project.integration_association_name(underscored_service) service = project.public_send(accessor) # rubocop:disable GitlabSecurity/PublicSend diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 416e36c7ccb..0796f23fbfe 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -89,9 +89,11 @@ module Gitlab job.user end - # We only allow Private Access Tokens with `api` scope to be used by web + # We allow Private Access Tokens with `api` scope to be used by web # requests on RSS feeds or ICS files for backwards compatibility. # It is also used by GraphQL/API requests. + # And to allow accessing /archive programatically as it was a big pain point + # for users https://gitlab.com/gitlab-org/gitlab/-/issues/28978. def find_user_from_web_access_token(request_format, scopes: [:api]) return unless access_token && valid_web_access_format?(request_format) @@ -269,6 +271,8 @@ module Gitlab ics_request? when :api api_request? + when :archive + archive_request? if Feature.enabled?(:allow_archive_as_web_access_format, default_enabled: :yaml) end end diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 3853709698b..47eca74aa5b 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -53,11 +53,7 @@ module Gitlab if results.nil? response = ldap.get_operation_result - - unless response.code == 0 - Gitlab::AppLogger.warn("LDAP search error: #{response.message}") - end - + check_empty_response_code(response) [] else results @@ -136,6 +132,16 @@ module Gitlab def renew_connection_adapter @ldap = Net::LDAP.new(config.adapter_options) end + + def check_empty_response_code(response) + if config.retry_empty_result_with_codes.include?(response.code) + raise Net::LDAP::Error, "Got empty results with response code: #{response.code}, message: #{response.message}" + end + + unless response.code == 0 + Gitlab::AppLogger.warn("LDAP search error: #{response.message}") + end + end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 441f0d14b39..7bfe776fed0 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -163,6 +163,10 @@ module Gitlab options['timeout'].to_i end + def retry_empty_result_with_codes + options.fetch('retry_empty_result_with_codes', []) + end + def external_groups options['external_groups'] || [] end diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb index f85b2248aeb..20b5d2ddc88 100644 --- a/lib/gitlab/auth/u2f_webauthn_converter.rb +++ b/lib/gitlab/auth/u2f_webauthn_converter.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'webauthn/u2f_migrator' + module Gitlab module Auth class U2fWebauthnConverter diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb new file mode 100644 index 00000000000..a0d0791b6af --- /dev/null +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill draft column on open merge requests based on regex parsing of + # their titles. + # + class BackfillDraftStatusOnMergeRequests + # Migration only version of MergeRequest table + class MergeRequest < ActiveRecord::Base + include EachBatch + + self.table_name = 'merge_requests' + + def self.eligible + where(state_id: 1) + .where(draft: false) + .where("title ~* ?", '^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP') + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) + + return if eligible_mrs.empty? + + eligible_mrs.each_slice(10) do |slice| + MergeRequest.where(id: slice).update_all(draft: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb new file mode 100644 index 00000000000..170af90805a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will populate the upvotes_count field + # for each issue + class BackfillUpvotesCountOnIssues + BATCH_SIZE = 1_000 + + def perform(start_id, stop_id) + (start_id..stop_id).step(BATCH_SIZE).each do |offset| + update_issue_upvotes_count(offset, offset + BATCH_SIZE) + end + end + + private + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + + def update_issue_upvotes_count(batch_start, batch_stop) + execute(<<~SQL) + UPDATE issues + SET upvotes_count = sub_q.count_all + FROM ( + SELECT COUNT(*) AS count_all, e.awardable_id AS issue_id + FROM award_emoji AS e + WHERE e.name = 'thumbsup' AND + e.awardable_type = 'Issue' AND + e.awardable_id BETWEEN #{batch_start} AND #{batch_stop} + GROUP BY issue_id + ) AS sub_q + WHERE sub_q.issue_id = issues.id; + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb new file mode 100644 index 00000000000..9ac4111ff0f --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_deployments.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned deployments. + class DeleteOrphanedDeployments + include Database::MigrationHelpers + + def perform(start_id, end_id) + orphaned_deployments + .where(id: start_id..end_id) + .delete_all + + mark_job_as_succeeded(start_id, end_id) + end + + def orphaned_deployments + define_batchable_model('deployments') + .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb new file mode 100644 index 00000000000..e694e5359cd --- /dev/null +++ b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb @@ -0,0 +1,287 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates author and committer names and emails from + # merge_request_diff_commits to two columns that point to + # merge_request_diff_commit_users. + # + # rubocop: disable Metrics/ClassLength + class MigrateMergeRequestDiffCommitUsers + # The number of user rows in merge_request_diff_commit_users to get in a + # single query. + USER_ROWS_PER_QUERY = 1_000 + + # The number of rows in merge_request_diff_commits to get in a single + # query. + COMMIT_ROWS_PER_QUERY = 10_000 + + # The number of rows in merge_request_diff_commits to update in a single + # query. + # + # Tests in staging revealed that increasing the number of updates per + # query translates to a longer total runtime for a migration. For example, + # given the same range of rows to migrate, 1000 updates per query required + # a total of roughly 15 seconds. On the other hand, 5000 updates per query + # required a total of roughly 25 seconds. For this reason, we use a value + # of 1000 rows per update. + UPDATES_PER_QUERY = 1_000 + + # rubocop: disable Style/Documentation + class MergeRequestDiffCommit < ActiveRecord::Base + include FromUnion + extend ::SuppressCompositePrimaryKeyWarning + + self.table_name = 'merge_request_diff_commits' + + # Yields each row to migrate in the given range. + # + # This method uses keyset pagination to ensure we don't retrieve + # potentially tens of thousands (or even hundreds of thousands) of rows + # in a single query. Such queries could time out, or increase the amount + # of memory needed to process the data. + # + # We can't use `EachBatch` and similar approaches, as + # merge_request_diff_commits doesn't have a single monotonically + # increasing primary key. + def self.each_row_to_migrate(start_id, stop_id, &block) + order = Pagination::Keyset::Order.build( + %w[merge_request_diff_id relative_order].map do |col| + Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: col, + order_expression: self.arel_table[col.to_sym].asc, + nullable: :not_nullable, + distinct: false + ) + end + ) + + scope = MergeRequestDiffCommit + .where(merge_request_diff_id: start_id...stop_id) + .order(order) + + Pagination::Keyset::Iterator + .new(scope: scope, use_union_optimization: true) + .each_batch(of: COMMIT_ROWS_PER_QUERY) { |rows| rows.each(&block) } + end + end + # rubocop: enable Style/Documentation + + # rubocop: disable Style/Documentation + class MergeRequestDiffCommitUser < ActiveRecord::Base + self.table_name = 'merge_request_diff_commit_users' + + def self.union(queries) + from("(#{queries.join("\nUNION ALL\n")}) #{table_name}") + end + end + # rubocop: enable Style/Documentation + + def perform(start_id, stop_id) + # This Hash maps user names + emails to their corresponding rows in + # merge_request_diff_commit_users. + user_mapping = {} + + user_details, diff_rows_to_update = get_data_to_update(start_id, stop_id) + + get_user_rows_in_batches(user_details, user_mapping) + create_missing_users(user_details, user_mapping) + update_commit_rows(diff_rows_to_update, user_mapping) + + Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'MigrateMergeRequestDiffCommitUsers', + [start_id, stop_id] + ) + end + + # Returns the data we'll use to determine what merge_request_diff_commits + # rows to update, and what data to use for populating their + # commit_author_id and committer_id columns. + def get_data_to_update(start_id, stop_id) + # This Set is used to retrieve users that already exist in + # merge_request_diff_commit_users. + users = Set.new + + # This Hash maps the primary key of every row in + # merge_request_diff_commits to the (trimmed) author and committer + # details to use for updating the row. + to_update = {} + + MergeRequestDiffCommit.each_row_to_migrate(start_id, stop_id) do |row| + author = [prepare(row.author_name), prepare(row.author_email)] + committer = [prepare(row.committer_name), prepare(row.committer_email)] + + to_update[[row.merge_request_diff_id, row.relative_order]] = + [author, committer] + + users << author if author[0] || author[1] + users << committer if committer[0] || committer[1] + end + + [users, to_update] + end + + # Gets any existing rows in merge_request_diff_commit_users in batches. + # + # This method may end up having to retrieve lots of rows. To reduce the + # overhead, we batch queries into a UNION query. We limit the number of + # queries per UNION so we don't end up sending a single query containing + # too many SELECT statements. + def get_user_rows_in_batches(users, user_mapping) + users.each_slice(USER_ROWS_PER_QUERY) do |pairs| + queries = pairs.map do |(name, email)| + MergeRequestDiffCommitUser.where(name: name, email: email).to_sql + end + + MergeRequestDiffCommitUser.union(queries).each do |row| + user_mapping[[row.name.to_s, row.email.to_s]] = row + end + end + end + + # Creates any users for which no row exists in + # merge_request_diff_commit_users. + # + # Not all users queried may exist yet, so we need to create any missing + # ones; making sure we handle concurrent creations of the same user + def create_missing_users(users, mapping) + create = [] + + users.each do |(name, email)| + create << { name: name, email: email } unless mapping[[name, email]] + end + + return if create.empty? + + MergeRequestDiffCommitUser + .insert_all(create, returning: %w[id name email]) + .each do |row| + mapping[[row['name'], row['email']]] = MergeRequestDiffCommitUser + .new(id: row['id'], name: row['name'], email: row['email']) + end + + # It's possible for (name, email) pairs to be inserted concurrently, + # resulting in the above insert not returning anything. Here we get any + # remaining users that were created concurrently. + get_user_rows_in_batches( + users.reject { |pair| mapping.key?(pair) }, + mapping + ) + end + + # Updates rows in merge_request_diff_commits with their new + # commit_author_id and committer_id values. + def update_commit_rows(to_update, user_mapping) + to_update.each_slice(UPDATES_PER_QUERY) do |slice| + updates = {} + + slice.each do |(diff_id, order), (author, committer)| + author_id = user_mapping[author]&.id + committer_id = user_mapping[committer]&.id + + updates[[diff_id, order]] = [author_id, committer_id] + end + + bulk_update_commit_rows(updates) + end + end + + # Bulk updates rows in the merge_request_diff_commits table with their new + # author and/or committer ID values. + # + # Updates are batched together to reduce the overhead of having to produce + # a single UPDATE for every row, as we may end up having to update + # thousands of rows at once. + # + # The query produced by this method is along the lines of the following: + # + # UPDATE merge_request_diff_commits + # SET commit_author_id = + # CASE + # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN X + # WHEN ... + # END, + # committer_id = + # CASE + # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN Y + # WHEN ... + # END + # WHERE (merge_request_diff_id, relative_order) IN ( (x, y), ... ) + # + # The `mapping` argument is a Hash in the following format: + # + # { [merge_request_diff_id, relative_order] => [author_id, committer_id] } + # + # rubocop: disable Metrics/AbcSize + def bulk_update_commit_rows(mapping) + author_case = Arel::Nodes::Case.new + committer_case = Arel::Nodes::Case.new + primary_values = [] + + mapping.each do |diff_id_and_order, (author_id, committer_id)| + primary_value = Arel::Nodes::Grouping.new(diff_id_and_order) + + primary_values << primary_value + + if author_id + author_case.when(primary_key.eq(primary_value)).then(author_id) + end + + if committer_id + committer_case.when(primary_key.eq(primary_value)).then(committer_id) + end + end + + if author_case.conditions.empty? && committer_case.conditions.empty? + return + end + + fields = [] + + # Statements such as `SET x = CASE END` are not valid SQL statements, so + # we omit setting an ID field if there are no values to populate it + # with. + if author_case.conditions.any? + fields << [arel_table[:commit_author_id], author_case] + end + + if committer_case.conditions.any? + fields << [arel_table[:committer_id], committer_case] + end + + query = Arel::UpdateManager.new + .table(arel_table) + .where(primary_key.in(primary_values)) + .set(fields) + .to_sql + + MergeRequestDiffCommit.connection.execute(query) + end + # rubocop: enable Metrics/AbcSize + + def primary_key + Arel::Nodes::Grouping.new( + [arel_table[:merge_request_diff_id], arel_table[:relative_order]] + ) + end + + def arel_table + MergeRequestDiffCommit.arel_table + end + + # Prepares a value to be inserted into a column in the table + # `merge_request_diff_commit_users`. Values in this table are limited to + # 512 characters. + # + # We treat empty strings as NULL values, as there's no point in (for + # example) storing a row where both the name and Email are an empty + # string. In addition, if we treated them differently we could end up with + # two rows: one where field X is NULL, and one where field X is an empty + # string. This is redundant, so we avoid storing such data. + def prepare(value) + value.present? ? value[0..511] : nil + end + end + # rubocop: enable Metrics/ClassLength + end +end diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb index 091e6660bac..83aa36a11e6 100644 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation -require "webauthn/u2f_migrator" module Gitlab module BackgroundMigration diff --git a/lib/gitlab/background_migration/populate_latest_pipeline_ids.rb b/lib/gitlab/background_migration/populate_latest_pipeline_ids.rb new file mode 100644 index 00000000000..a2b25a293fe --- /dev/null +++ b/lib/gitlab/background_migration/populate_latest_pipeline_ids.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +module Gitlab + module BackgroundMigration + class PopulateLatestPipelineIds + class ProjectSetting < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_settings' + + scope :in_range, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :has_vulnerabilities_without_latest_pipeline_set, -> do + joins('LEFT OUTER JOIN vulnerability_statistics vs ON vs.project_id = project_settings.project_id') + .where(vs: { latest_pipeline_id: nil }) + .where('has_vulnerabilities IS TRUE') + end + end + + def perform(start_id, end_id) + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::PopulateLatestPipelineIds.prepend_mod diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index 7da933c7b11..4026a91903f 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -21,7 +21,7 @@ module Gitlab belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" def for_personal_snippet? - noteable && noteable.class.name == 'PersonalSnippet' + noteable && noteable.instance_of?(PersonalSnippet) end def for_project_noteable? diff --git a/lib/gitlab/backup_logger.rb b/lib/gitlab/backup_logger.rb new file mode 100644 index 00000000000..fad36b860ae --- /dev/null +++ b/lib/gitlab/backup_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class BackupLogger < Gitlab::JsonLogger + def self.file_name_noext + 'backup_json' + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e59494c9d9c..f79baadb8ea 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -63,10 +63,7 @@ module Gitlab return users[username] if users.key?(username) - users[username] = User.select(:id) - .joins(:identities) - .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) - .try(:id) + users[username] = User.by_provider_and_extern_uid(:bitbucket, username).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9e958eb52fb..137f76bc96d 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -50,8 +50,6 @@ module Gitlab def load_status return if loaded? - return unless Gitlab::Ci::Features.pipeline_status_omit_commit_sha_in_cache_key?(project) || commit - if has_cache? load_from_cache else @@ -119,11 +117,7 @@ module Gitlab end def cache_key - if Gitlab::Ci::Features.pipeline_status_omit_commit_sha_in_cache_key?(project) - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status" - else - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status:#{commit&.sha}" - end + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:#{project.id}:pipeline_status" end def commit diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb new file mode 100644 index 00000000000..7b11d6bc9ff --- /dev/null +++ b/lib/gitlab/cache/helpers.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module Helpers + # @return [ActiveSupport::Duration] + DEFAULT_EXPIRY = 1.day + + # @return [ActiveSupport::Cache::Store] + def cache + Rails.cache + end + + def render_cached(obj_or_collection, with:, cache_context: -> (_) { current_user&.cache_key }, expires_in: Gitlab::Cache::Helpers::DEFAULT_EXPIRY, **presenter_args) + json = + if obj_or_collection.is_a?(Enumerable) + cached_collection( + obj_or_collection, + presenter: with, + presenter_args: presenter_args, + context: cache_context, + expires_in: expires_in + ) + else + cached_object( + obj_or_collection, + presenter: with, + presenter_args: presenter_args, + context: cache_context, + expires_in: expires_in + ) + end + + render Gitlab::Json::PrecompiledJson.new(json) + end + + private + + # Optionally uses a `Proc` to add context to a cache key + # + # @param object [Object] must respond to #cache_key + # @param context [Proc] a proc that will be called with the object as an argument, and which should return a + # string or array of strings to be combined into the cache key + # @return [String] + def contextual_cache_key(presenter, object, context) + return object.cache_key if context.nil? + + [presenter.class.name, object.cache_key, context.call(object)].flatten.join(":") + end + + # Used for fetching or rendering a single object + # + # @param object [Object] the object to render + # @param presenter [Grape::Entity] + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @param context [Proc] + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [String] + def cached_object(object, presenter:, presenter_args:, context:, expires_in:) + cache.fetch(contextual_cache_key(presenter, object, context), expires_in: expires_in) do + Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + end + end + + # Used for fetching or rendering multiple objects + # + # @param objects [Enumerable<Object>] the objects to render + # @param presenter [Grape::Entity] + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @param context [Proc] + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [Array<String>] + def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:) + json = fetch_multi(presenter, collection, context: context, expires_in: expires_in) do |obj| + Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + end + + json.values + end + + # An adapted version of ActiveSupport::Cache::Store#fetch_multi. + # + # The original method only provides the missing key to the block, + # not the missing object, so we have to create a map of cache keys + # to the objects to allow us to pass the object to the missing value + # block. + # + # The result is that this is functionally identical to `#fetch`. + def fetch_multi(presenter, *objs, context:, **kwargs) + objs.flatten! + map = multi_key_map(presenter, objs, context: context) + + # TODO: `contextual_cache_key` should be constructed based on the guideline https://docs.gitlab.com/ee/development/redis.html#multi-key-commands. + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + cache.fetch_multi(*map.keys, **kwargs) do |key| + yield map[key] + end + end + end + + # @param objects [Enumerable<Object>] objects which _must_ respond to `#cache_key` + # @param context [Proc] a proc that can be called to help generate each cache key + # @return [Hash] + def multi_key_map(presenter, objects, context:) + objects.index_by do |object| + contextual_cache_key(presenter, object, context) + end + end + end + end +end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 86441973941..4cbc0231bce 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -173,6 +173,34 @@ module Gitlab val ? true : false end + # Adds a value to a hash. + # + # raw_key - The key of the hash to add to. + # field - The field to add to the hash. + # value - The field value to add to the hash. + # timeout - The new timeout of the key. + def self.hash_add(raw_key, field, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.multi do |m| + m.hset(key, field, value) + m.expire(key, timeout) + end + end + end + + # Returns the values of the given hash. + # + # raw_key - The key of the set to check. + def self.values_from_hash(raw_key) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.hgetall(key) + end + end + def self.cache_key_for(raw_key) "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" end diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index be8009750da..0538fe68474 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -52,7 +52,12 @@ module Gitlab end if (template = hash['template']) - config.template = Parser.new.parse_and_transform(template) + config.template = + begin + TemplateParser::Parser.new.parse_and_transform(template) + rescue TemplateParser::Error => e + raise Error, e.message + end end if (categories = hash['categories']) @@ -73,7 +78,12 @@ module Gitlab def initialize(project) @project = project @date_format = DEFAULT_DATE_FORMAT - @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + @template = + begin + TemplateParser::Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + rescue TemplateParser::Error => e + raise Error, e.message + end @categories = {} @tag_regex = DEFAULT_TAG_REGEX end diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb index f2a01c2b0dc..c0b6a5c5679 100644 --- a/lib/gitlab/changelog/release.rb +++ b/lib/gitlab/changelog/release.rb @@ -54,7 +54,7 @@ module Gitlab end def to_markdown - state = EvalState.new + state = TemplateParser::EvalState.new data = { 'categories' => entries_for_template } # While not critical, we would like release sections to be separated by @@ -63,7 +63,12 @@ module Gitlab # # Since it can be a bit tricky to get this right in a template, we # enforce an empty line separator ourselves. - markdown = @config.template.evaluate(state, data).strip + markdown = + begin + @config.template.evaluate(state, data).strip + rescue TemplateParser::ParseError => e + raise Error, e.message + end # The release header can't be changed using the Liquid template, as we # need this to be in a known format. Without this restriction, we won't diff --git a/lib/gitlab/checks/container_moved.rb b/lib/gitlab/checks/container_moved.rb new file mode 100644 index 00000000000..41180bbf246 --- /dev/null +++ b/lib/gitlab/checks/container_moved.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class ContainerMoved < PostPushMessage + REDIRECT_NAMESPACE = "redirect_namespace" + + def initialize(repository, user, protocol, redirected_path) + @redirected_path = redirected_path + + super(repository, user, protocol) + end + + def message + <<~MESSAGE + #{container.class.model_name.human} '#{redirected_path}' was moved to '#{container.full_path}'. + + Please update your Git remote: + + git remote set-url origin #{url_to_repo} + MESSAGE + end + + private + + attr_reader :redirected_path + + def self.message_key(user, repository) + "#{REDIRECT_NAMESPACE}:#{user.id}:#{repository.gl_repository}" + end + + # TODO: Remove in the next release + # https://gitlab.com/gitlab-org/gitlab/-/issues/292030 + def self.legacy_message_key(user, repository) + return unless repository.project + + "#{REDIRECT_NAMESPACE}:#{user.id}:#{repository.project.id}" + end + end + end +end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index 51013b69755..84069a1249b 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -7,9 +7,9 @@ module Gitlab ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' def validate! - # This feature flag is used for disabling integrify check on some envs + # This feature flag is used for disabling integrity check on some envs # because these costy calculations may cause performance issues - return unless Feature.enabled?(:lfs_check, default_enabled: true) + return unless Feature.enabled?(:lfs_check, project, default_enabled: :yaml) return unless project.lfs_enabled? diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb index 0d93e7ac8a1..09407f45e54 100644 --- a/lib/gitlab/checks/post_push_message.rb +++ b/lib/gitlab/checks/post_push_message.rb @@ -9,21 +9,32 @@ module Gitlab @protocol = protocol end - def self.fetch_message(user_id, project_id) - key = message_key(user_id, project_id) + def self.fetch_message(user, repository) + key = message_key(user, repository) + + # Also check for messages in the legacy key + # TODO: Remove in the next release + # https://gitlab.com/gitlab-org/gitlab/-/issues/292030 + legacy_key = legacy_message_key(user, repository) if respond_to?(:legacy_message_key) Gitlab::Redis::SharedState.with do |redis| message = redis.get(key) redis.del(key) - message + + if legacy_key + legacy_message = redis.get(legacy_key) + redis.del(legacy_key) + end + + legacy_message || message end end def add_message - return unless user.present? && project.present? + return unless user && repository Gitlab::Redis::SharedState.with do |redis| - key = self.class.message_key(user.id, project.id) + key = self.class.message_key(user, repository) redis.setex(key, 5.minutes, message) end end @@ -39,7 +50,7 @@ module Gitlab delegate :project, to: :repository, allow_nil: true delegate :container, to: :repository, allow_nil: false - def self.message_key(user_id, project_id) + def self.message_key(user, repository) raise NotImplementedError end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb index 362562068e9..ea3be883be6 100644 --- a/lib/gitlab/checks/project_created.rb +++ b/lib/gitlab/checks/project_created.rb @@ -21,8 +21,16 @@ module Gitlab private - def self.message_key(user_id, project_id) - "#{PROJECT_CREATED}:#{user_id}:#{project_id}" + def self.message_key(user, repository) + "#{PROJECT_CREATED}:#{user.id}:#{repository.gl_repository}" + end + + # TODO: Remove in the next release + # https://gitlab.com/gitlab-org/gitlab/-/issues/292030 + def self.legacy_message_key(user, repository) + return unless repository.project + + "#{PROJECT_CREATED}:#{user.id}:#{repository.project.id}" end def project_url diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb deleted file mode 100644 index 4cc35de9c2d..00000000000 --- a/lib/gitlab/checks/project_moved.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Checks - class ProjectMoved < PostPushMessage - REDIRECT_NAMESPACE = "redirect_namespace" - - def initialize(repository, user, protocol, redirected_path) - @redirected_path = redirected_path - - super(repository, user, protocol) - end - - def message - <<~MESSAGE - Project '#{redirected_path}' was moved to '#{project.full_path}'. - - Please update your Git remote: - - git remote set-url origin #{url_to_repo} - MESSAGE - end - - private - - attr_reader :redirected_path - - def self.message_key(user_id, project_id) - "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" - end - end - end -end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 466706384c0..8f2d47e7ccc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -76,8 +76,14 @@ module Gitlab @section_header = true end - def set_section_duration(duration) - @section_duration = Time.at(duration.to_i).utc.strftime('%M:%S') + def set_section_duration(duration_in_seconds) + duration = ActiveSupport::Duration.build(duration_in_seconds.to_i) + hours = duration.in_hours.floor + hours = hours > 0 ? "%02d" % hours : nil + minutes = "%02d" % duration.parts[:minutes].to_i + seconds = "%02d" % duration.parts[:seconds].to_i + + @section_duration = [hours, minutes, seconds].compact.join(':') end def flush_current_segment! diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 6118ff49928..56eeb5eeb06 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -36,8 +36,7 @@ module Gitlab }, if: :expose_as_present? validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present? validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present? - validates :exclude, array_of_strings: true, if: :exclude_enabled? - validates :exclude, absence: { message: 'feature is disabled' }, unless: :exclude_enabled? + validates :exclude, array_of_strings: true validates :reports, type: Hash validates :when, inclusion: { in: %w[on_success on_failure always], @@ -60,10 +59,6 @@ module Gitlab !@config[:expose_as].nil? end - - def exclude_enabled? - ::Gitlab::Ci::Features.artifacts_exclude_enabled? - end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 4db25fb0930..e45dbfa243f 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -15,7 +15,7 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing api_fuzzing].freeze + requirements coverage_fuzzing api_fuzzing cluster_image_scanning].freeze attributes ALLOWED_KEYS @@ -32,6 +32,7 @@ module Gitlab validates :secret_detection, array_of_strings_or_string: true validates :dependency_scanning, array_of_strings_or_string: true validates :container_scanning, array_of_strings_or_string: true + validates :cluster_image_scanning, array_of_strings_or_string: true validates :dast, array_of_strings_or_string: true validates :performance, array_of_strings_or_string: true validates :browser_performance, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index fe69a170404..d26a903c1f8 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -6,18 +6,6 @@ module Gitlab # Ci::Features is a class that aggregates all CI/CD feature flags in one place. # module Features - def self.artifacts_exclude_enabled? - ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) - end - - def self.pipeline_latest? - ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) - end - - def self.pipeline_status_omit_commit_sha_in_cache_key?(project) - Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -34,10 +22,6 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.display_quality_on_mr_diff?(project) - ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: :yaml) - end - def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end diff --git a/lib/gitlab/ci/matching/runner_matcher.rb b/lib/gitlab/ci/matching/runner_matcher.rb index 63642674936..a729ca8a821 100644 --- a/lib/gitlab/ci/matching/runner_matcher.rb +++ b/lib/gitlab/ci/matching/runner_matcher.rb @@ -18,6 +18,7 @@ module Gitlab # class RunnerMatcher ATTRIBUTES = %i[ + runner_ids runner_type public_projects_minutes_cost_factor private_projects_minutes_cost_factor diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 66fc6741252..ef7447fa83d 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -10,10 +10,7 @@ module Gitlab def perform! raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result - - if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) - raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result - end + raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! @@ -51,13 +48,9 @@ module Gitlab end def root_variables - if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables - ) - else - @command.yaml_processor_result.root_variables - end + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 299b27a5f13..54d92745992 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,11 +11,16 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(context, attributes, previous_stages) + def initialize(context, attributes, previous_stages, current_stage) @context = context @pipeline = context.pipeline @seed_attributes = attributes - @previous_stages = previous_stages + @stages_for_needs_lookup = if Feature.enabled?(:ci_same_stage_job_needs, @pipeline.project, default_enabled: :yaml) + (previous_stages + [current_stage]).compact + else + previous_stages + end + @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) @job_variables = @seed_attributes.delete(:job_variables) @@ -67,6 +72,7 @@ module Gitlab .deep_merge(rules_attributes) .deep_merge(allow_failure_criteria_attributes) .deep_merge(@cache.cache_attributes) + .deep_merge(runner_tags) end def bridge? @@ -148,14 +154,18 @@ module Gitlab @needs_attributes.flat_map do |need| next if need[:optional] - result = @previous_stages.any? do |stage| - stage.seeds_names.include?(need[:name]) - end + result = need_present?(need) - "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result + "'#{name}' job needs '#{need[:name]}' job, but '#{need[:name]}' is not in any previous stage" unless result end.compact end + def need_present?(need) + @stages_for_needs_lookup.any? do |stage| + stage.seeds_names.include?(need[:name]) + end + end + def max_needs_allowed @pipeline.project.actual_limits.ci_needs_size_limit end @@ -202,6 +212,16 @@ module Gitlab end end + def runner_tags + { tag_list: evaluate_runner_tags }.compact + end + + def evaluate_runner_tags + @seed_attributes[:tag_list]&.map do |tag| + ExpandVariables.expand_existing(tag, evaluate_context.variables) + end + end + # If a job uses `allow_failure:exit_codes` and `rules:allow_failure` # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. @@ -213,8 +233,6 @@ module Gitlab end def recalculate_yaml_variables! - return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml) - @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) @@ -224,3 +242,5 @@ module Gitlab end end end + +Gitlab::Ci::Pipeline::Seed::Build.prepend_mod_with('Gitlab::Ci::Pipeline::Seed::Build') diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index c988ea10e41..018fb260986 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -17,7 +17,7 @@ module Gitlab @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(context, attributes, previous_stages) + Seed::Build.new(context, attributes, previous_stages, self) end end diff --git a/lib/gitlab/ci/pipeline_object_hierarchy.rb b/lib/gitlab/ci/pipeline_object_hierarchy.rb index de3262b10e0..e05a617f4fc 100644 --- a/lib/gitlab/ci/pipeline_object_hierarchy.rb +++ b/lib/gitlab/ci/pipeline_object_hierarchy.rb @@ -21,7 +21,7 @@ module Gitlab middle_table[:source_pipeline_id].eq(objects_table[:id]).and( middle_table[:pipeline_id].eq(cte.table[:id]) ).and( - same_project_condition + project_condition ) end @@ -29,15 +29,15 @@ module Gitlab middle_table[:pipeline_id].eq(objects_table[:id]).and( middle_table[:source_pipeline_id].eq(cte.table[:id]) ).and( - same_project_condition + project_condition ) end - def same_project_condition - if options[:same_project] - middle_table[:source_project_id].eq(middle_table[:project_id]) - else - Arel.sql('TRUE') + def project_condition + case options[:project_condition] + when :same then middle_table[:source_project_id].eq(middle_table[:project_id]) + when :different then middle_table[:source_project_id].not_eq(middle_table[:project_id]) + else Arel.sql('TRUE') end end end diff --git a/lib/gitlab/ci/reports/security/analyzer.rb b/lib/gitlab/ci/reports/security/analyzer.rb new file mode 100644 index 00000000000..b88eaf87cef --- /dev/null +++ b/lib/gitlab/ci/reports/security/analyzer.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Analyzer + attr_reader :id, :name, :version, :vendor + + def initialize(id:, name:, version:, vendor:) + @id = id + @name = name + @version = version + @vendor = vendor + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/concerns/fingerprint_path_from_file.rb b/lib/gitlab/ci/reports/security/concerns/fingerprint_path_from_file.rb new file mode 100644 index 00000000000..ec1d80e11c8 --- /dev/null +++ b/lib/gitlab/ci/reports/security/concerns/fingerprint_path_from_file.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Concerns + module FingerprintPathFromFile + extend ActiveSupport::Concern + + def fingerprint_path + File.basename(file_path.to_s) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/identifier.rb b/lib/gitlab/ci/reports/security/identifier.rb new file mode 100644 index 00000000000..4ba943cdcbc --- /dev/null +++ b/lib/gitlab/ci/reports/security/identifier.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Identifier + attr_reader :external_id + attr_reader :external_type + attr_reader :fingerprint + attr_reader :name + attr_reader :url + + def initialize(external_id:, external_type:, name:, url: nil) + @external_id = external_id + @external_type = external_type + @name = name + @url = url + + @fingerprint = generate_fingerprint + end + + def key + fingerprint + end + + def to_hash + %i[ + external_id + external_type + fingerprint + name + url + ].each_with_object({}) do |key, hash| + hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def ==(other) + other.external_type == external_type && + other.external_id == external_id + end + + def type_identifier? + cwe? || wasc? + end + + def cve? + external_type.to_s.casecmp?('cve') + end + + def cwe? + external_type.to_s.casecmp?('cwe') + end + + def wasc? + external_type.to_s.casecmp?('wasc') + end + + private + + def generate_fingerprint + Digest::SHA1.hexdigest("#{external_type}:#{external_id}") + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/link.rb b/lib/gitlab/ci/reports/security/link.rb new file mode 100644 index 00000000000..1c4c05cd9ac --- /dev/null +++ b/lib/gitlab/ci/reports/security/link.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Link + attr_accessor :name, :url + + def initialize(name: nil, url: nil) + @name = name + @url = url + end + + def to_hash + { + name: name, + url: url + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/scan.rb b/lib/gitlab/ci/reports/security/scan.rb new file mode 100644 index 00000000000..7dd0acc868b --- /dev/null +++ b/lib/gitlab/ci/reports/security/scan.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Scan + attr_accessor :type, :status, :start_time, :end_time + + def initialize(params = {}) + @type = params.dig('type') + @status = params.dig('status') + @start_time = params.dig('start_time') + @end_time = params.dig('end_time') + end + + def to_hash + { + type: type, + status: status, + start_time: start_time, + end_time: end_time + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/scanned_resource.rb b/lib/gitlab/ci/reports/security/scanned_resource.rb new file mode 100644 index 00000000000..605577eafcd --- /dev/null +++ b/lib/gitlab/ci/reports/security/scanned_resource.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class ScannedResource + include Gitlab::Utils::StrongMemoize + + attr_reader :request_method + attr_reader :request_uri + + delegate :scheme, :host, :port, :path, :query, to: :request_uri, prefix: :url + + def initialize(uri, request_method) + raise ArgumentError unless uri.is_a?(URI) + + @request_method = request_method + @request_uri = uri + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb new file mode 100644 index 00000000000..c1de03cea44 --- /dev/null +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Scanner + ANALYZER_ORDER = { + "bundler_audit" => 1, + "retire.js" => 2, + "gemnasium" => 3, + "gemnasium-maven" => 3, + "gemnasium-python" => 3, + "bandit" => 1, + "semgrep" => 2 + }.freeze + + attr_accessor :external_id, :name, :vendor, :version + + alias_method :key, :external_id + + def initialize(external_id:, name:, vendor:, version:) + @external_id = external_id + @name = name + @vendor = vendor + @version = version + end + + def to_hash + { + external_id: external_id.to_s, + name: name.to_s, + vendor: vendor.presence + }.compact + end + + def ==(other) + other.external_id == external_id + end + + def <=>(other) + sort_keys.compact <=> other.sort_keys.compact + end + + protected + + def sort_keys + @sort_keys ||= [order, external_id, name, vendor] + end + + private + + def order + ANALYZER_ORDER.fetch(external_id, Float::INFINITY) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 66f51f63585..dbbb9a01dab 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -31,7 +31,8 @@ module Gitlab project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', ci_quota_exceeded: 'no more CI minutes available', - no_matching_runner: 'no matching runner available' + no_matching_runner: 'no matching runner available', + trace_size_exceeded: 'log size limit exceeded' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 5368e020a50..3b2da773102 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -95,11 +95,7 @@ module Gitlab end def any_skipped_or_ignored? - if ::Feature.enabled?(:ci_fix_pipeline_status_for_dag_needs_manual, @project, default_enabled: :yaml) - any_of?(:skipped) || any_of?(:ignored) - else - any_of?(:skipped) - end + any_of?(:skipped) || any_of?(:ignored) end def consume_all_statuses(all_statuses) diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index e7ed2081f6a..f60f5243666 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -18,6 +18,10 @@ module Gitlab @user = user end + def id + "#{group}-#{subject.id}" + end + def icon raise NotImplementedError end diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml index c06ef83c180..ebb0b5948f1 100644 --- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml + # This template is on early stage of development. # Use it with caution. For usage instruction please read # https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md diff --git a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml index 267027a1b8a..60173cab54a 100644 --- a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml + stages: - provision - review diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 453803a6f7e..17e49440784 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml + stages: - build - test diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 2ff36bcc657..64e3b695e27 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml + # Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ # You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. # If you are looking for a simpler template that does not publish, see the Android template. diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index d20dabc0b00..b8a4c59c233 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml + # Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny # If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 207e2cf074a..adb5d430d46 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -1,4 +1,10 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml + # Auto DevOps +# # This CI/CD configuration provides a standard pipeline for # * building a Docker image (using a buildpack if necessary), # * storing the image in the container registry, diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 67e58d9ee99..1910913f2bd 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,4 +1,9 @@ -# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml + +# See https://docs.gitlab.com/ee/ci/yaml/README.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index 33a2a534508..bdcd3240380 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -1,6 +1,12 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/C++.gitlab-ci.yml + # use the official gcc image, based on debian # can use verions as well, like gcc:5.2 # see https://hub.docker.com/_/gcc/ + image: gcc build: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index d879e27dfcb..f166da9bdd6 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -1,4 +1,9 @@ -# This file uses Test Kitchen with the kitchen-dokken driver to +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml + +# This template uses Test Kitchen with the kitchen-dokken driver to # perform functional testing. Doing so requires that your runner be a # Docker runner configured for privileged mode. Please see # https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 0c5850bdb8e..0f9e28c9a8e 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml + # Based on openjdk:8, already includes lein image: clojure:lein-2.7.0 # If you need to configure a database, add a `services` section here diff --git a/lib/gitlab/ci/templates/Composer.gitlab-ci.yml b/lib/gitlab/ci/templates/Composer.gitlab-ci.yml index 5d9c68d3031..911acf8aff2 100644 --- a/lib/gitlab/ci/templates/Composer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Composer.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Composer.gitlab-ci.yml + # Publishes a tag/branch to Composer Packages of the current project publish: image: curlimages/curl:latest diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 538f96c4084..856a097e6e0 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/crystallang/crystal/ image: "crystallang/crystal:latest" diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index cc383f89b0c..a50e722f18a 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Dart.gitlab-ci.yml + # https://hub.docker.com/r/google/dart image: google/dart:2.8.4 diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index c657c7e8eb1..d2d3b3ed61e 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Django.gitlab-ci.yml + # Official framework image. Look for the different tagged releases at: # https://hub.docker.com/r/library/python image: python:latest diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index d0c63ab6edf..8f5f0e2c451 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,8 +1,14 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Docker.gitlab-ci.yml + # Build a Docker image with CI/CD and push to the GitLab registry. # Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html # # This template uses one generic job with conditional builds # for the default branch and all other (MR) branches. + docker-build: # Use the official docker image. image: docker:latest diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 7271526ab1b..1ceaf9fc86b 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml + image: elixir:latest # Pick zero or more services to be used on all builds. diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index 504ece611ca..d176ce19299 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml + code_quality: stage: test image: "cirrusci/flutter:1.22.5" diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 07d0de5f9e5..38036c1f964 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml + # This is a sample GitLab CI/CD configuration file that should run without any modifications. # It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, # it uses echo commands to simulate the pipeline execution. diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index 1b686bc6cc0..b5dd0005013 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Go.gitlab-ci.yml + image: golang:latest variables: diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index cbf4d58bdad..76f0c9f8427 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -1,6 +1,12 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml + # This is the Gradle build system for JVM applications # https://gradle.org/ # https://github.com/gradle/gradle + image: gradle:alpine # Disable the Gradle daemon for Continuous Integration servers as correctness diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index efcd1d3ddc0..3c514d7b0c6 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml + # This template uses the java:8 docker image because there isn't any # official Grails image at this moment # diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml deleted file mode 100644 index 90812083917..00000000000 --- a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml +++ /dev/null @@ -1,9 +0,0 @@ -# This file is a template demonstrating the `script` keyword. -# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script - -# After committing this template, visit CI/CD > Jobs to see the script output. - -job: - script: - # provide a shell script as argument for this keyword. - - echo "Hello World" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 6af79728dc8..80125a9bc01 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -1,6 +1,6 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ # -# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables variables: @@ -38,9 +38,6 @@ bandit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -57,9 +54,6 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -77,9 +71,6 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -100,9 +91,6 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -120,9 +108,6 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -138,9 +123,6 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -157,9 +139,6 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" @@ -194,9 +173,6 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -213,9 +189,6 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -232,9 +205,6 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -251,9 +221,6 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -271,9 +238,6 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -294,9 +258,6 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: @@ -313,9 +274,6 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. SAST_ANALYZER_IMAGE_TAG: 2 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index be0efc9180b..4687a07d05b 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml + # This is an example .gitlab-ci.yml file to test (and optionally report the coverage # results of) your [Julia][1] packages. Please refer to the [documentation][2] # for more information about package development in Julia. diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml index e4ed7fadfaa..1bc258d30c4 100644 --- a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml + --- variables: # Feel free to choose the image that suits you best. diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 5d2c8024524..43e4ac02d41 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml + # Official framework image. Look for the different tagged releases at: # https://hub.docker.com/r/library/php image: php:latest diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 97d0f611f47..dfa46d7af61 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Maven.gitlab-ci.yml + # Build JAVA applications using Apache Maven (http://maven.apache.org) # For docker image tags see https://hub.docker.com/_/maven/ # diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 36fe27f54c2..2f214347ec3 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml + # This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) # using the official mono docker image to build a visual studio project. # diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 92379ded77c..e48801b7970 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml + # Official framework image. Look for the different tagged releases at: # https://hub.docker.com/r/library/node/tags/ image: node:latest diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 45bddb1bc6a..7c8bbe464af 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml + image: openshift/origin-cli stages: diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 84e8223e69b..281bf7e3dd9 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/PHP.gitlab-ci.yml + # Select image from https://hub.docker.com/_/php/ image: php:latest @@ -8,9 +13,9 @@ cache: before_script: - apt-get update -yqq - - apt-get install -yqq git libmcrypt-dev libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev + - apt-get install -yqq git libpq-dev libcurl4-gnutls-dev libicu-dev libvpx-dev libjpeg-dev libpng-dev libxpm-dev zlib1g-dev libfreetype6-dev libxml2-dev libexpat1-dev libbz2-dev libgmp3-dev libldap2-dev unixodbc-dev libsqlite3-dev libaspell-dev libsnmp-dev libpcre3-dev libtidy-dev libonig-dev libzip-dev # Install PHP extensions - - docker-php-ext-install mbstring mcrypt pdo_pgsql curl json intl gd xml zip bz2 opcache + - docker-php-ext-install mbstring pdo_pgsql curl intl gd xml zip bz2 opcache # Install & enable Xdebug for code coverage reports - pecl install xdebug - docker-php-ext-enable xdebug diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 0b03ba6c3d8..3db712c6dc5 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Packer.gitlab-ci.yml + image: name: hashicorp/packer:latest entrypoint: diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 90cd8472916..55cf22b6601 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml + # Full project: https://gitlab.com/pages/brunch image: node:4.2.2 diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index 7435afef572..2f518d667a5 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml + # Full project: https://gitlab.com/pages/doxygen image: alpine diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 708c5063cc6..d3726fe34c5 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml + image: node:latest # This folder is cached between builds diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 694446dd6c9..17ed1d2e87f 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml + # Full project: https://gitlab.com/pages/plain-html pages: stage: deploy diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index a2fd6620909..9e48ac9fcdc 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml + # Full project: https://gitlab.com/pages/harp image: node:4.2.2 diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index fd75e47e899..a6f94a4d80e 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml + # Full project: https://gitlab.com/pages/hexo image: node:10.15.3 diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index a6a605e35f0..cfc4a1d904a 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml + --- # All available Hugo versions are listed here: # https://gitlab.com/pages/hugo/container_registry diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 1be2f4bad76..59e55efaee0 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml + # Full project: https://gitlab.com/pages/hyde image: python:2.7 diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 886b6c36249..8e15570fd1a 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml + # This template uses the java:8 docker image because there isn't any # official JBake image at this moment # diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 01e063c50ad..e0ad2e55f7d 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml + # Template project: https://gitlab.com/pages/jekyll # Docs: https://docs.gitlab.com/ee/pages/ image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index e39aa8a2063..26fac92d0dc 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml + # Jigsaw is a simple static sites generator with Laravel's Blade. # # Full project: https://github.com/tightenco/jigsaw diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 13d3089f4fa..9b5c1198c6c 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml + # Full project: https://gitlab.com/pages/hyde image: python:2.7 diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index e65cf3928f2..d97f0b7beb7 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml + # Full project: https://gitlab.com/pages/metalsmith image: node:4.2.2 diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 377fd8c396e..17ce0ef3659 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml + # Full project: https://gitlab.com/pages/middleman image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index 89281b41b66..a3ce96da244 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml + # Full project: https://gitlab.com/pages/nanoc image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 8fd4702b90d..4abdf66a21c 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml + # Full project: https://gitlab.com/pages/octopress image: ruby:2.6 diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index 09c6649fc13..7d52a407848 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml + # Full project: https://gitlab.com/pages/pelican image: python:2.7-alpine diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 9fa8b07f7cb..961941ac4d0 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml + image: node:10-alpine # specify the location of the Open API Specification files within your project diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index abce887d45b..aec41c137a4 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/python/tags/ image: python:latest diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 1bdaaeede43..490fc779e17 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/ruby/tags/ image: ruby:latest diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index 94117a79d1c..869c1782352 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Rust.gitlab-ci.yml + # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/rust/tags/ image: "rust:latest" diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index e081e20564a..ff8f9601189 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Scala.gitlab-ci.yml + # Official OpenJDK Java image. Look for the different tagged releases at # https://hub.docker.com/_/openjdk/ . A Java image is not required # but an image with a JVM speeds up the build a bit. diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 0c4c39cbcd6..009061ce844 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,6 +1,11 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 0c4c39cbcd6..ceeefa8aea6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,6 +1,11 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.lastest.gitlab-ci.yml -# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ +# +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml new file mode 100644 index 00000000000..f4f066cc7c2 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml @@ -0,0 +1,34 @@ +# Use this template to enable cluster image scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#customize-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables + +variables: + CIS_ANALYZER_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/cluster-image-scanning:0 + +cluster_image_scanning: + image: "$CIS_ANALYZER_IMAGE" + stage: test + allow_failure: true + artifacts: + reports: + cluster_image_scanning: gl-cluster-image-scanning-report.json + paths: [gl-cluster-image-scanning-report.json] + dependencies: [] + script: + - /analyzer run + rules: + - if: $CLUSTER_IMAGE_SCANNING_DISABLED + when: never + - if: '($KUBECONFIG == null || $KUBECONFIG == "") && ($CIS_KUBECONFIG == null || $CIS_KUBECONFIG == "")' + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcluster_image_scanning\b/ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index bd163f9db94..89e6743b0e4 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + # Use this template to enable container scanning in your project. # You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` # keyword. @@ -13,7 +18,7 @@ # - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the # DOCKERFILE_PATH variable. # -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 2dbfb80b419..7243f240eed 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -1,6 +1,11 @@ -# Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml -# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing +# +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index 9170e943e9d..a2933085d4e 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml + # To use this template, add the following to your .gitlab-ci.yml file: # # include: @@ -13,7 +18,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html -# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables variables: diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index a0564a16c07..3e7ab9b5c3b 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml + stages: - build - test @@ -5,7 +10,7 @@ stages: - dast variables: - DAST_VERSION: 1 + DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 5521a4a781b..0802868d67f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml + # To use this template, add the following to your .gitlab-ci.yml file: # # include: @@ -10,10 +15,10 @@ # - test # - deploy # - dast - +# # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ - -# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: @@ -43,15 +48,10 @@ dast: $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED && $DAST_WEBSITE == null && - $DAST_API_SPECIFICATION == null + $REVIEW_DISABLED when: never - if: $CI_COMMIT_BRANCH && $CI_KUBERNETES_ACTIVE && $GITLAB_FEATURES =~ /\bdast\b/ - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ && - $DAST_WEBSITE - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ && - $DAST_API_SPECIFICATION + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index e936364c86c..ac7d87a4cda 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.lastest.gitlab-ci.yml + # To use this template, add the following to your .gitlab-ci.yml file: # # include: @@ -13,7 +18,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 8df5ce79fe8..aa7b394a13c 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -1,6 +1,11 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ # -# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 870684c9f1d..1249b8d6fdc 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,6 +1,11 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml + # Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # -# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index d410c49b9a4..e30777d8401 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -1,16 +1,18 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml + # This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products) # have to be downloaded and stored locally. # # Usage: # -# ``` -# include: -# - template: Secure-Binaries.gitlab-ci.yml -# ``` +# include: +# - template: Secure-Binaries.gitlab-ci.yml # # Docs: https://docs.gitlab.com/ee/topics/airgap/ - variables: SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, @@ -222,7 +224,7 @@ license-finder: dast: extends: .download_images variables: - SECURE_BINARIES_ANALYZER_VERSION: "1" + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index 280e75d46f5..55648437191 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml + # GitLab Serverless template image: alpine:latest diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index cca0ba5d38e..eedb3b7a310 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Swift.gitlab-ci.yml + # Lifted from: https://about.gitlab.com/2016/03/10/setting-up-gitlab-ci-for-ios-projects/ # This file assumes an own GitLab CI runner, setup on a macOS system. stages: diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 62b32d7c2db..272b980b4b2 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml + include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 5963d7138c5..d34a847f2d5 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml + include: - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index e8a99a6ea06..22c40d8a8b8 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html stages: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index f0621165f8a..e0df9799917 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html stages: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index f0621165f8a..ad24ebae8d4 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html stages: diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml index 584e6966180..4d0f8c10a20 100644 --- a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml + rspec-rails-modified-path-specs: image: ruby:2.6 stage: .pre diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index cd23af562e5..53fabcfc721 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml + # Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/load_performance_testing.html stages: diff --git a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml index 05635cf71be..26b2c8694cd 100644 --- a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml + # Read more on when to use this template at # https://docs.gitlab.com/ee/ci/yaml/#workflowrules diff --git a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml index 50ff4c1f60b..28c25f48972 100644 --- a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml + # Read more on when to use this template at # https://docs.gitlab.com/ee/ci/yaml/#workflowrules diff --git a/lib/gitlab/ci/templates/dotNET-Core.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml index 40ca296d7bd..edd0fb0ba07 100644 --- a/lib/gitlab/ci/templates/dotNET-Core.yml +++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml @@ -1,7 +1,11 @@ ---- +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml + # This is a simple example illustrating how to build and test .NET Core project # with GitLab Continuous Integration / Continuous Delivery. - +# # ### Specify the Docker image # # Instead of installing .NET Core SDK manually, a docker image is used @@ -26,14 +30,6 @@ variables: # NOTE: Please edit this path so it matches the structure of your project! SOURCE_CODE_PATH: '*/*/' -# ### Define stage list -# -# In this example there are only two stages. -# Initially, the project will be built and then tested. -stages: - - build - - test - # ### Define global cache rule # # Before building the project, all dependencies (e.g. third-party NuGet packages) diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index b29f45323f5..dd88953b9a4 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml + # The following script will work for any project that can be built from command line by msbuild # It uses powershell shell executor, so you need to add the following line to your config.toml file # (located in gitlab-runner.exe directory): @@ -19,7 +24,6 @@ # The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473), # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes - # place project specific paths in variables to make the rest of the script more generic variables: EXE_RELEASE_FOLDER: 'YourApp\bin\Release' diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 87aea8527d1..0b75c298167 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml + # This is a very simple template that mainly relies on FastLane to build and distribute your app. # Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ # You will also need fastlane and signing configuration for this to work, along with a MacOS runner. diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 536cf9bd8d8..bfea437b8f1 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/npm.gitlab-ci.yml + publish: image: node:latest stage: deploy diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 84eb860a168..f9798023838 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -189,7 +189,7 @@ module Gitlab raise ArchiveError, 'Job is not finished yet' unless job.complete? if trace_artifact - unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml) + unsafe_trace_cleanup! raise AlreadyArchivedError, 'Could not archive again' end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a8c1002f2b9..c94fa84f608 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -46,6 +46,10 @@ module Gitlab @jobs.each do |name, job| validate_job!(name, job) end + + if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) + YamlProcessor::Dag.check_circular_dependencies!(@jobs) + end end def validate_job!(name, job) @@ -99,10 +103,16 @@ module Gitlab job_stage_index = stage_index(name) dependency_stage_index = stage_index(dependency) - # A dependency might be defined later in the configuration - # with a stage that does not exist - unless dependency_stage_index.present? && dependency_stage_index < job_stage_index - error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages") + if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml) + unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index + error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages") + end + else + # A dependency might be defined later in the configuration + # with a stage that does not exist + unless dependency_stage_index.present? && dependency_stage_index < job_stage_index + error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages") + end end end diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb new file mode 100644 index 00000000000..0140218d9bc --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Represents Dag pipeline +module Gitlab + module Ci + class YamlProcessor + class Dag + include TSort + + MissingNodeError = Class.new(StandardError) + + def initialize(nodes) + @nodes = nodes + end + + def self.check_circular_dependencies!(jobs) + nodes = jobs.values.to_h do |job| + name = job[:name].to_s + needs = job.dig(:needs, :job).to_a + + [name, needs.map { |need| need[:name].to_s }] + end + + new(nodes).tsort + rescue TSort::Cyclic + raise ValidationError, 'The pipeline has circular dependencies.' + rescue MissingNodeError + end + + def tsort_each_child(node, &block) + raise MissingNodeError, "node #{node} is missing" unless @nodes[node] + + @nodes[node].each(&block) + end + + def tsort_each_node(&block) + @nodes.each_key(&block) + end + end + end + end +end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index d7b31946ab0..842920ba02e 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -37,6 +37,7 @@ module Gitlab allow_webpack_dev_server(settings_hash) if Rails.env.development? allow_cdn(settings_hash) if ENV['GITLAB_CDN_HOST'].present? + allow_customersdot(settings_hash) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present? settings_hash end @@ -85,6 +86,12 @@ module Gitlab def self.append_to_directive(settings_hash, directive, text) settings_hash['directives'][directive] = "#{settings_hash['directives'][directive]} #{text}".strip end + + def self.allow_customersdot(settings_hash) + customersdot_host = ENV['CUSTOMER_PORTAL_URL'] + + append_to_directive(settings_hash, 'frame_src', customersdot_host) + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index aa419d75df2..a269b8d0366 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,15 +2,15 @@ module Gitlab module Database + CI_DATABASE_NAME = 'ci' + # This constant is used when renaming tables concurrently. # If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename. # Example: # TABLES_TO_BE_RENAMED = { # 'old_name' => 'new_name' # }.freeze - TABLES_TO_BE_RENAMED = { - 'services' => 'integrations' - }.freeze + TABLES_TO_BE_RENAMED = {}.freeze # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements @@ -68,6 +68,25 @@ module Gitlab end end + def self.has_config?(database_name) + Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s) + end + + def self.main_database?(name) + # The database is `main` if it is a first entry in `database.yml` + # Rails internally names them `primary` to avoid confusion + # with broad `primary` usage we use `main` instead + # + # TODO: The explicit `== 'main'` is needed in a transition period till + # the `database.yml` is not migrated into `main:` syntax + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65243 + ActiveRecord::Base.configurations.primary?(name.to_s) || name.to_s == 'main' + end + + def self.ci_database?(name) + name.to_s == CI_DATABASE_NAME + end + def self.username config['username'] || ENV['USER'] end @@ -333,6 +352,16 @@ module Gitlab end end + def self.dbname(ar_connection) + if ar_connection.respond_to?(:pool) && + ar_connection.pool.respond_to?(:db_config) && + ar_connection.pool.db_config.respond_to?(:database) + return ar_connection.pool.db_config.database + end + + 'unknown' + end + # inside_transaction? will return true if the caller is running within a transaction. Handles special cases # when running inside a test environment, where tests may be wrapped in transactions def self.inside_transaction? diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index e7e3c1766a9..eda991efbd5 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -24,6 +24,8 @@ module Gitlab end # Note: to be deleted after the minimum PG version is set to 12.0 + # Update the documentation together when deleting the method + # https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html#use-ctes-wisely def self.materialized_if_supported materialized_supported? ? 'MATERIALIZED' : '' end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 9a1dc4ee17d..03bd02d7554 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -44,6 +44,51 @@ module Gitlab # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801) duration.to_f / batched_migration.interval end + + def split_and_retry! + with_lock do + raise 'Only failed jobs can be split' unless failed? + + new_batch_size = batch_size / 2 + + raise 'Job cannot be split further' if new_batch_size < 1 + + batching_strategy = batched_migration.batch_class.new + next_batch_bounds = batching_strategy.next_batch( + batched_migration.table_name, + batched_migration.column_name, + batch_min_value: min_value, + batch_size: new_batch_size + ) + midpoint = next_batch_bounds.last + + # We don't want the midpoint to go over the existing max_value because + # those IDs would already be in the next batched migration job. + # This could happen when a lot of records in the current batch are deleted. + # + # In this case, we just lower the batch size so that future calls to this + # method could eventually split the job if it continues to fail. + if midpoint >= max_value + update!(batch_size: new_batch_size, attempts: 0) + else + old_max_value = max_value + + update!( + batch_size: new_batch_size, + max_value: midpoint, + attempts: 0, + started_at: nil, + finished_at: nil, + metrics: {} + ) + + new_record = dup + new_record.min_value = midpoint.next + new_record.max_value = old_max_value + new_record.save! + end + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 36e89023c86..9d66824da51 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -10,7 +10,7 @@ module Gitlab self.table_name = :batched_background_migrations has_many :batched_jobs, foreign_key: :batched_background_migration_id - has_one :last_job, -> { order(id: :desc) }, + has_one :last_job, -> { order(max_value: :desc) }, class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', foreign_key: :batched_background_migration_id @@ -29,11 +29,16 @@ module Gitlab paused: 0, active: 1, finished: 3, - failed: 4 + failed: 4, + finalizing: 5 } attribute :pause_ms, :integer, default: 100 + def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + for_configuration(job_class_name, table_name, column_name, job_arguments).first + end + def self.active_migration active.queue_order.first end diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 67fe6c536e6..14e3919986e 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -4,6 +4,12 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationRunner + FailedToFinalize = Class.new(RuntimeError) + + def self.finalize(job_class_name, table_name, column_name, job_arguments) + new.finalize(job_class_name, table_name, column_name, job_arguments) + end + def initialize(migration_wrapper = BatchedMigrationWrapper.new) @migration_wrapper = migration_wrapper end @@ -37,10 +43,35 @@ module Gitlab raise 'this method is not intended for use in real environments' end - while migration.active? - run_migration_job(migration) + run_migration_while(migration, :active) + end - migration.reload_last_job + # Finalize migration for given configuration. + # + # If the migration is already finished, do nothing. Otherwise change its status to `finalizing` + # in order to prevent it being picked up by the background worker. Perform all pending jobs, + # then keep running until migration is finished. + def finalize(job_class_name, table_name, column_name, job_arguments) + migration = BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + if migration.nil? + Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" + elsif migration.finished? + Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" + else + migration.finalizing! + migration.batched_jobs.pending.each { |job| migration_wrapper.perform(job) } + + run_migration_while(migration, :finalizing) + + raise FailedToFinalize unless migration.finished? end end @@ -90,6 +121,14 @@ module Gitlab active_migration.finished! end end + + def run_migration_while(migration, status) + while migration.status == status.to_s + run_migration_job(migration) + + migration.reload_last_job + end + end end end end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 9002d39e1ee..49f56b5be97 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -18,7 +18,7 @@ # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) # batch_count(Namespace.group(:type)) # batch_distinct_count(::Project, :creator_id) -# batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) +# batch_distinct_count(::Project.aimed_for_deletion.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) # batch_distinct_count(Project.group(:visibility_level), :creator_id) # batch_sum(User, :sign_in_count) # batch_sum(Issue.group(:state_id), :weight)) @@ -41,159 +41,5 @@ module Gitlab include BatchCount end end - - class BatchCounter - FALLBACK = -1 - MIN_REQUIRED_BATCH_SIZE = 1_250 - DEFAULT_SUM_BATCH_SIZE = 1_000 - MAX_ALLOWED_LOOPS = 10_000 - SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep - ALLOWED_MODES = [:itself, :distinct].freeze - FALLBACK_FINISH = 0 - OFFSET_BY_ONE = 1 - - # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 - DEFAULT_DISTINCT_BATCH_SIZE = 10_000 - DEFAULT_BATCH_SIZE = 100_000 - - def initialize(relation, column: nil, operation: :count, operation_args: nil) - @relation = relation - @column = column || relation.primary_key - @operation = operation - @operation_args = operation_args - end - - def unwanted_configuration?(finish, batch_size, start) - (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || - (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || - (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || - start >= finish - end - - def count(batch_size: nil, mode: :itself, start: nil, finish: nil) - raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? - - check_mode!(mode) - - # non-distinct have better performance - batch_size ||= batch_size_for_mode_and_operation(mode, @operation) - - start = actual_start(start) - finish = actual_finish(finish) - - raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) - - results = nil - batch_start = start - - while batch_start < finish - begin - batch_end = [batch_start + batch_size, finish].min - batch_relation = build_relation_batch(batch_start, batch_end, mode) - - op_args = @operation_args - if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) - op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] - end - - results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend - batch_start = batch_end - rescue ActiveRecord::QueryCanceled => error - # retry with a safe batch size & warmer cache - if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE - batch_size /= 2 - else - log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) - return FALLBACK - end - rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error - Gitlab::AppJsonLogger - .error( - event: 'batch_count', - relation: @relation.table_name, - operation: @operation, - operation_args: @operation_args, - mode: mode, - message: "LooseIndexScanDistinctCount column error: #{error.message}" - ) - - return FALLBACK - end - - sleep(SLEEP_TIME_IN_SECONDS) - end - - results - end - - def merge_results(results, object) - return object unless results - - if object.is_a?(Hash) - results.merge!(object) { |_, a, b| a + b } - else - results + object - end - end - - private - - def build_relation_batch(start, finish, mode) - if use_loose_index_scan_for_distinct_values?(mode) - Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) - else - @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def batch_size_for_mode_and_operation(mode, operation) - return DEFAULT_SUM_BATCH_SIZE if operation == :sum - - mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE - end - - def between_condition(start, finish) - return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute) - - { @column => start...finish } - end - - def actual_start(start) - start || @relation.unscope(:group, :having).minimum(@column) || 0 - end - - def actual_finish(finish) - (finish || @relation.unscope(:group, :having).maximum(@column) || FALLBACK_FINISH) + OFFSET_BY_ONE - end - - def check_mode!(mode) - raise "The mode #{mode.inspect} is not supported" unless ALLOWED_MODES.include?(mode) - raise 'Use distinct count for optimized distinct counting' if @relation.limit(1).distinct_value.present? && mode != :distinct - raise 'Use distinct count only with non id fields' if @column == :id && mode == :distinct - end - - def log_canceled_batch_fetch(batch_start, mode, query, error) - Gitlab::AppJsonLogger - .error( - event: 'batch_count', - relation: @relation.table_name, - operation: @operation, - operation_args: @operation_args, - start: batch_start, - mode: mode, - query: query, - message: "Query has been canceled with message: #{error.message}" - ) - end - - def use_loose_index_scan_for_distinct_values?(mode) - Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct - end - - def not_group_by_query? - !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? - end - end end end diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb new file mode 100644 index 00000000000..5f2e404c9da --- /dev/null +++ b/lib/gitlab/database/batch_counter.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class BatchCounter + FALLBACK = -1 + MIN_REQUIRED_BATCH_SIZE = 1_250 + DEFAULT_SUM_BATCH_SIZE = 1_000 + MAX_ALLOWED_LOOPS = 10_000 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + ALLOWED_MODES = [:itself, :distinct].freeze + FALLBACK_FINISH = 0 + OFFSET_BY_ONE = 1 + + # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 + DEFAULT_DISTINCT_BATCH_SIZE = 10_000 + DEFAULT_BATCH_SIZE = 100_000 + + def initialize(relation, column: nil, operation: :count, operation_args: nil) + @relation = relation + @column = column || relation.primary_key + @operation = operation + @operation_args = operation_args + end + + def unwanted_configuration?(finish, batch_size, start) + (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || + (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || + (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + start >= finish + end + + def count(batch_size: nil, mode: :itself, start: nil, finish: nil) + raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + + check_mode!(mode) + + # non-distinct have better performance + batch_size ||= batch_size_for_mode_and_operation(mode, @operation) + + start = actual_start(start) + finish = actual_finish(finish) + + raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 + return FALLBACK if unwanted_configuration?(finish, batch_size, start) + + results = nil + batch_start = start + + while batch_start < finish + begin + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end, mode) + + op_args = @operation_args + if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) + op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] + end + + results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend + batch_start = batch_end + rescue ActiveRecord::QueryCanceled => error + # retry with a safe batch size & warmer cache + if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE + batch_size /= 2 + else + log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) + return FALLBACK + end + rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + mode: mode, + message: "LooseIndexScanDistinctCount column error: #{error.message}" + ) + + return FALLBACK + end + + sleep(SLEEP_TIME_IN_SECONDS) + end + + results + end + + def merge_results(results, object) + return object unless results + + if object.is_a?(Hash) + results.merge!(object) { |_, a, b| a + b } + else + results + object + end + end + + private + + def build_relation_batch(start, finish, mode) + if use_loose_index_scan_for_distinct_values?(mode) + Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) + else + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def batch_size_for_mode_and_operation(mode, operation) + return DEFAULT_SUM_BATCH_SIZE if operation == :sum + + mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE + end + + def between_condition(start, finish) + return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute) + + { @column => start...finish } + end + + def actual_start(start) + start || @relation.unscope(:group, :having).minimum(@column) || 0 + end + + def actual_finish(finish) + (finish || @relation.unscope(:group, :having).maximum(@column) || FALLBACK_FINISH) + OFFSET_BY_ONE + end + + def check_mode!(mode) + raise "The mode #{mode.inspect} is not supported" unless ALLOWED_MODES.include?(mode) + raise 'Use distinct count for optimized distinct counting' if @relation.limit(1).distinct_value.present? && mode != :distinct + raise 'Use distinct count only with non id fields' if @column == :id && mode == :distinct + end + + def log_canceled_batch_fetch(batch_start, mode, query, error) + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + start: batch_start, + mode: mode, + query: query, + message: "Query has been canceled with message: #{error.message}" + ) + end + + def use_loose_index_scan_for_distinct_values?(mode) + Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct + end + + def not_group_by_query? + !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? + end + end + end +end diff --git a/lib/gitlab/database/custom_structure.rb b/lib/gitlab/database/custom_structure.rb deleted file mode 100644 index e4404e73a63..00000000000 --- a/lib/gitlab/database/custom_structure.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class CustomStructure - CUSTOM_DUMP_FILE = 'db/gitlab_structure.sql' - - def dump - File.open(self.class.custom_dump_filepath, 'wb') do |io| - io << "-- this file tracks custom GitLab data, such as foreign keys referencing partitioned tables\n" - io << "-- more details can be found in the issue: https://gitlab.com/gitlab-org/gitlab/-/issues/201872\n\n" - - dump_partitioned_foreign_keys(io) if partitioned_foreign_keys_exist? - end - end - - def self.custom_dump_filepath - Rails.root.join(CUSTOM_DUMP_FILE) - end - - private - - def dump_partitioned_foreign_keys(io) - io << "COPY partitioned_foreign_keys (#{partitioned_fk_columns.join(", ")}) FROM STDIN;\n" - - PartitioningMigrationHelpers::PartitionedForeignKey.find_each do |fk| - io << fk.attributes.values_at(*partitioned_fk_columns).join("\t") << "\n" - end - io << "\\.\n" - end - - def partitioned_foreign_keys_exist? - return false unless PartitioningMigrationHelpers::PartitionedForeignKey.table_exists? - - PartitioningMigrationHelpers::PartitionedForeignKey.exists? - end - - def partitioned_fk_columns - @partitioned_fk_columns ||= PartitioningMigrationHelpers::PartitionedForeignKey.column_names - end - end - end -end diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 7439591be99..220062f1bc6 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -3,6 +3,8 @@ module Gitlab module Database module DynamicModelHelpers + BATCH_SIZE = 1_000 + def define_batchable_model(table_name) Class.new(ActiveRecord::Base) do include EachBatch @@ -12,7 +14,7 @@ module Gitlab end end - def each_batch(table_name, scope: ->(table) { table.all }, of: 1000) + def each_batch(table_name, scope: ->(table) { table.all }, of: BATCH_SIZE) if transaction_open? raise <<~MSG.squish each_batch should not run inside a transaction, you can disable @@ -25,7 +27,7 @@ module Gitlab .each_batch(of: of) { |batch| yield batch } end - def each_batch_range(table_name, scope: ->(table) { table.all }, of: 1000) + def each_batch_range(table_name, scope: ->(table) { table.all }, of: BATCH_SIZE) each_batch(table_name, scope: scope, of: of) do |batch| yield batch.pluck('MIN(id), MAX(id)').first end diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 88743cd2e75..31d41a6d6c0 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -85,7 +85,6 @@ module Gitlab # Returns true if load balancing is to be enabled. def self.enable? return false if Gitlab::Runtime.rake? - return false if Gitlab::Runtime.sidekiq? && !Gitlab::Utils.to_boolean(ENV['ENABLE_LOAD_BALANCING_FOR_SIDEKIQ'], default: false) return false unless self.configured? true diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index a833bb8491f..a5d67ebc050 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -147,15 +147,15 @@ module Gitlab raise 'Failed to determine the write location of the primary database' end - # Returns true if all hosts have caught up to the given transaction - # write location. - def all_caught_up?(location) - @host_list.hosts.all? { |host| host.caught_up?(location) } - end - # Returns true if there was at least one host that has caught up with the given transaction. # # In case of a retry, this method also stores the set of hosts that have caught up. + # + # UPD: `select_caught_up_hosts` seems to have redundant logic managing host list (`:gitlab_load_balancer_valid_hosts`), + # while we only need a single host: https://gitlab.com/gitlab-org/gitlab/-/issues/326125#note_615271604 + # Also, shuffling the list afterwards doesn't seem to be necessary. + # This may be improved by merging this method with `select_up_to_date_host`. + # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out def select_caught_up_hosts(location) all_hosts = @host_list.hosts valid_hosts = all_hosts.select { |host| host.caught_up?(location) } @@ -179,6 +179,8 @@ module Gitlab # Returns true if there was at least one host that has caught up with the given transaction. # Similar to `#select_caught_up_hosts`, picks a random host, to rotate replicas we use. # Unlike `#select_caught_up_hosts`, does not iterate over all hosts if finds any. + # + # It is going to be merged with `select_caught_up_hosts`, because they intend to do the same. def select_up_to_date_host(location) all_hosts = @host_list.hosts.shuffle host = all_hosts.find { |host| host.caught_up?(location) } @@ -190,6 +192,7 @@ module Gitlab true end + # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out def set_consistent_hosts_for_request(hosts) RequestStore[VALID_HOSTS_CACHE_KEY] = hosts end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb index 4734ff99bd3..8e7e6865402 100644 --- a/lib/gitlab/database/load_balancing/rack_middleware.rb +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -39,6 +39,8 @@ module Gitlab result = @app.call(env) + ActiveSupport::Notifications.instrument('web_transaction_completed.load_balancing') + stick_if_necessary(env) result diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 524d69c00c0..0e36ebbc3ee 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -5,27 +5,29 @@ module Gitlab module LoadBalancing class SidekiqClientMiddleware def call(worker_class, job, _queue, _redis_pool) + # Mailers can't be constantized worker_class = worker_class.to_s.safe_constantize - mark_data_consistency_location(worker_class, job) + if load_balancing_enabled?(worker_class) + job['worker_data_consistency'] = worker_class.get_data_consistency + set_data_consistency_location!(job) unless location_already_provided?(job) + else + job['worker_data_consistency'] = ::WorkerAttributes::DEFAULT_DATA_CONSISTENCY + end yield end private - def mark_data_consistency_location(worker_class, job) - # Mailers can't be constantized - return unless worker_class - return unless worker_class.include?(::ApplicationWorker) - return unless worker_class.get_data_consistency_feature_flag_enabled? - - return if location_already_provided?(job) - - job['worker_data_consistency'] = worker_class.get_data_consistency - - return unless worker_class.utilizes_load_balancing_capabilities? + def load_balancing_enabled?(worker_class) + worker_class && + worker_class.include?(::ApplicationWorker) && + worker_class.utilizes_load_balancing_capabilities? && + worker_class.get_data_consistency_feature_flag_enabled? + end + def set_data_consistency_location!(job) if Session.current.use_primary? job['database_write_location'] = load_balancer.primary_write_location else diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 9bd0adf8dbd..0551750568a 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -7,8 +7,18 @@ module Gitlab JobReplicaNotUpToDate = Class.new(StandardError) def call(worker, job, _queue) - if requires_primary?(worker.class, job) + worker_class = worker.class + strategy = select_load_balancing_strategy(worker_class, job) + + job['load_balancing_strategy'] = strategy.to_s + + if use_primary?(strategy) Session.current.use_primary! + elsif strategy == :retry + raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ + " Replica was not up to date." + else + # this means we selected an up-to-date replica, but there is nothing to do in this case. end yield @@ -23,31 +33,42 @@ module Gitlab Session.clear_session end - def requires_primary?(worker_class, job) - return true unless worker_class.include?(::ApplicationWorker) - return true unless worker_class.utilizes_load_balancing_capabilities? - return true unless worker_class.get_data_consistency_feature_flag_enabled? - - location = job['database_write_location'] || job['database_replica_location'] + def use_primary?(strategy) + strategy.start_with?('primary') + end - return true unless location + def select_load_balancing_strategy(worker_class, job) + return :primary unless load_balancing_available?(worker_class) - job_data_consistency = worker_class.get_data_consistency - job[:data_consistency] = job_data_consistency.to_s + location = job['database_write_location'] || job['database_replica_location'] + return :primary_no_wal unless location if replica_caught_up?(location) - job[:database_chosen] = 'replica' - false - elsif job_data_consistency == :delayed && not_yet_retried?(job) - job[:database_chosen] = 'retry' - raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ - " Replica was not up to date." + # Happy case: we can read from a replica. + retried_before?(worker_class, job) ? :replica_retried : :replica + elsif can_retry?(worker_class, job) + # Optimistic case: The worker allows retries and we have retries left. + :retry else - job[:database_chosen] = 'primary' - true + # Sad case: we need to fall back to the primary. + :primary end end + def load_balancing_available?(worker_class) + worker_class.include?(::ApplicationWorker) && + worker_class.utilizes_load_balancing_capabilities? && + worker_class.get_data_consistency_feature_flag_enabled? + end + + def can_retry?(worker_class, job) + worker_class.get_data_consistency == :delayed && not_yet_retried?(job) + end + + def retried_before?(worker_class, job) + worker_class.get_data_consistency == :delayed && !not_yet_retried?(job) + end + def not_yet_retried?(job) # if `retry_count` is `nil` it indicates that this job was never retried # the `0` indicates that this is a first retry @@ -59,11 +80,7 @@ module Gitlab end def replica_caught_up?(location) - if Feature.enabled?(:sidekiq_load_balancing_rotate_up_to_date_replica) - load_balancer.select_up_to_date_host(location) - else - load_balancer.host.caught_up?(location) - end + load_balancer.select_up_to_date_host(location) end end end diff --git a/lib/gitlab/database/load_balancing/srv_resolver.rb b/lib/gitlab/database/load_balancing/srv_resolver.rb index 20da525f4d2..1f599ef4a27 100644 --- a/lib/gitlab/database/load_balancing/srv_resolver.rb +++ b/lib/gitlab/database/load_balancing/srv_resolver.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require 'net/dns' + module Gitlab module Database module LoadBalancing diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index efbd7099300..8e1aa079216 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -33,8 +33,10 @@ module Gitlab return true unless location - load_balancer.all_caught_up?(location).tap do |caught_up| - unstick(namespace, id) if caught_up + load_balancer.select_up_to_date_host(location).tap do |found| + ActiveSupport::Notifications.instrument('caught_up_replica_pick.load_balancing', { result: found } ) + + unstick(namespace, id) if found end end @@ -51,8 +53,14 @@ module Gitlab # write location. If no such location exists, err on the side of caution. return false unless location - load_balancer.select_caught_up_hosts(location).tap do |selected| - unstick(namespace, id) if selected + if ::Feature.enabled?(:load_balancing_refine_load_balancer_methods) + load_balancer.select_up_to_date_host(location).tap do |selected| + unstick(namespace, id) if selected + end + else + load_balancer.select_caught_up_hosts(location).tap do |selected| + unstick(namespace, id) if selected + end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d155abefdc8..842ab4f7b80 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -217,11 +217,12 @@ module Gitlab # source - The source table containing the foreign key. # target - The target table the key points to. # column - The name of the column to create the foreign key on. + # target_column - The name of the referenced column, defaults to "id". # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". # name - The name of the foreign key. # - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, name: nil, validate: true) + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -231,7 +232,8 @@ module Gitlab options = { column: column, on_delete: on_delete, - name: name.presence || concurrent_foreign_key_name(source, column) + name: name.presence || concurrent_foreign_key_name(source, column), + primary_key: target_column } if foreign_key_exists?(source, target, **options) @@ -252,7 +254,7 @@ module Gitlab ALTER TABLE #{source} ADD CONSTRAINT #{options[:name]} FOREIGN KEY (#{options[:column]}) - REFERENCES #{target} (id) + REFERENCES #{target} (#{target_column}) #{on_delete_statement(options[:on_delete])} NOT VALID; EOF @@ -389,12 +391,14 @@ module Gitlab # * +logger+ - [Gitlab::JsonLogger] # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` def with_lock_retries(*args, **kwargs, &block) + raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) merged_args = { klass: self.class, logger: Gitlab::BackgroundMigration::Logger }.merge(kwargs) - Gitlab::Database::WithLockRetries.new(**merged_args).run(&block) + Gitlab::Database::WithLockRetries.new(**merged_args) + .run(raise_on_exhaustion: raise_on_exhaustion, &block) end def true_value @@ -1106,7 +1110,16 @@ module Gitlab Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" elsif !migration.finished? raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ - "but it is '#{migration.status}': #{configuration}" + "but it is '#{migration.status}':" \ + "\t#{configuration}" \ + "\n\n" \ + "Finalize it manualy by running" \ + "\n\n" \ + "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.inspect.gsub(',', '\,')}']" \ + "\n\n" \ + "For more information, check the documentation" \ + "\n\n" \ + "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" end end @@ -1610,6 +1623,13 @@ into similar problems in the future (e.g. when new tables are created). raise end + def rename_constraint(table_name, old_name, new_name) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + private def validate_check_constraint_name!(constraint_name) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index fa30ffb62f5..28491a934a0 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -107,7 +107,10 @@ module Gitlab batch_counter = 0 model_class.each_batch(of: batch_size) do |relation, index| - start_id, end_id = relation.pluck(Arel.sql("MIN(#{primary_column_name}), MAX(#{primary_column_name})")).first + max = relation.arel_table[primary_column_name].maximum + min = relation.arel_table[primary_column_name].minimum + + start_id, end_id = relation.pluck(min, max).first # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 82ea1ce26fb..4c68399cb68 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -4,16 +4,17 @@ module Gitlab module Database module Partitioning class MonthlyStrategy - attr_reader :model, :partitioning_key + attr_reader :model, :partitioning_key, :retain_for # We create this many partitions in the future HEADROOM = 6.months delegate :table_name, to: :model - def initialize(model, partitioning_key) + def initialize(model, partitioning_key, retain_for: nil) @model = model @partitioning_key = partitioning_key + @retain_for = retain_for end def current_partitions @@ -27,13 +28,21 @@ module Gitlab desired_partitions - current_partitions end + def extra_partitions + current_partitions - desired_partitions + end + private def desired_partitions [].tap do |parts| min_date, max_date = relevant_range - parts << partition_for(upper_bound: min_date) + if pruning_old_partitions? && min_date <= oldest_active_date + min_date = oldest_active_date.beginning_of_month + else + parts << partition_for(upper_bound: min_date) + end while min_date < max_date next_date = min_date.next_month @@ -52,13 +61,17 @@ module Gitlab # to start from MINVALUE to a specific date `x`. The range returned # does not include the range of the first, half-unbounded partition. def relevant_range - if first_partition = current_partitions.min + if (first_partition = current_partitions.min) # Case 1: First partition starts with MINVALUE, i.e. from is nil -> start with first real partition # Case 2: Rather unexpectedly, first partition does not start with MINVALUE, i.e. from is not nil # In this case, use first partition beginning as a start min_date = first_partition.from || first_partition.to end + if pruning_old_partitions? + min_date ||= oldest_active_date + end + # In case we don't have a partition yet min_date ||= Date.today min_date = min_date.beginning_of_month @@ -72,6 +85,14 @@ module Gitlab TimePartition.new(table_name, lower_bound, upper_bound) end + def pruning_old_partitions? + Feature.enabled?(:partition_pruning_dry_run) && retain_for.present? + end + + def oldest_active_date + (Date.today - retain_for).beginning_of_month + end + def connection ActiveRecord::Base.connection end diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_manager.rb index d4b2b8d50e2..c2a9422a42a 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -3,7 +3,7 @@ module Gitlab module Database module Partitioning - class PartitionCreator + class PartitionManager def self.register(model) raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy) @@ -15,7 +15,7 @@ module Gitlab end LEASE_TIMEOUT = 1.minute - LEASE_KEY = 'database_partition_creation_%s' + MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' attr_reader :models @@ -23,23 +23,25 @@ module Gitlab @models = models end - def create_partitions + def sync_partitions Gitlab::AppLogger.info("Checking state of dynamic postgres partitions") models.each do |model| # Double-checking before getting the lease: - # The prevailing situation is no missing partitions - next if missing_partitions(model).empty? + # The prevailing situation is no missing partitions and no extra partitions + next if missing_partitions(model).empty? && extra_partitions(model).empty? - only_with_exclusive_lease(model) do + only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do partitions_to_create = missing_partitions(model) + create(partitions_to_create) unless partitions_to_create.empty? - next if partitions_to_create.empty? - - create(model, partitions_to_create) + if Feature.enabled?(:partition_pruning_dry_run) + partitions_to_detach = extra_partitions(model) + detach(partitions_to_detach) unless partitions_to_detach.empty? + end end rescue StandardError => e - Gitlab::AppLogger.error("Failed to create partition(s) for #{model.table_name}: #{e.class}: #{e.message}") + Gitlab::AppLogger.error("Failed to create / detach partition(s) for #{model.table_name}: #{e.class}: #{e.message}") end end @@ -51,15 +53,22 @@ module Gitlab model.partitioning_strategy.missing_partitions end - def only_with_exclusive_lease(model) - lease = Gitlab::ExclusiveLease.new(LEASE_KEY % model.table_name, timeout: LEASE_TIMEOUT) + def extra_partitions(model) + return [] unless Feature.enabled?(:partition_pruning_dry_run) + return [] unless connection.table_exists?(model.table_name) + + model.partitioning_strategy.extra_partitions + end + + def only_with_exclusive_lease(model, lease_key:) + lease = Gitlab::ExclusiveLease.new(lease_key % model.table_name, timeout: LEASE_TIMEOUT) yield if lease.try_obtain ensure lease&.cancel end - def create(model, partitions) + def create(partitions) connection.transaction do with_lock_retries do partitions.each do |partition| @@ -71,6 +80,18 @@ module Gitlab end end + def detach(partitions) + connection.transaction do + with_lock_retries do + partitions.each { |p| detach_one_partition(p) } + end + end + end + + def detach_one_partition(partition) + Gitlab::AppLogger.info("Planning to detach #{partition.partition_name} for table #{partition.table}") + end + def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( klass: self.class, diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb index 9ec9ae684a5..ad122fd47fe 100644 --- a/lib/gitlab/database/partitioning/partition_monitoring.rb +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -6,7 +6,7 @@ module Gitlab class PartitionMonitoring attr_reader :models - def initialize(models = PartitionCreator.models) + def initialize(models = PartitionManager.models) @models = models end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 4402c42b136..f1aa7871245 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -79,135 +79,6 @@ module Gitlab "#{prefix}#{hashed_identifier}" end - - # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned - # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the - # same functionality at the database level by using triggers. - # - # Example: - # - # add_partitioned_foreign_key :issues, :projects - # - # Available options: - # - # :column - name of the referencing column (otherwise inferred from the referenced table name) - # :primary_key - name of the primary key in the referenced table (defaults to id) - # :on_delete - supports either :cascade for ON DELETE CASCADE or :nullify for ON DELETE SET NULL - # - def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade) - cascade_delete = extract_cascade_option(on_delete) - - update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key| - if existing_key.nil? - unless specified_key.save - raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}" - end - - current_keys << specified_key - else - Gitlab::AppLogger.warn "foreign key not added because it already exists: #{specified_key}" - current_keys - end - end - end - - # Drops a "foreign key" that references a partitioned table. This method ONLY applies to foreign keys previously - # created through the `add_partitioned_foreign_key` method. Standard database foreign keys should be managed - # through the familiar Rails helpers. - # - # Example: - # - # remove_partitioned_foreign_key :issues, :projects - # - # Available options: - # - # :column - name of the referencing column (otherwise inferred from the referenced table name) - # :primary_key - name of the primary key in the referenced table (defaults to id) - # - def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id) - update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key| - if existing_key - existing_key.delete - current_keys.delete(existing_key) - else - Gitlab::AppLogger.warn "foreign key not removed because it doesn't exist: #{specified_key}" - end - - current_keys - end - end - - private - - def fk_function_name(table) - object_name(table, 'fk_cascade_function') - end - - def fk_trigger_name(table) - object_name(table, 'fk_cascade_trigger') - end - - def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) - PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s, - to_column: to_column.to_s, cascade_delete: cascade_delete) - end - - def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil) - assert_not_in_transaction_block(scope: 'partitioned foreign key') - - from_column ||= "#{to_table.to_s.singularize}_id" - specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) - - current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a - existing_key = find_existing_key(current_keys, specified_key) - - final_keys = yield current_keys, existing_key, specified_key - - fn_name = fk_function_name(to_table) - trigger_name = fk_trigger_name(to_table) - - with_lock_retries do - drop_trigger(to_table, trigger_name, if_exists: true) - - if final_keys.empty? - drop_function(fn_name, if_exists: true) - else - create_or_replace_fk_function(fn_name, final_keys) - create_trigger(to_table, trigger_name, fn_name, fires: 'AFTER DELETE') - end - end - end - - def extract_cascade_option(on_delete) - case on_delete - when :cascade then true - when :nullify then false - else raise ArgumentError, "invalid option #{on_delete} for :on_delete" - end - end - - def find_existing_key(keys, key) - keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column } - end - - def create_or_replace_fk_function(fn_name, fk_specs) - create_trigger_function(fn_name, replace: true) do - cascade_statements = build_cascade_statements(fk_specs) - cascade_statements << 'RETURN OLD;' - - cascade_statements.join("\n") - end - end - - def build_cascade_statements(foreign_keys) - foreign_keys.map do |fks| - if fks.cascade_delete? - "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};" - else - "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};" - end - end - end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb deleted file mode 100644 index f9a90511f9b..00000000000 --- a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module PartitioningMigrationHelpers - class PartitionedForeignKey < ApplicationRecord - validates_with PartitionedForeignKeyValidator - - scope :by_referenced_table, ->(table) { where(to_table: table) } - end - end - end -end diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb deleted file mode 100644 index 089cf2b8931..00000000000 --- a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module PartitioningMigrationHelpers - class PartitionedForeignKeyValidator < ActiveModel::Validator - def validate(record) - validate_key_part(record, :from_table, :from_column) - validate_key_part(record, :to_table, :to_column) - end - - private - - def validate_key_part(record, table_field, column_field) - if !connection.table_exists?(record[table_field]) - record.errors.add(table_field, 'must be a valid table') - elsif !connection.column_exists?(record[table_field], record[column_field]) - record.errors.add(column_field, 'must be a valid column') - end - end - - def connection - ActiveRecord::Base.connection - end - end - end - end -end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index aa46b98be5d..580cab5622d 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -11,17 +11,17 @@ module Gitlab # In order to not use a possible complex time consuming query when calculating min and max values, # the start and finish can be sent specifically, start and finish should contain max and min values for PRIMARY KEY of # relation (most cases `id` column) rather than counted attribute eg: - # estimate_distinct_count(start: ::Project.with_active_services.minimum(:id), finish: ::Project.with_active_services.maximum(:id)) + # estimate_distinct_count(start: ::Project.aimed_for_deletion.minimum(:id), finish: ::Project.aimed_for_deletion.maximum(:id)) # # Grouped relations are NOT supported yet. # # @example Usage # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).execute - # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.with_active_services.service_desk_enabled.where(time_period)) + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.aimed_for_deletion.service_desk_enabled.where(time_period)) # .execute( # batch_size: 1_000, - # start: ::Project.with_active_services.service_desk_enabled.where(time_period).minimum(:id), - # finish: ::Project.with_active_services.service_desk_enabled.where(time_period).maximum(:id) + # start: ::Project.aimed_for_deletion.service_desk_enabled.where(time_period).minimum(:id), + # finish: ::Project.aimed_for_deletion.service_desk_enabled.where(time_period).maximum(:id) # ) # # @note HyperLogLog is an PROBABILISTIC algorithm that ESTIMATES distinct count of given attribute value for supplied relation diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 6e734834841..58e4e7e7924 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -7,6 +7,7 @@ module Gitlab self.table_name = 'postgres_indexes' self.primary_key = 'identifier' + self.inheritance_column = :_type_disabled has_one :bloat_estimate, class_name: 'Gitlab::Database::PostgresIndexBloatEstimate', foreign_key: :identifier has_many :reindexing_actions, class_name: 'Gitlab::Database::Reindexing::ReindexAction', foreign_key: :index_identifier @@ -17,12 +18,12 @@ module Gitlab find(identifier) end - # A 'regular' index is a non-unique index, - # that does not serve an exclusion constraint and - # is defined on a table that is not partitioned. - scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)} + # Indexes with reindexing support + scope :reindexing_support, -> { where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) } - scope :not_match, ->(regex) { where("name !~ ?", regex)} + scope :not_match, ->(regex) { where("name !~ ?", regex) } + + scope :match, ->(regex) { where("name ~* ?", regex) } scope :not_recently_reindexed, -> do recent_actions = Reindexing::ReindexAction.recent.where('index_identifier = identifier') @@ -30,10 +31,19 @@ module Gitlab where('NOT EXISTS (?)', recent_actions) end + def reset + reload # rubocop:disable Cop/ActiveRecordAssociationReload + clear_memoization(:bloat_size) + end + def bloat_size strong_memoize(:bloat_size) { bloat_estimate&.bloat_size || 0 } end + def relative_bloat_level + bloat_size / ondisk_size_bytes.to_f + end + def to_s name end diff --git a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb index 59bd24d3c37..a2e7f4befab 100644 --- a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb @@ -7,8 +7,7 @@ module Gitlab extend ActiveSupport::Concern def dump_schema_information # :nodoc: - versions = schema_migration.all_versions - Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any? + Gitlab::Database::SchemaMigrations.touch_all(self) nil end diff --git a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb index 9f664fa2137..71d2554844e 100644 --- a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb @@ -22,7 +22,7 @@ module Gitlab end def force_disconnect_timer - @force_disconnect_timer ||= ConnectionTimer.starting_now + @force_disconnect_timer ||= ::Gitlab::Database::ConnectionTimer.starting_now end end end diff --git a/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb index cf8342941c4..30060ecb34f 100644 --- a/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb +++ b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb @@ -6,9 +6,10 @@ module Gitlab module LoadSchemaVersionsMixin extend ActiveSupport::Concern - def structure_load(*args) - super(*args) - Gitlab::Database::SchemaVersionFiles.load_all + def structure_load(...) + super(...) + + Gitlab::Database::SchemaMigrations.load_all(connection) end end end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 0cfad690283..841e04ccbd1 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -6,6 +6,8 @@ module Gitlab # Number of indexes to reindex per invocation DEFAULT_INDEXES_PER_INVOCATION = 2 + SUPPORTED_TYPES = %w(btree gist).freeze + # candidate_indexes: Array of Gitlab::Database::PostgresIndex def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(how_many).each do |index| @@ -15,10 +17,8 @@ module Gitlab def self.candidate_indexes Gitlab::Database::PostgresIndex - .regular - .where('NOT expression') - .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") - .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") + .not_match("#{ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") + .reindexing_support end end end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb deleted file mode 100644 index 7e2dd55d21b..00000000000 --- a/lib/gitlab/database/reindexing/concurrent_reindex.rb +++ /dev/null @@ -1,154 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module Reindexing - class ConcurrentReindex - include Gitlab::Utils::StrongMemoize - - ReindexError = Class.new(StandardError) - - PG_IDENTIFIER_LENGTH = 63 - TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' - REPLACED_INDEX_PREFIX = 'old_reindex_' - STATEMENT_TIMEOUT = 9.hours - - # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, - # which only conflicts with DDL and vacuum. We therefore execute this with a rather - # high lock timeout and a long pause in between retries. This is an alternative to - # setting a high statement timeout, which would lead to a long running query with effects - # on e.g. vacuum. - REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 - - attr_reader :index, :logger - - def initialize(index, logger: Gitlab::AppLogger) - @index = index - @logger = logger - end - - def perform - raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique? - raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned? - raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? - raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX) - - logger.info "Starting reindex of #{index}" - - with_rebuilt_index do |replacement_index| - swap_index(replacement_index) - end - end - - private - - def with_rebuilt_index - if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) - logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}") - remove_index(index.schema, replacement_index_name) - end - - create_replacement_index_statement = index.definition - .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}") - - logger.info("creating replacement index #{replacement_index_name}") - logger.debug("replacement index definition: #{create_replacement_index_statement}") - - set_statement_timeout do - connection.execute(create_replacement_index_statement) - end - - replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) - - unless replacement_index.valid_index? - message = 'replacement index was created as INVALID' - logger.error("#{message}, cleaning up") - raise ReindexError, "failed to reindex #{index}: #{message}" - end - - # Some expression indexes (aka functional indexes) - # require additional statistics. The existing statistics - # are tightly bound to the original index. We have to - # rebuild statistics for the new index before dropping - # the original one. - rebuild_statistics if index.expression? - - yield replacement_index - ensure - begin - remove_index(index.schema, replacement_index_name) - rescue StandardError => e - logger.error(e) - end - end - - def swap_index(replacement_index) - logger.info("swapping replacement index #{replacement_index} with #{index}") - - with_lock_retries do - rename_index(index.schema, index.name, replaced_index_name) - rename_index(replacement_index.schema, replacement_index.name, index.name) - rename_index(index.schema, replaced_index_name, replacement_index.name) - end - end - - def rename_index(schema, old_index_name, new_index_name) - connection.execute(<<~SQL) - ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)} - RENAME TO #{quote_table_name(new_index_name)} - SQL - end - - def remove_index(schema, name) - logger.info("Removing index #{schema}.#{name}") - - retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( - timing_configuration: REMOVE_INDEX_RETRY_CONFIG, - klass: self.class, - logger: logger - ) - - retries.run(raise_on_exhaustion: false) do - connection.execute(<<~SQL) - DROP INDEX CONCURRENTLY - IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} - SQL - end - end - - def rebuild_statistics - logger.info("rebuilding table statistics for #{index.schema}.#{index.tablename}") - - connection.execute(<<~SQL) - ANALYZE #{quote_table_name(index.schema)}.#{quote_table_name(index.tablename)} - SQL - end - - def replacement_index_name - @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}" - end - - def replaced_index_name - @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}" - end - - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) - end - - def set_statement_timeout - execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) - yield - ensure - execute('RESET statement_timeout') - end - - delegate :execute, :quote_table_name, to: :connection - def connection - @connection ||= ActiveRecord::Base.connection - end - end - end - end -end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index d68f47b5b6c..13298f67ca9 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -41,7 +41,7 @@ module Gitlab end def perform_for(index, action) - ConcurrentReindex.new(index).perform + ReindexConcurrently.new(index).perform rescue StandardError action.state = :failed diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb index 406e70791df..2186384e7d7 100644 --- a/lib/gitlab/database/reindexing/index_selection.rb +++ b/lib/gitlab/database/reindexing/index_selection.rb @@ -6,6 +6,12 @@ module Gitlab class IndexSelection include Enumerable + # Only reindex indexes with a relative bloat level (bloat estimate / size) higher than this + MINIMUM_RELATIVE_BLOAT = 0.2 + + # Only consider indexes with a total ondisk size in this range (before reindexing) + INDEX_SIZE_RANGE = (1.gigabyte..100.gigabyte).freeze + delegate :each, to: :indexes def initialize(candidates) @@ -24,11 +30,12 @@ module Gitlab # we force a N+1 pattern here and estimate bloat on a per-index # basis. - @indexes ||= filter_candidates.sort_by(&:bloat_size).reverse - end - - def filter_candidates - candidates.not_recently_reindexed + @indexes ||= candidates + .not_recently_reindexed + .where(ondisk_size_bytes: INDEX_SIZE_RANGE) + .sort_by(&:relative_bloat_level) # forced N+1 + .reverse + .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT } end end end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb index 7e58201889f..ff465fffb74 100644 --- a/lib/gitlab/database/reindexing/reindex_action.rb +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -10,7 +10,7 @@ module Gitlab enum state: { started: 0, finished: 1, failed: 2 } # Amount of time to consider a previous reindexing *recent* - RECENT_THRESHOLD = 7.days + RECENT_THRESHOLD = 10.days scope :recent, -> { where(state: :finished).where('action_end > ?', Time.zone.now - RECENT_THRESHOLD) } diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb new file mode 100644 index 00000000000..8d9f9f5abdd --- /dev/null +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + # This is a >= PG12 reindexing strategy based on `REINDEX CONCURRENTLY` + class ReindexConcurrently + ReindexError = Class.new(StandardError) + + TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*' + STATEMENT_TIMEOUT = 9.hours + PG_MAX_INDEX_NAME_LENGTH = 63 + + # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, + # which only conflicts with DDL and vacuum. We therefore execute this with a rather + # high lock timeout and a long pause in between retries. This is an alternative to + # setting a high statement timeout, which would lead to a long running query with effects + # on e.g. vacuum. + REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 + + attr_reader :index, :logger + + def initialize(index, logger: Gitlab::AppLogger) + @index = index + @logger = logger + end + + def perform + raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name =~ /#{TEMPORARY_INDEX_PATTERN}/ + + # Expression indexes require additional statistics in `pg_statistic`: + # select * from pg_statistic where starelid = (select oid from pg_class where relname = 'some_index'); + # + # In PG12, this has been fixed in https://gitlab.com/postgres/postgres/-/commit/b17ff07aa3eb142d2cde2ea00e4a4e8f63686f96. + # Discussion happened in https://www.postgresql.org/message-id/flat/CAFcNs%2BqpFPmiHd1oTXvcPdvAHicJDA9qBUSujgAhUMJyUMb%2BSA%40mail.gmail.com + # following a GitLab.com incident that surfaced this (https://gitlab.com/gitlab-com/gl-infra/production/-/issues/2885). + # + # While this has been backpatched, we continue to disable expression indexes until further review. + raise ReindexError, 'expression indexes are currently not supported' if index.expression? + + begin + with_logging do + set_statement_timeout do + execute("REINDEX INDEX CONCURRENTLY #{quote_table_name(index.schema)}.#{quote_table_name(index.name)}") + end + end + ensure + cleanup_dangling_indexes + end + end + + private + + def with_logging + bloat_size = index.bloat_size + ondisk_size_before = index.ondisk_size_bytes + + logger.info( + message: "Starting reindex of #{index}", + index: index.identifier, + table: index.tablename, + estimated_bloat_bytes: bloat_size, + index_size_before_bytes: ondisk_size_before, + relative_bloat_level: index.relative_bloat_level + ) + + duration = Benchmark.realtime do + yield + end + + index.reset + + logger.info( + message: "Finished reindex of #{index}", + index: index.identifier, + table: index.tablename, + estimated_bloat_bytes: bloat_size, + index_size_before_bytes: ondisk_size_before, + index_size_after_bytes: index.ondisk_size_bytes, + relative_bloat_level: index.relative_bloat_level, + duration_s: duration.round(2) + ) + end + + def cleanup_dangling_indexes + Gitlab::Database::PostgresIndex.match("#{TEMPORARY_INDEX_PATTERN}$").each do |lingering_index| + # Example lingering index name: some_index_ccnew1 + + # Example prefix: 'some_index' + prefix = lingering_index.name.gsub(/#{TEMPORARY_INDEX_PATTERN}/, '') + + # Example suffix: '_ccnew1' + suffix = lingering_index.name.match(/#{TEMPORARY_INDEX_PATTERN}/)[0] + + # Only remove if the lingering index name could have been chosen + # as a result of a REINDEX operation (considering that PostgreSQL + # truncates index names to 63 chars and adds a suffix). + if index.name[0...PG_MAX_INDEX_NAME_LENGTH - suffix.length] == prefix + remove_index(lingering_index) + end + end + end + + def remove_index(index) + logger.info("Removing dangling index #{index.identifier}") + + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: logger + ) + + retries.run(raise_on_exhaustion: false) do + execute("DROP INDEX CONCURRENTLY IF EXISTS #{quote_table_name(index.schema)}.#{quote_table_name(index.name)}") + end + end + + def with_lock_retries(&block) + arguments = { klass: self.class, logger: logger } + Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) + end + + def set_statement_timeout + execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) + yield + ensure + execute('RESET statement_timeout') + end + + delegate :execute, :quote_table_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index 8f93da2b66c..c3cdcf1450d 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -30,11 +30,7 @@ module Gitlab structure.gsub!(/\n{3,}/, "\n\n") io << structure.strip - io << <<~MSG - -- schema_migrations.version information is no longer stored in this file, - -- but instead tracked in the db/schema_migrations directory - -- see https://gitlab.com/gitlab-org/gitlab/-/issues/218590 for details - MSG + io << "\n" nil end diff --git a/lib/gitlab/database/schema_migrations.rb b/lib/gitlab/database/schema_migrations.rb new file mode 100644 index 00000000000..1c49ed8d946 --- /dev/null +++ b/lib/gitlab/database/schema_migrations.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaMigrations + def self.touch_all(connection) + context = Gitlab::Database::SchemaMigrations::Context.new(connection) + + Gitlab::Database::SchemaMigrations::Migrations.new(context).touch_all + end + + def self.load_all(connection) + context = Gitlab::Database::SchemaMigrations::Context.new(connection) + + Gitlab::Database::SchemaMigrations::Migrations.new(context).load_all + end + end + end +end diff --git a/lib/gitlab/database/schema_migrations/context.rb b/lib/gitlab/database/schema_migrations/context.rb new file mode 100644 index 00000000000..bd8b9bed2c1 --- /dev/null +++ b/lib/gitlab/database/schema_migrations/context.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaMigrations + class Context + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def schema_directory + @schema_directory ||= + if ActiveRecord::Base.configurations.primary?(database_name) + File.join(db_dir, 'schema_migrations') + else + File.join(db_dir, "#{database_name}_schema_migrations") + end + end + + def versions_to_create + versions_from_database = @connection.schema_migration.all_versions + versions_from_migration_files = @connection.migration_context.migrations.map { |m| m.version.to_s } + + versions_from_database & versions_from_migration_files + end + + private + + def database_name + @database_name ||= @connection.pool.db_config.name + end + + def db_dir + @db_dir ||= Rails.application.config.paths["db"].first + end + end + end + end +end diff --git a/lib/gitlab/database/schema_migrations/migrations.rb b/lib/gitlab/database/schema_migrations/migrations.rb new file mode 100644 index 00000000000..3b16b7f1b81 --- /dev/null +++ b/lib/gitlab/database/schema_migrations/migrations.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaMigrations + class Migrations + MIGRATION_VERSION_GLOB = '20[0-9][0-9]*' + + def initialize(context) + @context = context + end + + def touch_all + return unless @context.versions_to_create.any? + + version_filepaths = version_filenames.map { |f| File.join(schema_directory, f) } + FileUtils.rm(version_filepaths) + + @context.versions_to_create.each do |version| + version_filepath = File.join(schema_directory, version) + + File.open(version_filepath, 'w') do |file| + file << Digest::SHA256.hexdigest(version) + end + end + end + + def load_all + return if version_filenames.empty? + + values = version_filenames.map { |vf| "('#{@context.connection.quote_string(vf)}')" } + + @context.connection.execute(<<~SQL) + INSERT INTO schema_migrations (version) + VALUES #{values.join(',')} + ON CONFLICT DO NOTHING + SQL + end + + private + + def schema_directory + @context.schema_directory + end + + def version_filenames + @version_filenames ||= Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory) + end + end + end + end +end diff --git a/lib/gitlab/database/schema_version_files.rb b/lib/gitlab/database/schema_version_files.rb deleted file mode 100644 index 27a942404ef..00000000000 --- a/lib/gitlab/database/schema_version_files.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class SchemaVersionFiles - SCHEMA_DIRECTORY = 'db/schema_migrations' - MIGRATION_DIRECTORIES = %w[db/migrate db/post_migrate].freeze - MIGRATION_VERSION_GLOB = '20[0-9][0-9]*' - - def self.touch_all(versions_from_database) - versions_from_migration_files = find_versions_from_migration_files - - version_filepaths = find_version_filenames.map { |f| schema_directory.join(f) } - FileUtils.rm(version_filepaths) - - versions_to_create = versions_from_database & versions_from_migration_files - versions_to_create.each do |version| - version_filepath = schema_directory.join(version) - - File.open(version_filepath, 'w') do |file| - file << Digest::SHA256.hexdigest(version) - end - end - end - - def self.load_all - version_filenames = find_version_filenames - return if version_filenames.empty? - - values = version_filenames.map { |vf| "('#{connection.quote_string(vf)}')" } - connection.execute(<<~SQL) - INSERT INTO schema_migrations (version) - VALUES #{values.join(',')} - ON CONFLICT DO NOTHING - SQL - end - - def self.schema_directory - @schema_directory ||= Rails.root.join(SCHEMA_DIRECTORY) - end - - def self.migration_directories - @migration_directories ||= MIGRATION_DIRECTORIES.map { |dir| Rails.root.join(dir) } - end - - def self.find_version_filenames - Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory) - end - - def self.find_versions_from_migration_files - migration_directories.each_with_object([]) do |directory, migration_versions| - directory_migrations = Dir.glob(MIGRATION_VERSION_GLOB, base: directory) - directory_versions = directory_migrations.map! { |m| m.split('_').first } - - migration_versions.concat(directory_versions) - end - end - - def self.connection - ActiveRecord::Base.connection - end - end - end -end diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb index d9425810405..79244120776 100644 --- a/lib/gitlab/database_importers/instance_administrators/create_group.rb +++ b/lib/gitlab/database_importers/instance_administrators/create_group.rb @@ -89,7 +89,7 @@ module Gitlab end def track_event(result) - ::Gitlab::Tracking.event("instance_administrators_group", "group_created") + ::Gitlab::Tracking.event("instance_administrators_group", "group_created", namespace: result[:group]) success(result) end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index d1ada8c723e..57d354eb907 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -75,13 +75,13 @@ module Gitlab if response # In the add_prometheus_manual_configuration method, the Prometheus - # server_address config is saved as an api_url in the PrometheusService - # model. There are validates hooks in the PrometheusService model that - # check if the project associated with the PrometheusService is the + # server_address config is saved as an api_url in the Integrations::Prometheus + # model. There are validates hooks in the Integrations::Prometheus model that + # check if the project associated with the Integrations::Prometheus is the # self_monitoring project. It checks # Gitlab::CurrentSettings.self_monitoring_project_id, which is why the # Gitlab::CurrentSettings cache needs to be expired here, so that - # PrometheusService sees the latest self_monitoring_project_id. + # Integrations::Prometheus sees the latest self_monitoring_project_id. Gitlab::CurrentSettings.expire_current_application_settings success(result) else @@ -107,10 +107,10 @@ module Gitlab return success(result) unless prometheus_enabled? return success(result) unless prometheus_server_address.present? - service = result[:project].find_or_initialize_service('prometheus') + prometheus = result[:project].find_or_initialize_integration('prometheus') - unless service.update(prometheus_service_attributes) - log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: service.errors.full_messages }) + unless prometheus.update(prometheus_integration_attributes) + log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: prometheus.errors.full_messages }) return error(_('Could not save prometheus manual configuration')) end @@ -118,7 +118,8 @@ module Gitlab end def track_event(result) - ::Gitlab::Tracking.event("self_monitoring", "project_created") + project = result[:project] + ::Gitlab::Tracking.event("self_monitoring", "project_created", project: project, namespace: project.namespace) success(result) end @@ -156,7 +157,7 @@ module Gitlab ::Gitlab::Prometheus::Internal.uri end - def prometheus_service_attributes + def prometheus_integration_attributes { api_url: internal_prometheus_server_address_uri, manual_configuration: true, diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 35581952f4a..0ba23b8ffc7 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -449,7 +449,7 @@ module Gitlab end def alternate_viewer_class - return unless viewer.class == DiffViewer::Renamed + return unless viewer.instance_of?(DiffViewer::Renamed) find_renderable_viewer_class(RICH_VIEWERS) || (DiffViewer::Text if text?) end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index f3f0f227a8c..6d04c4874c7 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -85,14 +85,14 @@ module Gitlab # No-op end + def overflow? + raw_diff_files.overflow? + end + private def empty_pagination_data - { - current_page: nil, - next_page: nil, - total_pages: nil - } + { total_pages: nil } end def diff_stats_collection diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb index 7b1d6171e82..0f8f408d326 100644 --- a/lib/gitlab/diff/file_collection/commit.rb +++ b/lib/gitlab/diff/file_collection/commit.rb @@ -10,6 +10,10 @@ module Gitlab diff_options: diff_options, diff_refs: commit.diff_refs) end + + def cache_key + ['commit', @diffable.id] + end end end end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 663bad95db7..badebabb192 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -14,6 +14,10 @@ module Gitlab def unfold_diff_lines(positions) # no-op end + + def cache_key + ['compare', @diffable.head.id, @diffable.base.id] + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index d2ca86fdfe7..692186fc323 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -6,6 +6,8 @@ module Gitlab class MergeRequestDiffBase < Base extend ::Gitlab::Utils::Override + delegate :real_size, :overflow?, :cache_key, to: :@merge_request_diff + def initialize(merge_request_diff, diff_options:) @merge_request_diff = merge_request_diff @@ -44,10 +46,6 @@ module Gitlab diff_stats_cache.clear end - def real_size - @merge_request_diff.real_size - end - private def highlight_cache @@ -58,7 +56,7 @@ module Gitlab def diff_stats_cache strong_memoize(:diff_stats_cache) do - Gitlab::Diff::StatsCache.new(cachable_key: @merge_request_diff.cache_key) + Gitlab::Diff::StatsCache.new(cachable_key: cache_key) end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index 5ff7c88970c..0a601bde612 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -21,9 +21,7 @@ module Gitlab @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options) @pagination_data = { - current_page: current_page, - next_page: next_page, - total_pages: total_pages + total_pages: @paginated_collection.blank? ? nil : relation.size } end @@ -62,24 +60,6 @@ module Gitlab @merge_request_diff.merge_request_diff_files end - def current_page - return if @paginated_collection.blank? - - batch_gradual_load? ? nil : @paginated_collection.current_page - end - - def next_page - return if @paginated_collection.blank? - - batch_gradual_load? ? nil : @paginated_collection.next_page - end - - def total_pages - return if @paginated_collection.blank? - - batch_gradual_load? ? relation.size : @paginated_collection.total_pages - end - # rubocop: disable CodeReuse/ActiveRecord def load_paginated_collection(batch_page, batch_size, diff_options) batch_page ||= DEFAULT_BATCH_PAGE @@ -87,21 +67,12 @@ module Gitlab paths = diff_options&.fetch(:paths, nil) - paginated_collection = if batch_gradual_load? - relation.offset(batch_page).limit([batch_size.to_i, DEFAULT_BATCH_SIZE].min) - else - relation.page(batch_page).per(batch_size) - end - + paginated_collection = relation.offset(batch_page).limit([batch_size.to_i, DEFAULT_BATCH_SIZE].min) paginated_collection = paginated_collection.by_paths(paths) if paths paginated_collection end # rubocop: enable CodeReuse/ActiveRecord - - def batch_gradual_load? - Feature.enabled?(:diffs_gradual_load, @merge_request_diff.project, default_enabled: true) - end end end end diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb index e3c1e549b96..8bacc781f61 100644 --- a/lib/gitlab/diff/position_tracer/line_strategy.rb +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -95,7 +95,7 @@ module Gitlab if c_line # If the line is still in D but also in C, it has turned from an # added line into an unchanged one. - new_position = new_position(cd_diff, c_line, d_line) + new_position = new_position(cd_diff, c_line, d_line, position.line_range) if valid_position?(new_position) # If the line is still in the MR, we don't treat this as outdated. { position: new_position, outdated: false } @@ -108,7 +108,7 @@ module Gitlab end else # If the line is still in D and not in C, it is still added. - { position: new_position(cd_diff, nil, d_line), outdated: false } + { position: new_position(cd_diff, nil, d_line, position.line_range), outdated: false } end else # If the line is no longer in D, it has been removed from the MR. @@ -143,7 +143,7 @@ module Gitlab { position: new_position(bd_diff, nil, d_line), outdated: true } else # If the line is still in C and not in D, it is still removed. - { position: new_position(cd_diff, c_line, nil), outdated: false } + { position: new_position(cd_diff, c_line, nil, position.line_range), outdated: false } end else # If the line is no longer in C, it has been removed outside of the MR. @@ -174,7 +174,7 @@ module Gitlab if c_line && d_line # If the line is still in C and D, it is still unchanged. - new_position = new_position(cd_diff, c_line, d_line) + new_position = new_position(cd_diff, c_line, d_line, position.line_range) if valid_position?(new_position) # If the line is still in the MR, we don't treat this as outdated. { position: new_position, outdated: false } @@ -188,7 +188,7 @@ module Gitlab # If the line is still in D but no longer in C, it has turned from # an unchanged line into an added one. # We don't treat this as outdated since the line is still in the MR. - { position: new_position(cd_diff, nil, d_line), outdated: false } + { position: new_position(cd_diff, nil, d_line, position.line_range), outdated: false } else # !d_line && (c_line || !c_line) # If the line is no longer in D, it has turned from an unchanged line # into a removed one. @@ -196,12 +196,15 @@ module Gitlab end end - def new_position(diff_file, old_line, new_line) - Position.new( + def new_position(diff_file, old_line, new_line, line_range = nil) + params = { diff_file: diff_file, old_line: old_line, - new_line: new_line - ) + new_line: new_line, + line_range: line_range + }.compact + + Position.new(**params) end def valid_position?(position) diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index e927a5641e5..b110d39818d 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -61,7 +61,8 @@ module Gitlab params: { title: mail.subject, description: message_including_reply - } + }, + spam_params: nil ).execute end diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index fd3143488b1..d508cf9360e 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -4,14 +4,6 @@ module Gitlab module Email module Handler module ReplyProcessing - private - - attr_reader :project_id, :project_slug, :project_path, :incoming_email_token - - def author - raise NotImplementedError - end - # rubocop:disable Gitlab/ModuleWithInstanceVariables def project return @project if instance_variable_defined?(:@project) @@ -27,6 +19,14 @@ module Gitlab end # rubocop:enable Gitlab/ModuleWithInstanceVariables + private + + attr_reader :project_id, :project_slug, :project_path, :incoming_email_token + + def author + raise NotImplementedError + end + def message @message ||= process_message end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 05daa08530e..84b55079cea 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -42,18 +42,10 @@ module Gitlab end end - def metrics_params - super.merge(project: project&.full_path) - end - def metrics_event :receive_email_service_desk end - private - - attr_reader :project_id, :project_path, :service_desk_key - def project strong_memoize(:project) do @project = service_desk_key ? project_from_key : super @@ -62,6 +54,10 @@ module Gitlab end end + private + + attr_reader :project_id, :project_path, :service_desk_key + def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) @@ -83,7 +79,8 @@ module Gitlab description: message_including_template, confidential: true, external_author: from_address - } + }, + spam_params: nil ).execute raise InvalidIssueError unless @issue.persisted? @@ -95,6 +92,7 @@ module Gitlab def send_thank_you_email Notify.service_desk_thank_you_email(@issue.id).deliver_later + Gitlab::Metrics::BackgroundTransaction.current&.add_event(:service_desk_thank_you_email) end def message_including_template diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index d563de6c77e..88140c67804 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -68,7 +68,7 @@ module Gitlab private def ci_link - link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/README')) + link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/index')) end def quick_start_link diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 8139a294269..242def826be 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -22,6 +22,9 @@ module Gitlab handler.execute.tap do Gitlab::Metrics::BackgroundTransaction.current&.add_event(handler.metrics_event, handler.metrics_params) end + rescue StandardError => e + Gitlab::Metrics::BackgroundTransaction.current&.add_event('email_receiver_error', error: e.class.name) + raise e end def mail_metadata @@ -33,7 +36,11 @@ module Gitlab references: Array(mail.references), delivered_to: delivered_to.map(&:value), envelope_to: envelope_to.map(&:value), - x_envelope_to: x_envelope_to.map(&:value) + x_envelope_to: x_envelope_to.map(&:value), + meta: { + client_id: "email/#{mail.from.first}", + project: handler&.project&.full_path + } } end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index e2a9192806f..e835deeea2c 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -35,7 +35,12 @@ module Gitlab # Worse in new version, no setter! Have to poke at the # instance variable - exception.value = message if message + if message.present? + exceptions.each do |exception| + exception.value = message if valid_exception?(exception) + end + end + event.extra[:grpc_debug_error_string] = debug_str if debug_str end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index fe3dd4759d6..0cf3969b490 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -62,9 +62,6 @@ module Gitlab learn_gitlab_b: { tracking_category: 'Growth::Activation::Experiment::LearnGitLabB', rollout_strategy: :user - }, - in_product_marketing_emails: { - tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' } }.freeze diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index ca9205a8f8c..2a43f0d5ca9 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -11,6 +11,7 @@ module Gitlab module Experimentation module ControllerConcern include ::Gitlab::Experimentation::GroupTypes + include Gitlab::Tracking::Helpers extend ActiveSupport::Concern included do @@ -101,10 +102,6 @@ module Gitlab private - def dnt_enabled? - Gitlab::Utils.to_boolean(request.headers['DNT']) - end - def experimentation_subject_id cookies.signed[:experimentation_subject_id] end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index a2215366bdc..d5ced2045f5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -80,7 +80,7 @@ module Gitlab def shas_eql?(sha1, sha2) return true if sha1.nil? && sha2.nil? return false if sha1.nil? || sha2.nil? - return false unless sha1.class == sha2.class + return false unless sha1.instance_of?(sha2.class) # If either of the shas is below the minimum length, we cannot be sure # that they actually refer to the same commit because of hash collision. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e38c7b516ee..70d072e8082 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -354,9 +354,13 @@ module Gitlab end end - def new_commits(newrev) + def new_commits(newrevs) wrapped_gitaly_errors do - gitaly_ref_client.list_new_commits(newrev) + if Feature.enabled?(:list_commits) + gitaly_commit_client.list_commits(Array.wrap(newrevs) + %w[--not --all]) + else + Array.wrap(newrevs).flat_map { |newrev| gitaly_ref_client.list_new_commits(newrev) } + end end end @@ -370,6 +374,20 @@ module Gitlab end end + # List blobs reachable via a set of revisions. Supports the + # pseudo-revisions `--not` and `--all`. Uses the minimum of + # GitalyClient.medium_timeout and dynamic timeout if the dynamic + # timeout is set, otherwise it'll always use the medium timeout. + def blobs(revisions, dynamic_timeout: nil) + revisions = revisions.reject { |rev| rev.blank? || rev == ::Gitlab::Git::BLANK_SHA } + + return [] if revisions.blank? + + wrapped_gitaly_errors do + gitaly_blob_client.list_blobs(revisions, limit: REV_LIST_COMMIT_LIMIT, dynamic_timeout: dynamic_timeout) + end + end + def count_commits(options) options = process_count_commits_options(options.dup) @@ -869,12 +887,6 @@ module Gitlab end end - def rebase_in_progress?(rebase_id) - wrapped_gitaly_errors do - gitaly_repository_client.rebase_in_progress?(rebase_id) - end - end - def squash(user, squash_id, start_sha:, end_sha:, author:, message:) wrapped_gitaly_errors do gitaly_operation_client.user_squash(user, squash_id, start_sha, end_sha, author, message) diff --git a/lib/gitlab/git/user.rb b/lib/gitlab/git/user.rb index 2c798844798..05ae3391040 100644 --- a/lib/gitlab/git/user.rb +++ b/lib/gitlab/git/user.rb @@ -3,10 +3,10 @@ module Gitlab module Git class User - attr_reader :username, :name, :email, :gl_id + attr_reader :username, :name, :email, :gl_id, :timezone def self.from_gitlab(gitlab_user) - new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user)) + new(gitlab_user.username, gitlab_user.name, gitlab_user.commit_email, Gitlab::GlId.gl_id(gitlab_user), gitlab_user.timezone) end def self.from_gitaly(gitaly_user) @@ -14,23 +14,30 @@ module Gitlab gitaly_user.gl_username, Gitlab::EncodingHelper.encode!(gitaly_user.name), Gitlab::EncodingHelper.encode!(gitaly_user.email), - gitaly_user.gl_id + gitaly_user.gl_id, + gitaly_user.timezone ) end - def initialize(username, name, email, gl_id) + def initialize(username, name, email, gl_id, timezone) @username = username @name = name @email = email @gl_id = gl_id + + @timezone = if Feature.enabled?(:add_timezone_to_web_operations) + timezone + else + Time.zone.tzinfo.name + end end def ==(other) - [username, name, email, gl_id] == [other.username, other.name, other.email, other.gl_id] + [username, name, email, gl_id, timezone] == [other.username, other.name, other.email, other.gl_id, other.timezone] end def to_gitaly - Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b) + Gitaly::User.new(gl_username: username, gl_id: gl_id, name: name.b, email: email.b, timezone: timezone) end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 5616b61de07..194f5da0a5c 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -54,8 +54,11 @@ module Gitlab attr_reader :repository - def self.default_ref - 'master' + # TODO remove argument when issue + # https://gitlab.com/gitlab-org/gitlab/-/issues/329190 + # is closed. + def self.default_ref(container = nil) + Gitlab::DefaultBranch.value(object: container) end # Initialize with a Gitlab::Git::Repository instance diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b2a65d9f2d8..642a77ced11 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -9,7 +9,6 @@ module Gitlab ForbiddenError = Class.new(StandardError) NotFoundError = Class.new(StandardError) TimeoutError = Class.new(StandardError) - ProjectMovedError = Class.new(NotFoundError) # Use the magic string '_any' to indicate we do not know what the # changes are. This is also what gitlab-shell does. @@ -148,11 +147,11 @@ module Gitlab raise NotFoundError, not_found_message if container.nil? check_project! if project? + add_container_moved_message! end def check_project! check_project_accessibility! - add_project_moved_message! end def check_custom_action @@ -221,12 +220,12 @@ module Gitlab error_message(:project_not_found) end - def add_project_moved_message! + def add_container_moved_message! return if redirected_path.nil? - project_moved = Checks::ProjectMoved.new(repository, user, protocol, redirected_path) + container_moved = Checks::ContainerMoved.new(repository, user, protocol, redirected_path) - project_moved.add_message + container_moved.add_message end def check_command_disabled! @@ -499,13 +498,23 @@ module Gitlab end def check_changes_size - changes_size = 0 + changes_size = + if Feature.enabled?(:git_access_batched_changes_size, project, default_enabled: :yaml) + revs = ['--not', '--all', '--not'] + revs += changes_list.map { |change| change[:newrev] } - changes_list.each do |change| - changes_size += repository.new_blobs(change[:newrev]).sum(&:size) + repository.blobs(revs).sum(&:size) + else + changes_size = 0 - check_size_against_limit(changes_size) - end + changes_list.each do |change| + changes_size += repository.new_blobs(change[:newrev]).sum(&:size) + end + + changes_size + end + + check_size_against_limit(changes_size) end def check_size_against_limit(size) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f4a89edecd1..2c26da037da 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -120,7 +120,7 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless URI(address).scheme.in?(%w(tcp unix tls)) + unless %w(tcp unix tls).include?(URI(address).scheme) raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls'" end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index e4c8dc150a5..362ecbd845d 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -19,6 +19,25 @@ module Gitlab consume_blob_response(response) end + def list_blobs(revisions, limit: 0, bytes_limit: 0, dynamic_timeout: nil) + request = Gitaly::ListBlobsRequest.new( + repository: @gitaly_repo, + revisions: Array.wrap(revisions), + limit: limit, + bytes_limit: bytes_limit + ) + + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_blobs, request, timeout: timeout) + GitalyClient::BlobsStitcher.new(GitalyClient::ListBlobsAdapter.new(response)) + end + def batch_lfs_pointers(blob_ids) return [] if blob_ids.empty? diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb index 2f6d146b5c4..6c51b4cf8c6 100644 --- a/lib/gitlab/gitaly_client/blobs_stitcher.rb +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -35,8 +35,8 @@ module Gitlab Gitlab::Git::Blob.new( id: blob_data[:oid], - mode: blob_data[:mode].to_s(8), - name: File.basename(blob_data[:path]), + mode: blob_data[:mode]&.to_s(8), + name: blob_data[:path] && File.basename(blob_data[:path]), path: blob_data[:path], size: blob_data[:size], commit_id: blob_data[:revision], diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 3d24b4d53a4..b894207f0aa 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -248,6 +248,16 @@ module Gitlab consume_commits_response(response) end + def list_commits(revisions) + request = Gitaly::ListCommitsRequest.new( + repository: @gitaly_repo, + revisions: Array.wrap(revisions) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) + consume_commits_response(response) + end + def list_commits_by_oid(oids) return [] if oids.empty? diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index fc40c23611a..300800189f1 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -49,7 +49,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.long_timeout) if response.resolution_error.present? raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error diff --git a/lib/gitlab/gitaly_client/list_blobs_adapter.rb b/lib/gitlab/gitaly_client/list_blobs_adapter.rb new file mode 100644 index 00000000000..bba668e4dc6 --- /dev/null +++ b/lib/gitlab/gitaly_client/list_blobs_adapter.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + class ListBlobsAdapter + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + @rpc_response.each do |msg| + msg.blobs.each do |blob| + yield blob + end + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 6a75096ff80..009aeaf868a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -152,23 +152,6 @@ module Gitlab ) end - def rebase_in_progress?(rebase_id) - request = Gitaly::IsRebaseInProgressRequest.new( - repository: @gitaly_repo, - rebase_id: rebase_id.to_s - ) - - response = GitalyClient.call( - @storage, - :repository_service, - :is_rebase_in_progress, - request, - timeout: GitalyClient.fast_timeout - ) - - response.in_progress - end - def squash_in_progress?(squash_id) request = Gitaly::IsSquashInProgressRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index f66dc3010ea..4cc0269673f 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -52,7 +52,7 @@ module Gitlab @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] storage['path'] = Deprecated - @hash = storage.with_indifferent_access + @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) end def gitaly_address diff --git a/lib/gitlab/github_import/importer/diff_notes_importer.rb b/lib/gitlab/github_import/importer/diff_notes_importer.rb index 966f12c5c2f..49cbc8f7a42 100644 --- a/lib/gitlab/github_import/importer/diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/diff_notes_importer.rb @@ -22,6 +22,10 @@ module Gitlab :pull_requests_comments end + def object_type + :diff_note + end + def id_for_already_imported_cache(note) note.id end diff --git a/lib/gitlab/github_import/importer/issues_importer.rb b/lib/gitlab/github_import/importer/issues_importer.rb index ac6d0666b3a..6cc1a61b332 100644 --- a/lib/gitlab/github_import/importer/issues_importer.rb +++ b/lib/gitlab/github_import/importer/issues_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportIssueWorker end + def object_type + :issue + end + def collection_method :issues end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index c74a7706117..40248ecbd31 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportLfsObjectWorker end + def object_type + :lfs_object + end + def collection_method :lfs_objects end @@ -26,6 +30,8 @@ module Gitlab lfs_objects = Projects::LfsPointers::LfsObjectDownloadListService.new(project).execute lfs_objects.each do |object| + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield object end rescue StandardError => e diff --git a/lib/gitlab/github_import/importer/notes_importer.rb b/lib/gitlab/github_import/importer/notes_importer.rb index 5aec760ea5f..ca1d7d60515 100644 --- a/lib/gitlab/github_import/importer/notes_importer.rb +++ b/lib/gitlab/github_import/importer/notes_importer.rb @@ -18,6 +18,10 @@ module Gitlab ImportNoteWorker end + def object_type + :note + end + def collection_method :issues_comments end diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 8173fdd5e3e..640914acf4d 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -43,7 +43,7 @@ module Gitlab def missing_author_note s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { - author: pull_request.merged_by.login, + author: pull_request.merged_by&.login || 'ghost', timestamp: pull_request.merged_at } end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index f476ee13392..dd5b7c93ced 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -69,8 +69,8 @@ module Gitlab author_id: author_id, note: note, system: false, - created_at: review.submitted_at, - updated_at: review.submitted_at + created_at: submitted_at, + updated_at: submitted_at }.merge(extra) end @@ -80,8 +80,8 @@ module Gitlab approval_attribues = { merge_request_id: merge_request.id, user_id: user_id, - created_at: review.submitted_at, - updated_at: review.submitted_at + created_at: submitted_at, + updated_at: submitted_at } result = ::Approval.insert( @@ -105,6 +105,10 @@ module Gitlab Note.create!(attributes) end + + def submitted_at + @submitted_at ||= (review.submitted_at || merge_request.updated_at) + end end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 28cd3f802a2..b2f099761b1 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -22,6 +22,10 @@ module Gitlab pr.number end + def object_type + :pull_request + end + def each_object_to_import super do |pr| update_repository if update_repository?(pr) diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 94472cd341e..287e0ea7f7f 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -22,6 +22,10 @@ module Gitlab :pull_requests_merged_by end + def object_type + :pull_request_merged_by + end + def id_for_already_imported_cache(merge_request) merge_request.id end @@ -30,6 +34,8 @@ module Gitlab project.merge_requests.with_state(:merged).find_each do |merge_request| next if already_imported?(merge_request) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + pull_request = client.pull_request(project.import_source, merge_request.iid) yield(pull_request) diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 809a518d13a..e389acbf877 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -29,6 +29,10 @@ module Gitlab :pull_request_reviews end + def object_type + :pull_request_review + end + def id_for_already_imported_cache(review) review.id end @@ -57,6 +61,8 @@ module Gitlab project.merge_requests.find_each do |merge_request| next if already_imported?(merge_request) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + client .pull_request_reviews(project.import_source, merge_request.iid) .each do |review| @@ -81,6 +87,8 @@ module Gitlab page.objects.each do |review| next if already_imported?(review) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + review.merge_request_id = merge_request.id yield(review) diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index e5f4dabe42d..0b1c221bbec 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -13,7 +13,7 @@ module Gitlab # author - An instance of `Gitlab::GithubImport::Representation::User` # exists - Boolean that indicates the user exists in the GitLab database. def initialize(text, author, exists = false) - @text = text + @text = text.to_s @author = author @exists = exists end diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb new file mode 100644 index 00000000000..e4835504c2d --- /dev/null +++ b/lib/gitlab/github_import/object_counter.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# Count objects fetched or imported from Github. +module Gitlab + module GithubImport + class ObjectCounter + OPERATIONS = %w[fetched imported].freeze + PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' + PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + + GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' + GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' + + CACHING = Gitlab::Cache::Import::Caching + + class << self + def increment(project, object_type, operation) + validate_operation!(operation) + + increment_project_counter(project, object_type, operation) + increment_global_counter(object_type, operation) + end + + def summary(project) + OPERATIONS.each_with_object({}) do |operation, result| + result[operation] = {} + + CACHING + .values_from_set(counter_list_key(project, operation)) + .sort + .each do |counter| + object_type = counter.split('/').last + result[operation][object_type] = CACHING.read_integer(counter) + end + end + end + + private + + # Global counters are long lived, in Prometheus, + # and it's used to report the health of the Github Importer + # in the Grafana Dashboard + # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1 + def increment_global_counter(object_type, operation) + key = GLOBAL_COUNTER_KEY % { + operation: operation, + object_type: object_type + } + description = GLOBAL_COUNTER_DESCRIPTION % { + operation: operation, + object_type: object_type.to_s.humanize + } + + Gitlab::Metrics.counter(key.to_sym, description).increment + end + + # Project counters are short lived, in Redis, + # and it's used to report how successful a project + # import was with the #summary method. + def increment_project_counter(project, object_type, operation) + counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } + + add_counter_to_list(project, operation, counter_key) + + CACHING.increment(counter_key) + end + + def add_counter_to_list(project, operation, key) + CACHING.set_add(counter_list_key(project, operation), key) + end + + def counter_list_key(project, operation) + PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation } + end + + def validate_operation!(operation) + unless operation.to_s.presence_in(OPERATIONS) + raise ArgumentError, "Operation must be #{OPERATIONS.join(' or ')}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 92f9e8a646d..4598429d568 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -103,6 +103,8 @@ module Gitlab page.objects.each do |object| next if already_imported?(object) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield object # We mark the object as imported immediately so we don't end up @@ -129,6 +131,10 @@ module Gitlab Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, id) end + def object_type + raise NotImplementedError + end + # Returns the ID to use for the cache used for checking if an object has # already been imported or not. # diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 3205259a1ed..08b3160fc4c 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -29,7 +29,7 @@ module Gitlab hash = Representation.symbolize_hash(raw_hash) hash[:author] &&= Representation::User.from_json_hash(hash[:author]) - hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone + hash[:submitted_at] = Time.parse(hash[:submitted_at]).in_time_zone if hash[:submitted_at].present? new(hash) end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 8d584415202..058cd1ebd57 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -138,7 +138,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def query_id_for_github_id(id) - User.for_github_id(id).pluck(:id).first + User.by_provider_and_extern_uid(:github, id).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index e84863deba8..0cd33b9f5b7 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -56,8 +56,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def gitlab_user_id(project, gitlab_id) - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) - (user && user.id) || project.creator_id + user_id = User.by_provider_and_extern_uid(:gitlab, gitlab_id).select(:id).first&.id + user_id || project.creator_id end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/global_id/deprecations.rb b/lib/gitlab/global_id/deprecations.rb index ac4a44e0e10..2753e2b8372 100644 --- a/lib/gitlab/global_id/deprecations.rb +++ b/lib/gitlab/global_id/deprecations.rb @@ -9,11 +9,12 @@ module Gitlab # Example: # # DEPRECATIONS = [ - # Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.0') + # Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.1') # ].freeze DEPRECATIONS = [ # This works around an accidentally released argument named as `"EEIterationID"` in 7000489db. - Deprecation.new(old_model_name: 'EEIteration', new_model_name: 'Iteration', milestone: '13.3') + Deprecation.new(old_model_name: 'EEIteration', new_model_name: 'Iteration', milestone: '13.3'), + Deprecation.new(old_model_name: 'PrometheusService', new_model_name: 'Integrations::Prometheus', milestone: '14.1') ].freeze # Maps of the DEPRECATIONS Hash for quick access. diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 14f9c7f2191..16a8b5f959e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -23,6 +23,7 @@ module Gitlab gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab.revision + gon.feature_category = Gitlab::ApplicationContext.current_context_attribute(:feature_category).presence gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path @@ -32,6 +33,7 @@ module Gitlab gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week + gon.time_display_relative = true gon.ee = Gitlab.ee? gon.dot_com = Gitlab.com? @@ -40,6 +42,7 @@ module Gitlab gon.current_username = current_user.username gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url + gon.time_display_relative = current_user.time_display_relative end # Initialize gon.features with any flags that should be diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index f1b74999897..5a9d21e7469 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -4,11 +4,42 @@ module Gitlab module Graphql module Pagination module Keyset + # https://gitlab.com/gitlab-org/gitlab/-/issues/334973 # Use the generic keyset implementation if the given ActiveRecord scope supports it. # Note: this module is temporary, at some point it will be merged with Keyset::Connection module GenericKeysetPagination extend ActiveSupport::Concern + # rubocop: disable Naming/PredicateName + # rubocop: disable CodeReuse/ActiveRecord + def has_next_page + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + strong_memoize(:generic_keyset_pagination_has_next_page) do + if before + # If `before` is specified, that points to a specific record, + # even if it's the last one. Since we're asking for `before`, + # then the specific record we're pointing to is in the + # next page + true + elsif first + case sliced_nodes + when Array + sliced_nodes.size > limit_value + else + # If we count the number of requested items plus one (`limit_value + 1`), + # then if we get `limit_value + 1` then we know there is a next page + sliced_nodes.limit(1).offset(limit_value).exists? + # replacing relation count + # relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + end + else + false + end + end + end + + # rubocop: enable CodeReuse/ActiveRecord def ordered_items raise ArgumentError, 'Relation must have a primary key' unless items.primary_key.present? diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index d05ced00a6b..afe1554aec1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -11,11 +11,9 @@ module Gitlab end def self.too_large?(size) - file_size_limit = Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + return false unless size.to_i > self.file_size_limit - return false unless size.to_i > file_size_limit - - over_highlight_size_limit.increment(source: "file size: #{file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) + over_highlight_size_limit.increment(source: "file size: #{self.file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) true end @@ -51,6 +49,16 @@ module Gitlab attr_reader :context + def self.file_size_limit + if Feature.enabled?(:one_megabyte_file_size_limit) + 1024.kilobytes + else + Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + end + end + + private_class_method :file_size_limit + def custom_language return unless @language diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index 2d1bb515058..181ce447b52 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -8,6 +8,7 @@ module Gitlab labels total_time_spent time_change + severity ].freeze def self.safe_hook_attributes @@ -51,7 +52,8 @@ module Gitlab assignee_ids: issue.assignee_ids, assignee_id: issue.assignee_ids.first, # This key is deprecated labels: issue.labels_hook_attrs, - state: issue.state + state: issue.state, + severity: issue.severity } issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 30e72b58e21..7ebbe9f1c14 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -40,26 +40,26 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 1, 'cs_CZ' => 1, - 'de' => 18, + 'de' => 17, 'en' => 100, 'eo' => 1, - 'es' => 40, + 'es' => 38, 'fil_PH' => 1, - 'fr' => 13, + 'fr' => 12, 'gl_ES' => 1, 'id_ID' => 0, 'it' => 2, - 'ja' => 44, + 'ja' => 42, 'ko' => 13, 'nl_NL' => 1, - 'pl_PL' => 3, + 'pl_PL' => 5, 'pt_BR' => 21, - 'ru' => 30, - 'tr_TR' => 17, - 'uk' => 42, - 'zh_CN' => 69, - 'zh_HK' => 3, - 'zh_TW' => 4 + 'ru' => 29, + 'tr_TR' => 16, + 'uk' => 41, + 'zh_CN' => 67, + 'zh_HK' => 2, + 'zh_TW' => 3 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d000c331b6d..a84978a2a80 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -61,7 +61,9 @@ tree: - :push_event_payload - :suggestions - merge_request_diff: - - :merge_request_diff_commits + - merge_request_diff_commits: + - :commit_author + - :committer - :merge_request_diff_files - events: - :push_event_payload @@ -201,6 +203,10 @@ excluded_attributes: - :verification_failure merge_request_diff_commits: - :merge_request_diff_id + - :commit_author_id + - :committer_id + merge_request_diff_commit_user: + - :id merge_request_diff_detail: - :merge_request_diff_id - :verification_retry_at @@ -223,6 +229,7 @@ excluded_attributes: - :promoted_to_epic_id - :blocking_issues_count - :service_desk_reply_to + - :upvotes_count merge_request: - :milestone_id - :sprint_id diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index 831e38f3034..0753625b978 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -28,6 +28,7 @@ module Gitlab def find return if epic? && group.nil? + return find_diff_commit_user if diff_commit_user? super end @@ -81,6 +82,13 @@ module Gitlab end end + def find_diff_commit_user + find_with_cache do + MergeRequest::DiffCommitUser + .find_or_create(@attributes['name'], @attributes['email']) + end + end + def label? klass == Label end @@ -101,6 +109,10 @@ module Gitlab klass == DesignManagement::Design end + def diff_commit_user? + klass == MergeRequest::DiffCommitUser + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 4678396f97e..102fcedd2fc 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -31,7 +31,9 @@ module Gitlab ci_cd_settings: 'ProjectCiCdSetting', error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting' }.freeze + metrics_setting: 'ProjectMetricsSetting', + commit_author: 'MergeRequest::DiffCommitUser', + committer: 'MergeRequest::DiffCommitUser' }.freeze BUILD_MODELS = %i[Ci::Build commit_status].freeze @@ -56,6 +58,7 @@ module Gitlab external_pull_request external_pull_requests DesignManagement::Design + MergeRequest::DiffCommitUser ].freeze def create diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 46b82240ef7..d5f924ae2bd 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -37,7 +37,7 @@ module Gitlab ActiveRecord::Base.no_touching do update_params! - BulkInsertableAssociations.with_bulk_insert(enabled: @importable.class == ::Project) do + BulkInsertableAssociations.with_bulk_insert(enabled: @importable.instance_of?(::Project)) do fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index b5a5f8fd984..8a64abb9f62 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -1,14 +1,12 @@ # frozen_string_literal: true -require 'redis' - module Gitlab module Instrumentation module RedisInterceptor APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze def call(*args, &block) - start = Time.now # must come first so that 'start' is always defined + start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request instrumentation_class.redis_cluster_validate!(args.first) @@ -17,7 +15,7 @@ module Gitlab instrumentation_class.instance_count_exception(ex) raise ex ensure - duration = Time.now - start + duration = Gitlab::Metrics::System.monotonic_time - start unless APDEX_EXCLUDE.include?(command_from_args(args)) instrumentation_class.instance_observe_duration(duration) @@ -99,7 +97,3 @@ module Gitlab end end end - -class ::Redis::Client - prepend ::Gitlab::Instrumentation::RedisInterceptor -end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index a865a6392f0..c10c4a7bedf 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -29,6 +29,7 @@ module Gitlab instrument_rack_attack(payload) instrument_cpu(payload) instrument_thread_memory_allocations(payload) + instrument_load_balancing(payload) end def instrument_gitaly(payload) @@ -104,6 +105,12 @@ module Gitlab payload.merge!(counters) if counters end + def instrument_load_balancing(payload) + load_balancing_payload = ::Gitlab::Metrics::Subscribers::LoadBalancing.load_balancing_payload + + payload.merge!(load_balancing_payload) + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 9d7254f49f7..b87c9936570 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -5,9 +5,9 @@ module Gitlab class StiType < ActiveRecord::Type::String NAMESPACED_INTEGRATIONS = Set.new(%w( Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker - Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi Packagist PipelinesEmail Pivotaltracker - Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost + MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker + Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams )).freeze def cast(value) diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb index 75d6fdc07b6..60344e4be68 100644 --- a/lib/gitlab/jira_import.rb +++ b/lib/gitlab/jira_import.rb @@ -19,10 +19,10 @@ module Gitlab return unless configuration_check - jira_service = project.jira_service + jira_integration = project.jira_integration - raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_service&.active? - raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_service&.valid_connection? + raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_integration&.active? + raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_integration&.valid_connection? end def self.jira_item_cache_key(project_id, jira_item_id, collection_type) diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 688254bf91f..2b83f0492cb 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -14,7 +14,7 @@ module Gitlab raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key @project = project - @client = project.jira_service.client + @client = project.jira_integration.client @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 767ce310b5a..f1370a40222 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -228,6 +228,14 @@ module Gitlab raise UnsupportedFormatError end + + def render_in(_view_context) + to_s + end + + def format + :json + end end class LimitedEncoder diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index a4663314b3b..86c0aa2b48d 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -25,12 +25,6 @@ module Gitlab write_secret end - def included_in_gitlab_com_rollout?(project) - return true unless ::Gitlab.com? - - Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml) - end - # Return GitLab KAS version # # @return [String] version diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index 6675903e692..842ee98e4da 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -49,14 +49,14 @@ module Gitlab end def kas_endpoint_url - Gitlab::Kas.internal_url.delete_prefix('grpc://') + Gitlab::Kas.internal_url.sub(%r{^grpc://|^grpcs://}, '') end def credentials - if Rails.env.test? || Rails.env.development? - :this_channel_is_insecure - else + if URI(Gitlab::Kas.internal_url).scheme == 'grpcs' GRPC::Core::ChannelCredentials.new + else + :this_channel_is_insecure end end diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb index f77b3e8de99..e333d3818b9 100644 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -12,7 +12,7 @@ module Gitlab # We are modeling existing kubernetes resource and don't have # control over amount of parameters. # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil) + def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil, environment_ids: []) @name = name @description = description @namespace = namespace @@ -23,6 +23,7 @@ module Gitlab @ingress = ingress @egress = egress @annotations = annotations + @environment_ids = environment_ids end # rubocop:enable Metrics/ParameterLists @@ -49,7 +50,7 @@ module Gitlab nil end - def self.from_resource(resource) + def self.from_resource(resource, environment_ids = []) return unless resource return if !resource[:metadata] || !resource[:spec] @@ -65,7 +66,8 @@ module Gitlab creation_timestamp: metadata[:creationTimestamp], selector: spec[:endpointSelector], ingress: spec[:ingress], - egress: spec[:egress] + egress: spec[:egress], + environment_ids: environment_ids ) end @@ -83,7 +85,7 @@ module Gitlab private - attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations + attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations, :environment_ids def selector @selector ||= {} diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb index c8e9b987443..e6111db5b17 100644 --- a/lib/gitlab/kubernetes/network_policy.rb +++ b/lib/gitlab/kubernetes/network_policy.rb @@ -8,7 +8,8 @@ module Gitlab KIND = 'NetworkPolicy' - def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) + # rubocop:disable Metrics/ParameterLists + def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil, environment_ids: []) @name = name @namespace = namespace @labels = labels @@ -17,7 +18,9 @@ module Gitlab @policy_types = policy_types @ingress = ingress @egress = egress + @environment_ids = environment_ids end + # rubocop:enable Metrics/ParameterLists def self.from_yaml(manifest) return unless manifest @@ -40,7 +43,7 @@ module Gitlab nil end - def self.from_resource(resource) + def self.from_resource(resource, environment_ids = []) return unless resource return if !resource[:metadata] || !resource[:spec] @@ -54,7 +57,8 @@ module Gitlab selector: spec[:podSelector], policy_types: spec[:policyTypes], ingress: spec[:ingress], - egress: spec[:egress] + egress: spec[:egress], + environment_ids: environment_ids ) end @@ -69,7 +73,7 @@ module Gitlab private - attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress + attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress, :environment_ids def selector @selector ||= {} diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb index 99517454508..de91833b734 100644 --- a/lib/gitlab/kubernetes/network_policy_common.rb +++ b/lib/gitlab/kubernetes/network_policy_common.rb @@ -16,7 +16,8 @@ module Gitlab creation_timestamp: creation_timestamp, manifest: manifest, is_autodevops: autodevops?, - is_enabled: enabled? + is_enabled: enabled?, + environment_ids: environment_ids } end diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index e85d1314eda..7ae1b195ec6 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -35,12 +35,7 @@ module Gitlab def find_by_external_uid return unless id - identities = ::Identity.arel_table - - User.select(:id) - .joins(:identities) - .find_by(identities[:provider].eq(:github).and(identities[:extern_uid].eq(id))) - .try(:id) + User.by_provider_and_extern_uid(:github, id).select(:id).first&.id end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index dce7cdb31a1..83fd74310d0 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -25,7 +25,6 @@ module Gitlab ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload) payload[:queue_duration_s] = event.payload[:queue_duration_s] if event.payload[:queue_duration_s] - payload[:response] = event.payload[:response] if event.payload[:response] payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route] payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index e800fe14cf1..4e93d6a9ece 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -21,24 +21,13 @@ module Gitlab Thread.current.respond_to?(:memory_allocations) end - # This method changes a global state - def self.ensure_feature_flag! + def self.start_thread_memory_allocations return unless available? - enabled = Feature.enabled?(:trace_memory_allocations, default_enabled: true) - return if enabled == Thread.trace_memory_allocations - MUTEX.synchronize do - # This enables or disables feature dynamically - # based on a feature flag - Thread.trace_memory_allocations = enabled + # This method changes a global state + Thread.trace_memory_allocations = true end - end - - def self.start_thread_memory_allocations - return unless available? - - ensure_feature_flag! # it will return `nil` if disabled Thread.current.memory_allocations diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 2c4793eb75f..c8591a81a05 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -7,14 +7,9 @@ module Gitlab module Metrics module Dashboard class Finder - # Dashboards that should not be part of the list of all dashboards - # displayed on the metrics dashboard page. - PREDEFINED_DASHBOARD_EXCLUSION_LIST = [ - # This dashboard is only useful in the self monitoring project. - ::Metrics::Dashboard::SelfMonitoringDashboardService, - - # This dashboard is displayed on the K8s cluster settings health page. - ::Metrics::Dashboard::ClusterDashboardService + PREDEFINED_DASHBOARD_LIST = [ + ::Metrics::Dashboard::PodDashboardService, + ::Metrics::Dashboard::SystemDashboardService ].freeze class << self @@ -90,11 +85,7 @@ module Gitlab return [self_monitoring_service] end - predefined_dashboard_services - end - - def predefined_dashboard_services - ::Metrics::Dashboard::PredefinedDashboardService.descendants - PREDEFINED_DASHBOARD_EXCLUSION_LIST + PREDEFINED_DASHBOARD_LIST end def system_service diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 7111835c85a..ff8b8bf2237 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +require 'webrick' +require 'prometheus/client/rack/exporter' + module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 36262b09b2d..4d38d9e67bf 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'webrick' -require 'prometheus/client/rack/exporter' - module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index 756e6b0641a..f378577f08e 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -require 'webrick' -require 'prometheus/client/rack/exporter' - module Gitlab module Metrics module Exporter diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 757762499a9..848b73792cb 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'prometheus/client' - module Gitlab module Metrics module Prometheus diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb index a9355eeae40..631b9209f14 100644 --- a/lib/gitlab/metrics/subscribers/action_cable.rb +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -12,6 +12,7 @@ module Gitlab TRANSMIT_SUBSCRIPTION_CONFIRMATION = :action_cable_subscription_confirmations_total TRANSMIT_SUBSCRIPTION_REJECTION = :action_cable_subscription_rejections_total BROADCAST = :action_cable_broadcasts_total + DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes def transmit_subscription_confirmation(event) confirm_subscription_counter.increment @@ -23,6 +24,14 @@ module Gitlab def transmit(event) transmit_counter.increment + + if event.payload.present? + channel = event.payload[:channel_class] + operation = operation_name_from(event.payload) + data_size = ::ActiveSupport::JSON.encode(event.payload[:data]).bytesize + + transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size) + end end def broadcast(event) @@ -31,6 +40,13 @@ module Gitlab private + # When possible tries to query operation name + def operation_name_from(payload) + data = payload.dig(:data, 'result', 'data') || {} + + data.each_key.first + end + def transmit_counter strong_memoize("transmission_counter") do ::Gitlab::Metrics.counter( @@ -66,6 +82,12 @@ module Gitlab ) end end + + def transmitted_bytes_histogram + strong_memoize("transmitted_bytes_histogram") do + ::Gitlab::Metrics.histogram(DATA_TRANSMITTED_BYTES, 'Message size, in bytes, transmitted over action cable') + end + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 9f7884e1364..a8fcad9ff9f 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -15,8 +15,8 @@ module Gitlab TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze DB_LOAD_BALANCING_COUNTERS = %i{ - db_replica_count db_replica_cached_count db_replica_wal_count - db_primary_count db_primary_cached_count db_primary_wal_count + db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count + db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count }.freeze DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze @@ -72,6 +72,14 @@ module Gitlab DB_LOAD_BALANCING_DURATIONS.each do |duration| payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end + + if Feature.enabled?(:multiple_database_metrics, default_enabled: :yaml) + ::Gitlab::SafeRequestStore[:duration_by_database]&.each do |dbname, duration_by_role| + duration_by_role.each do |db_role, duration| + payload[:"db_#{db_role}_#{dbname}_duration_s"] = duration.to_f.round(3) + end + end + end end end end @@ -83,9 +91,14 @@ module Gitlab end def increment_db_role_counters(db_role, payload) + cached = cached_query?(payload) increment("db_#{db_role}_count".to_sym) - increment("db_#{db_role}_cached_count".to_sym) if cached_query?(payload) - increment("db_#{db_role}_wal_count".to_sym) if !cached_query?(payload) && wal_command?(payload) + increment("db_#{db_role}_cached_count".to_sym) if cached + + if wal_command?(payload) + increment("db_#{db_role}_wal_count".to_sym) + increment("db_#{db_role}_wal_cached_count".to_sym) if cached + end end def observe_db_role_duration(db_role, event) @@ -93,9 +106,18 @@ module Gitlab buckets ::Gitlab::Metrics::Subscribers::ActiveRecord::SQL_DURATION_BUCKET end + return unless ::Gitlab::SafeRequestStore.active? + duration = event.duration / 1000.0 duration_key = "db_#{db_role}_duration_s".to_sym ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration + + # Per database metrics + dbname = ::Gitlab::Database.dbname(event.payload[:connection]) + ::Gitlab::SafeRequestStore[:duration_by_database] ||= {} + ::Gitlab::SafeRequestStore[:duration_by_database][dbname] ||= {} + ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] ||= 0 + ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] += duration end def ignored_query?(payload) diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb new file mode 100644 index 00000000000..333fc63ebef --- /dev/null +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + class LoadBalancing < ActiveSupport::Subscriber + attach_to :load_balancing + + PROMETHEUS_COUNTER = :gitlab_transaction_caught_up_replica_pick_count_total + LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze + + def caught_up_replica_pick(event) + return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + result = event.payload[:result] + counter_name = counter(result) + + increment(counter_name) + end + + # we want to update Prometheus counter after the controller/action are set + def web_transaction_completed(_event) + return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } + end + + def self.load_balancing_payload + return {} unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + + {}.tap do |payload| + LOG_COUNTERS.values.each do |counter| + value = Gitlab::SafeRequestStore[counter] + + payload[counter] = value.to_i if value + end + end + end + + private + + def increment(counter) + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + end + + def increment_prometheus_for_result_label(label_value) + counter_name = counter(label_value) + return unless (counter_value = Gitlab::SafeRequestStore[counter_name]) + + increment_prometheus(labels: { result: label_value }, value: counter_value.to_i) + end + + def increment_prometheus(labels:, value:) + current_transaction&.increment(PROMETHEUS_COUNTER, value, labels) do + docstring 'Caught up replica pick result' + label_keys labels.keys + end + end + + def counter(result) + LOG_COUNTERS[result] + end + + def current_transaction + ::Gitlab::Metrics::WebTransaction.current + end + end + end + end +end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index e6e7d97d296..2e54e8bfc1a 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -65,32 +65,9 @@ module Gitlab # Note: By default the order is breadth-first # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - if use_distinct? - expose_depth = hierarchy_order.present? - hierarchy_order ||= :asc - - # if hierarchy_order is given, the calculated `depth` should be present in SELECT - if expose_depth - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all).distinct - read_only(unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) - else - recursive_query = base_and_ancestors_cte(upto).apply_to(unscoped_model.all) - - if skip_ordering? - recursive_query = recursive_query.distinct - else - recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - recursive_query = unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) - recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) - end - - read_only(recursive_query) - end - else - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) - recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order - read_only(recursive_query) - end + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) + recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order + read_only(recursive_query) end # rubocop: enable CodeReuse/ActiveRecord @@ -101,27 +78,7 @@ module Gitlab # and incremented as we go down the descendant tree # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants(with_depth: false) - if use_distinct? - # Always calculate `depth`, remove it later if with_depth is false - if with_depth - base_cte = base_and_descendants_cte(with_depth: true).apply_to(unscoped_model.all).distinct - read_only(unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) - else - base_cte = base_and_descendants_cte.apply_to(unscoped_model.all) - - if skip_ordering? - base_cte = base_cte.distinct - else - base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - base_cte = unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) - base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) - end - - read_only(base_cte) - end - else - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) - end + read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) end # rubocop: enable CodeReuse/ActiveRecord @@ -158,11 +115,6 @@ module Gitlab ancestors_scope = unscoped_model.from(ancestors_table) descendants_scope = unscoped_model.from(descendants_table) - if use_distinct? - ancestors_scope = ancestors_scope.distinct - descendants_scope = descendants_scope.distinct - end - relation = unscoped_model .with .recursive(ancestors.to_arel, descendants.to_arel) @@ -177,23 +129,6 @@ module Gitlab private - # Use distinct on the Namespace queries to avoid bad planner behavior in PG11. - def use_distinct? - return unless model <= Namespace - # Global use_distinct_for_all_object_hierarchy takes precedence over use_distinct_in_object_hierarchy - return true if Feature.enabled?(:use_distinct_for_all_object_hierarchy) - return options[:use_distinct] if options.key?(:use_distinct) - - false - end - - # Skips the extra ordering when using distinct on the namespace queries - def skip_ordering? - return options[:skip_ordering] if options.key?(:skip_ordering) - - false - end - # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries # and ordering the rows based on the `depth` column to maintain the row order. # diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 0c8ec02a56b..0755af9587b 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -120,7 +120,7 @@ module Gitlab AREL_ORDER_CLASSES = { Arel::Nodes::Ascending => :asc, Arel::Nodes::Descending => :desc }.freeze ALLOWED_NULLABLE_VALUES = [:not_nullable, :nulls_first, :nulls_last].freeze - attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections + attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections, :order_direction def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false) @attribute_name = attribute_name @@ -175,7 +175,7 @@ module Gitlab private - attr_reader :reversed_order_expression, :nullable, :distinct, :order_direction + attr_reader :reversed_order_expression, :nullable, :distinct def calculate_reversed_order(order_expression) unless AREL_ORDER_CLASSES.has_key?(order_expression.class) # Arel can reverse simple orders diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index 3bc8c0bf616..c6f0014a0f4 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -4,8 +4,12 @@ module Gitlab module Pagination module Keyset class Iterator - def initialize(scope:, use_union_optimization: false) - @scope = scope + UnsupportedScopeOrder = Class.new(StandardError) + + def initialize(scope:, use_union_optimization: true) + @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) + raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @use_union_optimization = use_union_optimization end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index cef3a7b291a..19d44ee69dd 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -139,6 +139,8 @@ module Gitlab verify_incoming_values!(values) + return use_composite_row_comparison(values) if composite_row_comparison_possible? + where_values = [] reversed_column_definitions = column_definitions.reverse @@ -187,6 +189,28 @@ module Gitlab private + def composite_row_comparison_possible? + !column_definitions.one? && + column_definitions.all?(&:not_nullable?) && + column_definitions.map(&:order_direction).uniq.one? # all columns uses the same order direction + end + + # composite row comparison works with NOT NULL columns and may use only one index scan given a proper index setup + # Example: (created_at, id) > ('2012-09-18 01:40:01+00', 15) + def use_composite_row_comparison(values) + columns = Arel::Nodes::Grouping.new(column_definitions.map(&:column_expression)) + values = Arel::Nodes::Grouping.new(column_definitions.map do |column_definition| + value = values[column_definition.attribute_name] + Arel::Nodes.build_quoted(value, column_definition.column_expression) + end) + + if column_definitions.first.ascending_order? + [columns.gt(values)] + else + [columns.lt(values)] + end + end + # Adds extra columns to the SELECT clause def apply_custom_projections(scope) additional_projections = column_definitions.select(&:add_to_projections).map do |column_definition| diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 2805b12d95d..4f8a6ffb2cc 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -19,11 +19,10 @@ module Gitlab private def paginate_with_limit_optimization(relation) - # do not paginate relation if it is already paginated - pagination_data = if relation.respond_to?(:current_page) && relation.current_page == params[:page] && relation.limit_value == params[:per_page] - relation - else + pagination_data = if needs_pagination?(relation) relation.page(params[:page]).per(params[:per_page]) + else + relation end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) @@ -39,6 +38,14 @@ module Gitlab end end + def needs_pagination?(relation) + return true unless relation.respond_to?(:current_page) + return true if params[:page].present? && relation.current_page != params[:page].to_i + return true if params[:per_page].present? && relation.limit_value != params[:per_page].to_i + + false + end + def add_default_order(relation) if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/patch/global_id.rb b/lib/gitlab/patch/global_id.rb index e99f36c7dca..145a7bfe842 100644 --- a/lib/gitlab/patch/global_id.rb +++ b/lib/gitlab/patch/global_id.rb @@ -4,7 +4,7 @@ # we alter GlobalID so it will correctly find the record with its new model name. module Gitlab module Patch - module GlobalID + module GlobalId def initialize(gid, options = {}) super diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index a977040ef6f..2c44e2cea4c 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -26,7 +26,7 @@ module Gitlab private def service_prometheus_adapter - project.find_or_initialize_service('prometheus') + project.find_or_initialize_integration('prometheus') end end end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index ce9fced9465..9d954a74948 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -10,6 +10,7 @@ module Gitlab :description, :label, :merge_when_pipeline_succeeds, + :milestone, :remove_source_branch, :target, :title, diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index e62e1172b65..7fee1d0727f 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -10,11 +10,12 @@ module Gitlab @expires_in = expires_in end - def cache_key(key) + # NOTE Remove as part of #331319 + def old_cache_key(key) "#{cache_namespace}:#{key}:set" end - def new_cache_key(key) + def cache_key(key) super(key) end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index a08cea5a435..1e9f6c497c0 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -24,3 +24,6 @@ module Gitlab end end end + +# call prepend_mod to ensure JH-specific module run even though there is no corresponding EE-specific module +Gitlab::Recaptcha.prepend_mod diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index d7501fc7068..547549361be 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -24,8 +24,8 @@ module Gitlab super(text, context.merge(project: project)) end - def references(type) - refs = super(type, project, current_user) + def references(type, ids_only: false) + refs = super(type, project, current_user, ids_only: ids_only) @stateful_not_visible_counter += refs[:not_visible].count refs[:visible] @@ -41,6 +41,12 @@ module Gitlab define_method(type.to_s.pluralize) do @references[type] ||= references(type) end + + if %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) + define_method("#{type}_ids") do + @references[type] ||= references(type, ids_only: true) + end + end end def issues diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a31f574fad2..0bd2ac180c3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -118,15 +118,15 @@ module Gitlab def debian_architecture_regex # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 # But we limit to lower case - @debian_architecture_regex ||= %r{\A[a-z0-9][-a-z0-9]*\z}.freeze + @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}.freeze end def debian_distribution_regex - @debian_distribution_regex ||= %r{\A[a-z0-9][a-z0-9\.-]*\z}i.freeze + @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}i.freeze end def debian_component_regex - @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze + @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}.freeze end def helm_channel_regex diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 42b94d5cf3b..a4d1adf7671 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -13,7 +13,6 @@ module Gitlab # @returns [HasRepository, Project, String, String] def self.parse(path) repo_path = path.delete_prefix('/').delete_suffix('.git') - redirected_path = nil # Detect the repo type based on the path, the first one tried is the project # type, which does not have a suffix. @@ -26,9 +25,11 @@ module Gitlab # Removing the suffix (.wiki, .design, ...) from the project path full_path = repo_path.chomp(type.path_suffix) - container, project, redirected_path = find_container(type, full_path) + container, project = find_container(type, full_path) + next unless container - return [container, project, type, redirected_path] if container + redirected_path = repo_path if redirected?(container, repo_path) + return [container, project, type, redirected_path] end # When a project did not exist, the parsed repo_type would be empty. @@ -40,32 +41,28 @@ module Gitlab # Returns an array containing: # - The repository container # - The related project (if available) - # - The original container path (if redirected) # # @returns [HasRepository, Project, String] def self.find_container(type, full_path) - return [nil, nil, nil] if full_path.blank? + return [nil, nil] if full_path.blank? if type.snippet? - snippet, redirected_path = find_snippet(full_path) + snippet = find_snippet(full_path) - [snippet, snippet&.project, redirected_path] + [snippet, snippet&.project] elsif type.wiki? - wiki, redirected_path = find_wiki(full_path) + wiki = find_wiki(full_path) - [wiki, wiki.try(:project), redirected_path] + [wiki, wiki.try(:project)] else - project, redirected_path = find_project(full_path) + project = find_project(full_path) - [project, project, redirected_path] + [project, project] end end def self.find_project(project_path) - project = Project.find_by_full_path(project_path, follow_redirects: true) - redirected_path = project_path if redirected?(project, project_path) - - [project, redirected_path] + Project.find_by_full_path(project_path, follow_redirects: true) end def self.redirected?(container, container_path) @@ -77,11 +74,11 @@ module Gitlab # - h5bp/html5-boilerplate/snippets/53 def self.find_snippet(snippet_path) snippet_id, project_path = extract_snippet_info(snippet_path) - return [nil, nil] unless snippet_id + return unless snippet_id - project, redirected_path = find_project(project_path) if project_path + project = find_project(project_path) if project_path - [Snippet.find_by_id_and_project(id: snippet_id, project: project), redirected_path] + Snippet.find_by_id_and_project(id: snippet_id, project: project) end # Wiki path can be either: @@ -93,10 +90,9 @@ module Gitlab # - group/subgroup def self.find_wiki(container_path) container = Routable.find_by_full_path(container_path, follow_redirects: true) - redirected_path = container_path if redirected?(container, container_path) # In CE, Group#wiki is not available so this will return nil for a group path. - [container&.try(:wiki), redirected_path] + container&.try(:wiki) end def self.extract_snippet_info(snippet_path) diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index a20e9845fe6..7de53c4b3ff 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -13,12 +13,12 @@ module Gitlab @expires_in = expires_in end - def cache_key(type) + # NOTE Remove as part of #331319 + def old_cache_key(type) "#{type}:#{namespace}:set" end - # NOTE Remove as part of #331319 - def new_cache_key(type) + def cache_key(type) super("#{type}:#{namespace}") end diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb index 2ab38147462..f8e5cf727ac 100644 --- a/lib/gitlab/search/sort_options.rb +++ b/lib/gitlab/search/sort_options.rb @@ -15,6 +15,10 @@ module Gitlab :updated_at_asc when %w[updated_at desc], [nil, 'updated_desc'] :updated_at_desc + when %w[popularity asc], [nil, 'popularity_asc'] + :popularity_asc + when %w[popularity desc], [nil, 'popularity_desc'] + :popularity_desc else :unknown end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 678c0b396ef..e6851af8264 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,6 +7,11 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 + SCOPE_ONLY_SORT = { + popularity_asc: %w[issues], + popularity_desc: %w[issues] + }.freeze + attr_reader :current_user, :query, :order_by, :sort, :filters # Limit search results by passed projects @@ -128,20 +133,29 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def apply_sort(scope) + def apply_sort(results, scope: nil) # Due to different uses of sort param we prefer order_by when # present - case ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) + sort_by = ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) + + # Reset sort to default if the chosen one is not supported by scope + sort_by = nil if SCOPE_ONLY_SORT[sort_by] && !SCOPE_ONLY_SORT[sort_by].include?(scope) + + case sort_by when :created_at_asc - scope.reorder('created_at ASC') + results.reorder('created_at ASC') when :created_at_desc - scope.reorder('created_at DESC') + results.reorder('created_at DESC') when :updated_at_asc - scope.reorder('updated_at ASC') + results.reorder('updated_at ASC') when :updated_at_desc - scope.reorder('updated_at DESC') + results.reorder('updated_at DESC') + when :popularity_asc + results.reorder('upvotes_count ASC') + when :popularity_desc + results.reorder('upvotes_count DESC') else - scope.reorder('created_at DESC') + results.reorder('created_at DESC') end end # rubocop: enable CodeReuse/ActiveRecord @@ -157,7 +171,7 @@ module Gitlab issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - apply_sort(issues) + apply_sort(issues, scope: 'issues') end # rubocop: disable CodeReuse/ActiveRecord @@ -177,7 +191,7 @@ module Gitlab merge_requests = merge_requests.in_projects(project_ids_relation) end - apply_sort(merge_requests) + apply_sort(merge_requests, scope: 'merge_requests') end def default_scope diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 30cd63e80c0..9fc7a44ec99 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -10,12 +10,12 @@ module Gitlab @expires_in = expires_in end - def cache_key(key) + # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 + def old_cache_key(key) "#{key}:set" end - # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 - def new_cache_key(key) + def cache_key(key) "#{cache_namespace}:#{key}:set" end @@ -25,7 +25,7 @@ module Gitlab with do |redis| keys_to_expire = keys.map { |key| cache_key(key) } - keys_to_expire += keys.map { |key| new_cache_key(key) } # NOTE Remove as part of #331319 + keys_to_expire += keys.map { |key| old_cache_key(key) } # NOTE Remove as part of #331319 Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.unlink(*keys_to_expire) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 16a0619daf6..bd6b80530c3 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -103,6 +103,21 @@ module Gitlab queues_for_sidekiq_queues_yml != config_queues end + # Returns a hash of worker class name => mapped queue name + def worker_queue_mappings + workers + .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } + .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } + end + + # Like worker_queue_mappings, but only for the queues running in + # the current Sidekiq process + def current_worker_queue_mappings + worker_queue_mappings + .select { |worker, queue| Sidekiq.options[:queues].include?(queue) } + .to_h + end + private def find_workers(root, ee:) diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 32194c4926e..842e53b2ffb 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -68,7 +68,7 @@ module Gitlab message = base_message(payload) - payload['database_chosen'] = job[:database_chosen] if job[:database_chosen] + payload['load_balancing_strategy'] = job['load_balancing_strategy'] if job['load_balancing_strategy'] if job_exception payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 30741f29563..3422cb47516 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -12,7 +12,13 @@ module Gitlab # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server chain.add ::Gitlab::SidekiqMiddleware::Monitor - chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics + + if metrics + chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics + + ::Gitlab::SidekiqMiddleware::ServerMetrics.initialize_process_metrics + end + chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb index 6bc08a97c07..e3cc7b28c41 100644 --- a/lib/gitlab/sidekiq_middleware/client_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -15,6 +15,7 @@ module Gitlab # worker_class can either be the string or class of the worker being enqueued. worker_class = worker_class.safe_constantize if worker_class.respond_to?(:safe_constantize) labels = create_labels(worker_class, queue, job) + labels[:scheduling] = job.key?('at') ? 'delayed' : 'immediate' @metrics.fetch(ENQUEUED).increment(labels, 1) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 4cf540ce3b8..c1dc616cbb2 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -133,11 +133,7 @@ module Gitlab end def idempotency_string - # TODO: dump the argument's JSON using `Sidekiq.dump_json` instead - # this should be done in the next release so all jobs are written - # with their idempotency key. - # see https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1090 - "#{worker_class_name}:#{arguments.join('-')}" + "#{worker_class_name}:#{Sidekiq.dump_json(arguments)}" end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 6d130957f36..2d9767e0266 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -9,10 +9,50 @@ module Gitlab # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze - def initialize - @metrics = init_metrics + class << self + include ::Gitlab::SidekiqMiddleware::MetricsHelper + + def metrics + { + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), + sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), + sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) + } + end + + def initialize_process_metrics + metrics = self.metrics + + metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + + return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize, default_enabled: :yaml) - @metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + ::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue| + worker_class = worker.safe_constantize + + next unless worker_class + + base_labels = create_labels(worker_class, queue, {}) + + %w[done fail].each do |status| + metrics[:sidekiq_jobs_completion_seconds].get(base_labels.merge(job_status: status)) + end + end + end + end + + def initialize + @metrics = self.class.metrics if ::Gitlab::Database::LoadBalancing.enable? @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing') @@ -74,10 +114,10 @@ module Gitlab @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) - if ::Gitlab::Database::LoadBalancing.enable? && job[:database_chosen] + with_load_balancing_settings(job) do |settings| load_balancing_labels = { - database_chosen: job[:database_chosen], - data_consistency: job[:data_consistency] + load_balancing_strategy: settings['load_balancing_strategy'], + data_consistency: settings['worker_data_consistency'] } @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) @@ -85,26 +125,17 @@ module Gitlab end end - def init_metrics - { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), - sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), - sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) - } - end - private + def with_load_balancing_settings(job) + return unless ::Gitlab::Database::LoadBalancing.enable? + + keys = %w[load_balancing_strategy worker_data_consistency] + return unless keys.all? { |k| job.key?(k) } + + yield job.slice(*keys) + end + def get_thread_cputime defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index d86f1609f14..b37eeb8bad1 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -99,6 +99,10 @@ module Gitlab return job_args unless compress_mode? return job_args if job_args.bytesize < @compression_threshold + # When a job was scheduled in the future, it runs through the middleware + # twice. Once on scheduling and once on queueing. No need to compress twice. + return job_args if ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compressed?(@job) + ::Gitlab::SidekiqMiddleware::SizeLimiter::Compressor.compress(@job, job_args) end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/client.rb b/lib/gitlab/sidekiq_middleware/worker_context/client.rb index 0eb52179db2..1a899b27ea3 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/client.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/client.rb @@ -15,7 +15,12 @@ module Gitlab context_for_args = worker_class.context_for_arguments(job['args']) - wrap_in_optional_context(context_for_args, &block) + wrap_in_optional_context(context_for_args) do + # This should be inside the context for the arguments so + # that we don't override the feature category on the worker + # with the one from the caller. + Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s, &block) + end end end end diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb index 4b71dfc0c1b..eb3a8e3d497 100644 --- a/lib/gitlab/sidekiq_queue.rb +++ b/lib/gitlab/sidekiq_queue.rb @@ -14,7 +14,7 @@ module Gitlab end def drop_jobs!(search_metadata, timeout:) - start_time = Gitlab::Metrics::System.monotonic_time + start_time = monotonic_time completed = true deleted_jobs = 0 @@ -62,7 +62,11 @@ module Gitlab end def timeout_exceeded?(start_time, timeout) - (Gitlab::Metrics::System.monotonic_time - start_time) > timeout + (monotonic_time - start_time) > timeout + end + + def monotonic_time + Gitlab::Metrics::System.monotonic_time end end end diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 99a056c97fc..fab016d2e1b 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -33,7 +33,7 @@ module Gitlab private def create_issue(title:, description:) - Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }).execute + Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute end def presenter(issue) diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 6afc21be4e0..df6d3eb7d0a 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -27,21 +27,18 @@ module Gitlab # connect with Spamcheck @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') - creds = + @creds = if Rails.env.development? || Rails.env.test? :this_channel_is_insecure else GRPC::Core::ChannelCredentials.new end - - @stub = ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, creds, - timeout: DEFAULT_TIMEOUT_SECS) end def issue_spam?(spam_issue:, user:, context: {}) issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) - response = @stub.check_for_spam_issue(issue, + response = grpc_client.check_for_spam_issue(issue, metadata: { 'authorization' => Gitlab::CurrentSettings.spam_check_api_key }) verdict = convert_verdict_to_gitlab_constant(response.verdict) @@ -100,6 +97,16 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: ar_timestamp.to_time.to_i, nanos: ar_timestamp.to_time.nsec) end + + def grpc_client + @grpc_client ||= ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, @creds, + interceptors: interceptors, + timeout: DEFAULT_TIMEOUT_SECS) + end + + def interceptors + [Labkit::Correlation::GRPC::ClientInterceptor.instance] + end end end end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index e302865c897..da925f0f83a 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -92,4 +92,4 @@ module Gitlab end end -Gitlab::Template::GitlabCiYmlTemplate.prepend_mod_with('Gitlab::Template::GitlabCiYmlTemplate') +Gitlab::Template::GitlabCiYmlTemplate.prepend_mod diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/template_parser/ast.rb index 2c787d396f5..89318ee0d68 100644 --- a/lib/gitlab/changelog/ast.rb +++ b/lib/gitlab/template_parser/ast.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module Changelog + module TemplateParser # AST nodes to evaluate when rendering a template. # # Evaluating an AST is done by walking over the nodes and calling diff --git a/lib/gitlab/template_parser/error.rb b/lib/gitlab/template_parser/error.rb new file mode 100644 index 00000000000..1dcde448749 --- /dev/null +++ b/lib/gitlab/template_parser/error.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module TemplateParser + # An error raised when a template couldn't be rendered. + Error = Class.new(StandardError) + end +end diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/template_parser/eval_state.rb index a0439df60cf..7cf2ab21f50 100644 --- a/lib/gitlab/changelog/eval_state.rb +++ b/lib/gitlab/template_parser/eval_state.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - module Changelog + module TemplateParser # A class for tracking state when evaluating a template class EvalState MAX_LOOPS = 4 diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/template_parser/parser.rb index fac6fc19148..157339414c4 100644 --- a/lib/gitlab/changelog/parser.rb +++ b/lib/gitlab/template_parser/parser.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true module Gitlab - module Changelog - # A parser for the template syntax used for generating changelogs. + module TemplateParser + # A parser for a simple template syntax, used for example to generate changelogs. # # As a quick primer on the template syntax, a basic template looks like # this: @@ -166,7 +166,7 @@ module Gitlab def parse_and_transform(input) AST::Transformer.new.apply(parse(input)) rescue Parslet::ParseFailed => ex - # We raise a custom error so it's easier to catch different changelog + # We raise a custom error so it's easier to catch different parser # related errors. In addition, this ensures the caller of this method # doesn't depend on a Parslet specific error class. raise Error, "Failed to parse the template: #{ex.message}" diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index e548532e061..07a53b0892b 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -13,12 +13,13 @@ module Gitlab return unless enabled? tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + increment_total_events_counter end private def enabled? - Gitlab::CurrentSettings.snowplow_enabled? + Gitlab::Tracking.enabled? end def tracker @@ -33,9 +34,46 @@ module Gitlab def emitter SnowplowTracker::AsyncEmitter.new( Gitlab::CurrentSettings.snowplow_collector_hostname, - protocol: 'https' + protocol: 'https', + on_success: method(:increment_successful_events_emissions), + on_failure: method(:failure_callback) ) end + + def failure_callback(success_count, failures) + increment_successful_events_emissions(success_count) + increment_failed_events_emissions(failures.size) + log_failures(failures) + end + + def increment_failed_events_emissions(value) + Gitlab::Metrics.counter( + :gitlab_snowplow_failed_events_total, + 'Number of failed Snowplow events emissions' + ).increment({}, value.to_i) + end + + def increment_successful_events_emissions(value) + Gitlab::Metrics.counter( + :gitlab_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions' + ).increment({}, value.to_i) + end + + def increment_total_events_counter + Gitlab::Metrics.counter( + :gitlab_snowplow_events_total, + 'Number of Snowplow events' + ).increment + end + + def log_failures(failures) + hostname = Gitlab::CurrentSettings.snowplow_collector_hostname + + failures.each do |failure| + Gitlab::AppLogger.error("#{failure["se_ca"]} #{failure["se_ac"]} failed to be reported to collector at #{hostname}") + end + end end end end diff --git a/lib/gitlab/tracking/helpers.rb b/lib/gitlab/tracking/helpers.rb new file mode 100644 index 00000000000..bf3cefb736c --- /dev/null +++ b/lib/gitlab/tracking/helpers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Helpers + def dnt_enabled? + Gitlab::Utils.to_boolean(request.headers['DNT']) + end + + def trackable_html_request? + request.format.html? && !dnt_enabled? + end + end + end +end diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index c2e5d467dbb..bfe674b945e 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -51,6 +51,10 @@ module Gitlab "Tiers:#{format(:tier, object[:tier])}" end + def render_data_category(object) + "Data Category: `#{object[:data_category]}`" + end + def format(key, value) Gitlab::Usage::Docs::ValueFormatter.format(key, value) end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 8911ac2ed1a..83a3a5b6698 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -38,6 +38,9 @@ = render_yaml_link(object.yaml_path) \ = render_owner(object.attributes) + - if object.attributes[:data_category].present? + \ + = render_data_category(object.attributes) \ = render_status(object.attributes) \ diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb new file mode 100644 index 00000000000..dd1f9948815 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CollectedDataCategoriesMetric < GenericMetric + def value + ::ServicePing::PermitDataCategoriesService.new.execute + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 69a288e5b6e..7b3a545185b 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -16,14 +16,20 @@ module Gitlab # end class << self def start(&block) + return @metric_start&.call unless block_given? + @metric_start = block end def finish(&block) + return @metric_finish&.call unless block_given? + @metric_finish = block end def relation(&block) + return @metric_relation&.call unless block_given? + @metric_relation = block end @@ -32,15 +38,21 @@ module Gitlab @column = column end - attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column + def cache_start_and_finish_as(cache_key) + @cache_key = cache_key + end + + attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column, :cache_key end def value + start, finish = get_or_cache_batch_ids + method(self.class.metric_operation) .call(relation, self.class.column, - start: self.class.metric_start&.call, - finish: self.class.metric_finish&.call) + start: start, + finish: finish) end def to_sql @@ -73,6 +85,22 @@ module Gitlab raise "Unknown time frame: #{time_frame} for DatabaseMetric" end end + + def get_or_cache_batch_ids + return [self.class.start, self.class.finish] unless self.class.cache_key.present? + + key_name = "metric_instrumentation/#{self.class.cache_key}" + + start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do + self.class.start + end + + finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do + self.class.finish + end + + [start, finish] + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 415a5bff261..aabc706901e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -256,7 +256,8 @@ module Gitlab settings: { ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, operating_system: alt_usage_data(fallback: nil) { operating_system }, - gitaly_apdex: alt_usage_data { gitaly_apdex } + gitaly_apdex: alt_usage_data { gitaly_apdex }, + collected_data_categories: alt_usage_data(fallback: []) { Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric.new(time_frame: 'none').value } } } end @@ -403,7 +404,7 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: - Integration.available_services_names(include_dev: false).each_with_object({}) do |name, response| + Integration.available_integration_names(include_dev: false).each_with_object({}) do |name, response| type = Integration.integration_name_to_type(name) response[:"projects_#{name}_active"] = count(Integration.active.where.not(project: nil).where(type: type)) @@ -426,9 +427,9 @@ module Gitlab projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - jira_service_data_hash = jira_service_data - results[:projects_jira_server_active] = jira_service_data_hash[:projects_jira_server_active] - results[:projects_jira_cloud_active] = jira_service_data_hash[:projects_jira_cloud_active] + jira_integration_data_hash = jira_integration_data + results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active] + results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] results rescue ActiveRecord::StatementInvalid @@ -650,9 +651,9 @@ module Gitlab todos: distinct_count(::Todo.where(time_period), :author_id), service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), service_desk_issues: count(::Issue.service_desk.where(time_period)), - projects_jira_active: distinct_count(::Project.with_active_jira_services.where(time_period), :creator_id), - projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_cloud.where(time_period), :creator_id), - projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_services.with_jira_dvcs_server.where(time_period), :creator_id) + projects_jira_active: distinct_count(::Project.with_active_jira_integrations.where(time_period), :creator_id), + projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_cloud.where(time_period), :creator_id), + projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_server.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index c72f487a442..21d637e7152 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -22,6 +22,7 @@ - i_package_golang_pull_package - i_package_golang_push_package - i_package_helm_pull_package +- i_package_helm_push_package - i_package_maven_delete_package - i_package_maven_pull_package - i_package_maven_push_package @@ -31,14 +32,24 @@ - i_package_nuget_delete_package - i_package_nuget_pull_package - i_package_nuget_push_package +- i_package_nuget_pull_symbol_package +- i_package_nuget_push_symbol_package - i_package_pull_package - i_package_pull_package_by_deploy_token - i_package_pull_package_by_guest - i_package_pull_package_by_user +- i_package_pull_symbol_package +- i_package_pull_symbol_package_by_deploy_token +- i_package_pull_symbol_package_by_guest +- i_package_pull_symbol_package_by_user - i_package_push_package - i_package_push_package_by_deploy_token - i_package_push_package_by_guest - i_package_push_package_by_user +- i_package_push_symbol_package +- i_package_push_symbol_package_by_deploy_token +- i_package_push_symbol_package_by_guest +- i_package_push_symbol_package_by_user - i_package_pypi_delete_package - i_package_pypi_pull_package - i_package_pypi_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 2a231f8fce0..597df9936ea 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -117,7 +117,7 @@ module Gitlab private def track(values, event_name, context: '', time: Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled? + return unless usage_ping_enabled? event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? @@ -131,6 +131,10 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end + def usage_ping_enabled? + Gitlab::CurrentSettings.usage_ping_enabled? + end + # The array of valid context on which we allow tracking def valid_context_list Plan.all_plans diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index f2e45a52434..fe1eb090fa4 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -369,3 +369,8 @@ category: testing aggregation: weekly feature_flag: users_expanding_widgets_usage_data +# Container Security - Network Policies +- name: clusters_using_network_policies_ui + redis_slot: network_policies + category: network_policies + aggregation: weekly diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index bc72a96a468..44d5baa42f6 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -31,7 +31,7 @@ module Gitlab def minimum_id(model, column = nil) end - def jira_service_data + def jira_integration_data { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index da01b68e8fc..63e6cf03d1f 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -48,7 +48,7 @@ module Gitlab end end - def jira_service_data + def jira_integration_data { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 77e0e2ca96c..0b1acaf7dd8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -169,6 +169,16 @@ module Gitlab end end + def deep_symbolized_access(data) + if data.is_a?(Array) + data.map(&method(:deep_symbolized_access)) + elsif data.is_a?(Hash) + data.deep_symbolize_keys + else + data + end + end + def string_to_ip_object(str) return unless str diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index 620d71a7814..ab5d18e9c8a 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -6,7 +6,7 @@ module Gitlab module Utils module SanitizeNodeLink UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze - ATTRS_TO_SANITIZE = %w(href src data-src).freeze + ATTRS_TO_SANITIZE = %w(href src data-src data-canonical-src).freeze def remove_unsafe_links(env, remove_invalid_links: true) node = env[:node] diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 4ea5b5a87de..faa524d171c 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -217,7 +217,7 @@ module Gitlab end # rubocop: disable UsageData/LargeTable: - def jira_service_data + def jira_integration_data data = { projects_jira_server_active: 0, projects_jira_cloud_active: 0 diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 7fbf01f3768..3dd4e5e27d4 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -112,7 +112,6 @@ module ObjectStorage end def use_workhorse_s3_client? - return false unless Feature.enabled?(:use_workhorse_s3_client, default_enabled: true) return false unless config.use_iam_profile? || config.consolidated_settings? # The Golang AWS SDK does not support V2 signatures return false unless credentials.fetch(:aws_signature_version, 4).to_i >= 4 diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb index ea2db2aa5fe..3eb0245f8a2 100644 --- a/lib/quality/seeders/issues.rb +++ b/lib/quality/seeders/issues.rb @@ -30,7 +30,7 @@ module Quality labels: labels.join(',') } params[:closed_at] = params[:created_at] + rand(35).days if params[:state] == 'closed' - issue = ::Issues::CreateService.new(project: project, current_user: team.sample, params: params).execute + issue = ::Issues::CreateService.new(project: project, current_user: team.sample, params: params, spam_params: nil).execute if issue.persisted? created_issues_count += 1 diff --git a/lib/security/ci_configuration/base_build_action.rb b/lib/security/ci_configuration/base_build_action.rb index e7a1b4770b9..880bfa6d61d 100644 --- a/lib/security/ci_configuration/base_build_action.rb +++ b/lib/security/ci_configuration/base_build_action.rb @@ -41,6 +41,7 @@ module Security # You can override the included template(s) by including variable overrides # SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings # Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings + # Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings # Note that environment variables can be set in several places # See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence YAML diff --git a/lib/security/ci_configuration/dependency_scanning_build_action.rb b/lib/security/ci_configuration/dependency_scanning_build_action.rb new file mode 100644 index 00000000000..3ee4ce4da62 --- /dev/null +++ b/lib/security/ci_configuration/dependency_scanning_build_action.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Security + module CiConfiguration + class DependencyScanningBuildAction < BaseBuildAction + private + + def update_existing_content! + @existing_gitlab_ci_content['include'] = generate_includes + end + + def template + return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled + + 'Security/Dependency-Scanning.gitlab-ci.yml' + end + end + end +end diff --git a/lib/serializers/symbolized_json.rb b/lib/serializers/symbolized_json.rb new file mode 100644 index 00000000000..78192ce3132 --- /dev/null +++ b/lib/serializers/symbolized_json.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Serializers + # Make the resulting hash have deep symbolized keys + class SymbolizedJson + class << self + def dump(obj) + obj + end + + def load(data) + return if data.nil? + + Gitlab::Utils.deep_symbolized_access(data) + end + end + end +end diff --git a/lib/sidebars/groups/context.rb b/lib/sidebars/groups/context.rb new file mode 100644 index 00000000000..6e0c6c1a2db --- /dev/null +++ b/lib/sidebars/groups/context.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + class Context < ::Sidebars::Context + def initialize(current_user:, container:, **args) + super(current_user: current_user, container: container, group: container, **args) + end + end + end +end diff --git a/lib/sidebars/groups/menus/scope_menu.rb b/lib/sidebars/groups/menus/scope_menu.rb new file mode 100644 index 00000000000..02c359e3c99 --- /dev/null +++ b/lib/sidebars/groups/menus/scope_menu.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class ScopeMenu < ::Sidebars::Menu + override :link + def link + group_path(context.group) + end + + override :title + def title + context.group.name + end + + override :active_routes + def active_routes + { path: %w[groups#show groups#details] } + end + + override :extra_nav_link_html_options + def extra_nav_link_html_options + { class: 'context-header' } + end + + override :render? + def render? + true + end + end + end + end +end diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb new file mode 100644 index 00000000000..fe669bf0b29 --- /dev/null +++ b/lib/sidebars/groups/panel.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + class Panel < ::Sidebars::Panel + override :configure_menus + def configure_menus + set_scope_menu(Sidebars::Groups::Menus::ScopeMenu.new(context)) + end + + override :render_raw_menus_partial + def render_raw_menus_partial + 'layouts/nav/sidebar/group_menus' + end + + override :aria_label + def aria_label + context.group.subgroup? ? _('Subgroup navigation') : _('Group navigation') + end + end + end +end diff --git a/lib/sidebars/projects/menus/analytics_menu.rb b/lib/sidebars/projects/menus/analytics_menu.rb index 660965005c3..9f366a19e90 100644 --- a/lib/sidebars/projects/menus/analytics_menu.rb +++ b/lib/sidebars/projects/menus/analytics_menu.rb @@ -79,7 +79,7 @@ module Sidebars end ::Sidebars::MenuItem.new( - title: _('Value Stream'), + title: _('Value stream'), link: project_cycle_analytics_path(context.project), container_html_options: { class: 'shortcuts-project-cycle-analytics' }, active_routes: { path: 'cycle_analytics#show' }, diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index f3d13e12258..fa6482562e8 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -6,8 +6,6 @@ module Sidebars class DeploymentsMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - add_item(feature_flags_menu_item) add_item(environments_menu_item) add_item(releases_menu_item) diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index 8cf7abc613c..aad1ce60d0e 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -6,7 +6,6 @@ module Sidebars class InfrastructureMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) return false unless context.project.feature_available?(:operations, context.current_user) add_item(kubernetes_menu_item) @@ -18,7 +17,7 @@ module Sidebars override :link def link - project_clusters_path(context.project) + renderable_items.first.link end override :extra_container_html_options diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb index 79603803b8f..fd57f21db88 100644 --- a/lib/sidebars/projects/menus/issues_menu.rb +++ b/lib/sidebars/projects/menus/issues_menu.rb @@ -12,7 +12,6 @@ module Sidebars add_item(list_menu_item) add_item(boards_menu_item) - add_item(labels_menu_item) add_item(service_desk_menu_item) add_item(milestones_menu_item) @@ -97,19 +96,6 @@ module Sidebars ) end - def labels_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - return ::Sidebars::NilMenuItem.new(item_id: :labels) - end - - ::Sidebars::MenuItem.new( - title: _('Labels'), - link: project_labels_path(context.project), - active_routes: { controller: :labels }, - item_id: :labels - ) - end - def service_desk_menu_item ::Sidebars::MenuItem.new( title: _('Service Desk'), diff --git a/lib/sidebars/projects/menus/labels_menu.rb b/lib/sidebars/projects/menus/labels_menu.rb deleted file mode 100644 index 7cb28ababdb..00000000000 --- a/lib/sidebars/projects/menus/labels_menu.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Sidebars - module Projects - module Menus - class LabelsMenu < ::Sidebars::Menu - override :link - def link - project_labels_path(context.project) - end - - override :extra_container_html_options - def extra_container_html_options - { - class: 'shortcuts-labels' - } - end - - override :title - def title - _('Labels') - end - - override :title_html_options - def title_html_options - { - id: 'js-onboarding-labels-link' - } - end - - override :active_routes - def active_routes - { controller: :labels } - end - - override :sprite_icon - def sprite_icon - 'label' - end - - override :render? - def render? - return false if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - - can?(context.current_user, :read_label, context.project) && !context.project.issues_enabled? - end - end - end - end -end diff --git a/lib/sidebars/projects/menus/members_menu.rb b/lib/sidebars/projects/menus/members_menu.rb deleted file mode 100644 index 498bfa74261..00000000000 --- a/lib/sidebars/projects/menus/members_menu.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Sidebars - module Projects - module Menus - class MembersMenu < ::Sidebars::Menu - override :link - def link - project_project_members_path(context.project) - end - - override :extra_container_html_options - def extra_container_html_options - { - id: 'js-onboarding-members-link' - } - end - - override :title - def title - _('Members') - end - - override :sprite_icon - def sprite_icon - 'users' - end - - override :render? - def render? - return false if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - - can?(context.current_user, :read_project_member, context.project) - end - - override :active_routes - def active_routes - { controller: :project_members } - end - end - end - end -end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb index 8ebdacc7c7e..0d7e0776d5b 100644 --- a/lib/sidebars/projects/menus/monitor_menu.rb +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -14,11 +14,6 @@ module Sidebars add_item(error_tracking_menu_item) add_item(alert_management_menu_item) add_item(incidents_menu_item) - add_item(serverless_menu_item) - add_item(terraform_menu_item) - add_item(kubernetes_menu_item) - add_item(environments_menu_item) - add_item(feature_flags_menu_item) add_item(product_analytics_menu_item) true @@ -26,28 +21,24 @@ module Sidebars override :link def link - if can?(context.current_user, :read_environment, context.project) - metrics_project_environments_path(context.project) - else - project_feature_flags_path(context.project) - end + renderable_items.first&.link end override :extra_container_html_options def extra_container_html_options { - class: Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? 'shortcuts-monitor' : 'shortcuts-operations' + class: 'shortcuts-monitor' } end override :title def title - Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? _('Monitor') : _('Operations') + _('Monitor') end override :sprite_icon def sprite_icon - Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? 'monitor' : 'cloud-gear' + 'monitor' end override :active_routes @@ -138,93 +129,6 @@ module Sidebars ) end - def serverless_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || - !can?(context.current_user, :read_cluster, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :serverless) - end - - ::Sidebars::MenuItem.new( - title: _('Serverless'), - link: project_serverless_functions_path(context.project), - active_routes: { controller: :functions }, - item_id: :serverless - ) - end - - def terraform_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || - !can?(context.current_user, :read_terraform_state, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :terraform) - end - - ::Sidebars::MenuItem.new( - title: _('Terraform'), - link: project_terraform_index_path(context.project), - active_routes: { controller: :terraform }, - item_id: :terraform - ) - end - - def kubernetes_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || - !can?(context.current_user, :read_cluster, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :kubernetes) - end - - ::Sidebars::MenuItem.new( - title: _('Kubernetes'), - link: project_clusters_path(context.project), - active_routes: { controller: [:cluster_agents, :clusters] }, - container_html_options: { class: 'shortcuts-kubernetes' }, - hint_html_options: kubernetes_hint_html_options, - item_id: :kubernetes - ) - end - - def kubernetes_hint_html_options - return {} unless context.show_cluster_hint - - { disabled: true, - data: { trigger: 'manual', - container: 'body', - placement: 'right', - highlight: UserCalloutsHelper::GKE_CLUSTER_INTEGRATION, - highlight_priority: UserCallout.feature_names[:GKE_CLUSTER_INTEGRATION], - dismiss_endpoint: user_callouts_path, - auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } - end - - def environments_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || - !can?(context.current_user, :read_environment, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :environments) - end - - ::Sidebars::MenuItem.new( - title: _('Environments'), - link: project_environments_path(context.project), - active_routes: { controller: :environments }, - container_html_options: { class: 'shortcuts-environments' }, - item_id: :environments - ) - end - - def feature_flags_menu_item - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || - !can?(context.current_user, :read_feature_flag, context.project) - return ::Sidebars::NilMenuItem.new(item_id: :feature_flags) - end - - ::Sidebars::MenuItem.new( - title: _('Feature Flags'), - link: project_feature_flags_path(context.project), - active_routes: { controller: :feature_flags }, - container_html_options: { class: 'shortcuts-feature-flags' }, - item_id: :feature_flags - ) - end - def product_analytics_menu_item if Feature.disabled?(:product_analytics, context.project) || !can?(context.current_user, :read_product_analytics, context.project) diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb index c148e7cf931..a5f06ebea20 100644 --- a/lib/sidebars/projects/menus/project_information_menu.rb +++ b/lib/sidebars/projects/menus/project_information_menu.rb @@ -6,9 +6,7 @@ module Sidebars class ProjectInformationMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - add_item(details_menu_item) add_item(activity_menu_item) - add_item(releases_menu_item) add_item(labels_menu_item) add_item(members_menu_item) @@ -22,11 +20,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - { class: 'shortcuts-project-information' } - else - { class: 'shortcuts-project rspec-project-link' } - end + { class: 'shortcuts-project-information' } end override :extra_nav_link_html_options @@ -36,39 +30,16 @@ module Sidebars override :title def title - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - _('Project information') - else - _('Project overview') - end + _('Project information') end override :sprite_icon def sprite_icon - if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - 'project' - else - 'home' - end + 'project' end private - def details_menu_item - return if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - - ::Sidebars::MenuItem.new( - title: _('Details'), - link: project_path(context.project), - active_routes: { path: 'projects#show' }, - item_id: :project_overview, - container_html_options: { - aria: { label: _('Project details') }, - class: 'shortcuts-project' - } - ) - end - def activity_menu_item ::Sidebars::MenuItem.new( title: _('Activity'), @@ -79,26 +50,8 @@ module Sidebars ) end - def releases_menu_item - return ::Sidebars::NilMenuItem.new(item_id: :releases) unless show_releases? - - ::Sidebars::MenuItem.new( - title: _('Releases'), - link: project_releases_path(context.project), - item_id: :releases, - active_routes: { controller: :releases }, - container_html_options: { class: 'shortcuts-project-releases' } - ) - end - - def show_releases? - Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) && - can?(context.current_user, :read_release, context.project) && - !context.project.empty_repo? - end - def labels_menu_item - if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + unless can?(context.current_user, :read_label, context.project) return ::Sidebars::NilMenuItem.new(item_id: :labels) end @@ -111,7 +64,7 @@ module Sidebars end def members_menu_item - if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + unless can?(context.current_user, :read_project_member, context.project) return ::Sidebars::NilMenuItem.new(item_id: :members) end diff --git a/lib/sidebars/projects/menus/scope_menu.rb b/lib/sidebars/projects/menus/scope_menu.rb index 1cd0218d4ac..539912aa77b 100644 --- a/lib/sidebars/projects/menus/scope_menu.rb +++ b/lib/sidebars/projects/menus/scope_menu.rb @@ -21,8 +21,6 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - return {} if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - { class: 'shortcuts-project rspec-project-link' } @@ -30,8 +28,6 @@ module Sidebars override :extra_nav_link_html_options def extra_nav_link_html_options - return {} if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) - { class: 'context-header' } end diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index c9d7e736b21..250143df649 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -112,9 +112,8 @@ module Sidebars return ::Sidebars::NilMenuItem.new(item_id: :monitor) end - title = Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? _('Monitor') : _('Operations') ::Sidebars::MenuItem.new( - title: title, + title: _('Monitor'), link: project_settings_operations_path(context.project), active_routes: { path: 'operations#show' }, item_id: :monitor @@ -136,7 +135,6 @@ module Sidebars def packages_and_registries_menu_item if !Gitlab.config.registry.enabled || - Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || !can?(context.current_user, :destroy_container_image, context.project) return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) end diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb index ac7c043a96e..d5311c0a0c1 100644 --- a/lib/sidebars/projects/panel.rb +++ b/lib/sidebars/projects/panel.rb @@ -23,7 +23,6 @@ module Sidebars add_menu(Sidebars::Projects::Menus::RepositoryMenu.new(context)) add_menu(Sidebars::Projects::Menus::IssuesMenu.new(context)) add_menu(Sidebars::Projects::Menus::ExternalIssueTrackerMenu.new(context)) - add_menu(Sidebars::Projects::Menus::LabelsMenu.new(context)) add_menu(Sidebars::Projects::Menus::MergeRequestsMenu.new(context)) add_menu(Sidebars::Projects::Menus::CiCdMenu.new(context)) add_menu(Sidebars::Projects::Menus::SecurityComplianceMenu.new(context)) @@ -35,7 +34,6 @@ module Sidebars add_menu(confluence_or_wiki_menu) add_menu(Sidebars::Projects::Menus::ExternalWikiMenu.new(context)) add_menu(Sidebars::Projects::Menus::SnippetsMenu.new(context)) - add_menu(Sidebars::Projects::Menus::MembersMenu.new(context)) add_menu(Sidebars::Projects::Menus::SettingsMenu.new(context)) end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index ea6bc9c4f01..2876f1eb688 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -31,7 +31,7 @@ module SystemCheck end try_fixing_it("mkdir #{backup_dir}", *instructions) - for_more_information('doc/ssh/README.md in section "Overriding SSH settings on the GitLab server"') + for_more_information('doc/ssh/index.md in section "Overriding SSH settings on the GitLab server"') fix_and_rerun end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index 54e74fd9c8b..db10428e0dc 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -5,8 +5,8 @@ require 'fileutils' module Tasks module Gitlab module Assets - FOSS_ASSET_FOLDERS = %w[app/assets app/views fixtures/emojis vendor/assets/javascripts].freeze - EE_ASSET_FOLDERS = %w[ee/app/assets ee/app/views].freeze + FOSS_ASSET_FOLDERS = %w[app/assets fixtures/emojis vendor/assets/javascripts].freeze + EE_ASSET_FOLDERS = %w[ee/app/assets].freeze JS_ASSET_PATTERNS = %w[*.js config/**/*.js].freeze JS_ASSET_FILES = %w[package.json yarn.lock].freeze MASTER_MD5_HASH_FILE = 'master-assets-hash.txt' diff --git a/lib/tasks/gitlab/background_migrations.rake b/lib/tasks/gitlab/background_migrations.rake new file mode 100644 index 00000000000..c978a2807ca --- /dev/null +++ b/lib/tasks/gitlab/background_migrations.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :background_migrations do + task :finalize, [:job_class_name, :table_name, :column_name, :job_arguments] => :environment do |_, args| + [:job_class_name, :table_name, :column_name, :job_arguments].each do |argument| + unless args[argument] + puts "Must specify #{argument} as an argument".color(:red) + exit 1 + end + end + + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + args[:job_class_name], + args[:table_name], + args[:column_name], + Gitlab::Json.parse(args[:job_arguments]) + ) + + puts "Done.".color(:green) + end + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 5b17a8c185a..ed74dd472ff 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -282,6 +282,7 @@ namespace :gitlab do def puts_time(msg) progress.puts "#{Time.now} -- #{msg}" + Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}") end def progress @@ -297,7 +298,9 @@ namespace :gitlab do def repository_backup_strategy if Feature.enabled?(:gitaly_backup) - Backup::GitalyBackup.new(progress) + max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence + max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence + Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency) else Backup::GitalyRpcBackup.new(progress) end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index ee986f4c503..2b508b341dd 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -90,73 +90,35 @@ namespace :gitlab do desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump' task :clean_structure_sql do |task_name| - structure_file = 'db/structure.sql' - schema = File.read(structure_file) + ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| + structure_file = ActiveRecord::Tasks::DatabaseTasks.dump_filename(db_config.name) - File.open(structure_file, 'wb+') do |io| - Gitlab::Database::SchemaCleaner.new(schema).clean(io) - end - - # Allow this task to be called multiple times, as happens when running db:migrate:redo - Rake::Task[task_name].reenable - end + schema = File.read(structure_file) - desc 'This dumps GitLab specific database details - it runs after db:structure:dump' - task :dump_custom_structure do |task_name| - Gitlab::Database::CustomStructure.new.dump + File.open(structure_file, 'wb+') do |io| + Gitlab::Database::SchemaCleaner.new(schema).clean(io) + end + end # Allow this task to be called multiple times, as happens when running db:migrate:redo Rake::Task[task_name].reenable end - desc 'This loads GitLab specific database details - runs after db:structure:dump' - task :load_custom_structure do - configuration = Rails.application.config_for(:database) - - ENV['PGHOST'] = configuration['host'] if configuration['host'] - ENV['PGPORT'] = configuration['port'].to_s if configuration['port'] - ENV['PGPASSWORD'] = configuration['password'].to_s if configuration['password'] - ENV['PGUSER'] = configuration['username'].to_s if configuration['username'] - - command = 'psql' - dump_filepath = Gitlab::Database::CustomStructure.custom_dump_filepath.to_path - args = ['-v', 'ON_ERROR_STOP=1', '-q', '-X', '-f', dump_filepath, configuration['database']] - - unless Kernel.system(command, *args) - raise "failed to execute:\n#{command} #{args.join(' ')}\n\n" \ - "Please ensure `#{command}` is installed in your PATH and has proper permissions.\n\n" - end - end - # Inform Rake that custom tasks should be run every time rake db:structure:dump is run # # Rails 6.1 deprecates db:structure:dump in favor of db:schema:dump Rake::Task['db:structure:dump'].enhance do Rake::Task['gitlab:db:clean_structure_sql'].invoke - Rake::Task['gitlab:db:dump_custom_structure'].invoke end # Inform Rake that custom tasks should be run every time rake db:schema:dump is run Rake::Task['db:schema:dump'].enhance do Rake::Task['gitlab:db:clean_structure_sql'].invoke - Rake::Task['gitlab:db:dump_custom_structure'].invoke - end - - # Inform Rake that custom tasks should be run every time rake db:structure:load is run - # - # Rails 6.1 deprecates db:structure:load in favor of db:schema:load - Rake::Task['db:structure:load'].enhance do - Rake::Task['gitlab:db:load_custom_structure'].invoke - end - - # Inform Rake that custom tasks should be run every time rake db:schema:load is run - Rake::Task['db:schema:load'].enhance do - Rake::Task['gitlab:db:load_custom_structure'].invoke end desc 'Create missing dynamic database partitions' - task :create_dynamic_partitions do - Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions + task create_dynamic_partitions: :environment do + Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions end # This is targeted towards deploys and upgrades of GitLab. @@ -192,7 +154,7 @@ namespace :gitlab do Rake::Task['gitlab:db:create_dynamic_partitions'].invoke end - desc 'reindex a regular (non-unique) index without downtime to eliminate bloat' + desc 'reindex a regular index without downtime to eliminate bloat' task :reindex, [:index_name] => :environment do |_, args| unless Feature.enabled?(:database_reindexing, type: :ops) puts "This feature (database_reindexing) is currently disabled.".color(:yellow) diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index b61b1833c5a..b467aa3819d 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -3,6 +3,8 @@ # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] -task gitlab_environment: :environment do +task :gitlab_environment do + Rake::Task[:environment].invoke unless ENV['SKIP_RAILS_ENV_IN_RAKE'] + extend SystemCheck::Helpers end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 6f5c3a86dd3..d3060d92e88 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -1,7 +1,5 @@ # frozen_string_literal: true -return if Rails.env.production? - namespace :gitlab do namespace :sidekiq do def write_yaml(path, banner, object) @@ -9,31 +7,28 @@ namespace :gitlab do end namespace :migrate_jobs do - def mappings - ::Gitlab::SidekiqConfig - .workers - .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } - .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } - end - desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' task schedule: :environment do ::Gitlab::SidekiqMigrateJobs .new('schedule', logger: Logger.new($stdout)) - .execute(mappings) + .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) end desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' task retry: :environment do ::Gitlab::SidekiqMigrateJobs .new('retry', logger: Logger.new($stdout)) - .execute(mappings) + .execute(::Gitlab::SidekiqConfig.worker_queue_mappings) end end + task :not_production do + raise 'This task cannot be run in the production environment' if Rails.env.production? + end + namespace :all_queues_yml do desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' - task generate: :environment do + task generate: ['gitlab:sidekiq:not_production', :environment] do banner = <<~BANNER # This file is generated automatically by # bin/rake gitlab:sidekiq:all_queues_yml:generate @@ -51,7 +46,7 @@ namespace :gitlab do end desc 'GitLab | Sidekiq | Validate that all_queues.yml matches worker definitions' - task check: :environment do + task check: ['gitlab:sidekiq:not_production', :environment] do if Gitlab::SidekiqConfig.all_queues_yml_outdated? raise <<~MSG Changes in worker queues found, please update the metadata by running: @@ -70,7 +65,7 @@ namespace :gitlab do namespace :sidekiq_queues_yml do desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions' - task generate: :environment do + task generate: ['gitlab:sidekiq:not_production', :environment] do banner = <<~BANNER # This file is generated automatically by # bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate @@ -104,7 +99,7 @@ namespace :gitlab do end desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions' - task check: :environment do + task check: ['gitlab:sidekiq:not_production', :environment] do if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated? raise <<~MSG Changes in worker queues found, please update the metadata by running: diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 0ad50c0fa53..166f08ef16a 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -19,7 +19,7 @@ namespace :gitlab do desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application' task generate_and_send: :environment do - result = SubmitUsagePingService.new.execute + result = ServicePing::SubmitService.new.execute puts Gitlab::Json.pretty_generate(result.attributes) end |