diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-19 15:44:42 +0000 |
| commit | 4555e1b21c365ed8303ffb7a3325d773c9b8bf31 (patch) | |
| tree | 5423a1c7516cffe36384133ade12572cf709398d /lib | |
| parent | e570267f2f6b326480d284e0164a6464ba4081bc (diff) | |
| download | gitlab-ce-4555e1b21c365ed8303ffb7a3325d773c9b8bf31.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42
Diffstat (limited to 'lib')
749 files changed, 10255 insertions, 4741 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 6a180fdf338..aea4231205d 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -16,7 +16,7 @@ module AfterCommitQueue def run_after_commit_or_now(&block) if Gitlab::Database.inside_transaction? - if ActiveRecord::Base.connection.current_transaction.records.include?(self) + if ActiveRecord::Base.connection.current_transaction.records&.include?(self) run_after_commit(&block) else # If the current transaction does not include this record, we can run diff --git a/lib/api/api.rb b/lib/api/api.rb index a287ffbfcd8..54e5cc5c8d0 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -130,32 +130,6 @@ module API formatter :json, Gitlab::Json::GrapeFormatter content_type :json, 'application/json' - # Remove the `text/plain+deprecated` with `api_always_use_application_json` feature flag - # There is a small chance some users depend on the old behavior. - # We this change under a feature flag to see if affects GitLab.com users. - # The `+deprecated` is added to distinguish content type - # as defined by `API::API` vs ex. `API::Repositories` - content_type :txt, 'text/plain+deprecated' - - before do - # the feature flag workaround is only for `.txt` - api_format = env[Grape::Env::API_FORMAT] - next unless api_format == :txt - - # get all defined content-types for the endpoint - api_endpoint = env[Grape::Env::API_ENDPOINT] - content_types = api_endpoint&.namespace_stackable_with_hash(:content_types).to_h - - # Only overwrite `text/plain+deprecated` - if content_types[api_format] == 'text/plain+deprecated' - if Feature.enabled?(:api_always_use_application_json, default_enabled: :yaml) - content_type 'application/json' - else - content_type 'text/plain' - end - end - end - # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers helpers ::API::Helpers::CommonHelpers @@ -267,6 +241,7 @@ module API mount ::API::ProjectTemplates mount ::API::Terraform::State mount ::API::Terraform::StateVersion + mount ::API::Terraform::Modules::V1::Packages mount ::API::PersonalAccessTokens mount ::API::ProtectedBranches mount ::API::ProtectedTags @@ -323,4 +298,4 @@ module API end end -API::API.prepend_ee_mod +API::API.prepend_mod diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 8ea4f32d3eb..c8485054377 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -4,23 +4,18 @@ module API class AwardEmoji < ::API::Base include PaginationParams + helpers ::API::Helpers::AwardEmoji + before { authenticate! } - AWARDABLES = [ - { type: 'issue', find_by: :iid, feature_category: :issue_tracking }, - { type: 'merge_request', find_by: :iid, feature_category: :code_review }, - { type: 'snippet', find_by: :id, feature_category: :snippets } - ].freeze - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - AWARDABLES.each do |awardable_params| + + Helpers::AwardEmoji.awardables.each do |awardable_params| + resource awardable_params[:resource], requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do awardable_string = awardable_params[:type].pluralize awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" params do - requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + requires :id, type: String, desc: "The ID of a #{awardable_params[:resource] == :projects ? 'project' : 'group'}" + requires :"#{awardable_id_string}", type: Integer, desc: Helpers::AwardEmoji.awardable_id_desc end [ @@ -104,25 +99,6 @@ module API awardable.user_can_award?(current_user) end - # rubocop: disable CodeReuse/ActiveRecord - def awardable - @awardable ||= - begin - if params.include?(:note_id) - note_id = params.delete(:note_id) - - awardable.notes.find(note_id) - elsif params.include?(:issue_iid) - user_project.issues.find_by!(iid: params[:issue_iid]) - elsif params.include?(:merge_request_iid) - user_project.merge_requests.find_by!(iid: params[:merge_request_iid]) - else - user_project.snippets.find(params[:snippet_id]) - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - def read_ability(awardable) case awardable when Note diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 79f4b02f26a..9e829dd5e05 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -5,7 +5,7 @@ module API include BoardsResponses include PaginationParams - prepend_if_ee('EE::API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule feature_category :boards diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 6842e93a4de..1ee120f982a 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -38,22 +38,38 @@ module API optional :page_token, type: String, desc: 'Name of branch to start the paginaition from' end get ':id/repository/branches' do - user_project.preload_protected_branches - - repository = user_project.repository - - branches_finder = BranchesFinder.new(repository, declared_params(include_missing: false)) - branches = Gitlab::Pagination::GitalyKeysetPager.new(self, user_project).paginate(branches_finder) - - merged_branch_names = repository.merged_branch_names(branches.map(&:name)) - - present( - branches, - with: Entities::Branch, - current_user: current_user, - project: user_project, - merged_branch_names: merged_branch_names - ) + ff_enabled = Feature.enabled?(:api_caching_rate_limit_branches, user_project, default_enabled: :yaml) + + cache_action_if(ff_enabled, [user_project, :branches, current_user, declared_params], expires_in: 30.seconds) do + user_project.preload_protected_branches + + repository = user_project.repository + + branches_finder = BranchesFinder.new(repository, declared_params(include_missing: false)) + branches = Gitlab::Pagination::GitalyKeysetPager.new(self, user_project).paginate(branches_finder) + + merged_branch_names = repository.merged_branch_names(branches.map(&:name)) + + if Feature.enabled?(:api_caching_branches, user_project, type: :development, default_enabled: :yaml) + present_cached( + branches, + with: Entities::Branch, + current_user: current_user, + project: user_project, + merged_branch_names: merged_branch_names, + expires_in: 10.minutes, + cache_context: -> (branch) { [current_user&.cache_key, merged_branch_names.include?(branch.name)] } + ) + else + present( + branches, + with: Entities::Branch, + current_user: current_user, + project: user_project, + merged_branch_names: merged_branch_names + ) + end + end end resource ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index c5249f1377b..33980b38e2b 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -184,6 +184,8 @@ module API .new(job, declared_params(include_missing: false)) service.execute.then do |result| + track_ci_minutes_usage!(job, current_runner) + header 'X-GitLab-Trace-Update-Interval', result.backoff status result.status body result.status.to_s @@ -214,6 +216,8 @@ module API break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{result.stream_size}" }) end + track_ci_minutes_usage!(job, current_runner) + status result.status header 'Job-Status', job.status header 'Range', "0-#{result.stream_size}" diff --git a/lib/api/concerns/packages/debian_endpoints.rb b/lib/api/concerns/packages/debian_endpoints.rb new file mode 100644 index 00000000000..6fc7c439464 --- /dev/null +++ b/lib/api/concerns/packages/debian_endpoints.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +module API + module Concerns + module Packages + module DebianEndpoints + extend ActiveSupport::Concern + + DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze + COMPONENT_REGEX = %r{[a-z-]+}.freeze + ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze + LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX + DISTRIBUTION_REQUIREMENTS = { + distribution: DISTRIBUTION_REGEX + }.freeze + COMPONENT_ARCHITECTURE_REQUIREMENTS = { + component: COMPONENT_REGEX, + architecture: ARCHITECTURE_REGEX + }.freeze + COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { + component: COMPONENT_REGEX, + letter: LETTER_REGEX, + source_package: PACKAGE_REGEX + }.freeze + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + included do + feature_category :package_registry + + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + + format :txt + content_type :txt, 'text/plain' + + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + before do + require_packages_enabled! + end + + namespace 'packages/debian' do + params do + requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex + end + + namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg + desc 'The Release file signature' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'Release.gpg' do + not_found! + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release + desc 'The unsigned Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'Release' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Release' + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease + desc 'The signed Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'InRelease' do + not_found! + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + end + + namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + desc 'The binary files index' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'Packages' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Packages' + end + end + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' + requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex + end + + namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name + params do + requires :file_name, type: String, desc: 'The Debian File Name' + end + desc 'The package' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get ':file_name', requirements: FILE_NAME_REQUIREMENTS do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO File' + end + end + end + end + end + end + end +end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index f138f400601..06edab662bf 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -15,8 +15,8 @@ module API authorize_read_package!(user_group) end - namespace ':id/packages/debian' do - include DebianPackageEndpoints + namespace ':id/-' do + include ::API::Concerns::Packages::DebianEndpoints end end end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb deleted file mode 100644 index e7689b3feff..00000000000 --- a/lib/api/debian_package_endpoints.rb +++ /dev/null @@ -1,127 +0,0 @@ -# frozen_string_literal: true - -module API - module DebianPackageEndpoints - extend ActiveSupport::Concern - - DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze - COMPONENT_REGEX = %r{[a-z-]+}.freeze - ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze - LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze - PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX - DISTRIBUTION_REQUIREMENTS = { - distribution: DISTRIBUTION_REGEX - }.freeze - COMPONENT_ARCHITECTURE_REQUIREMENTS = { - component: COMPONENT_REGEX, - architecture: ARCHITECTURE_REGEX - }.freeze - COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { - component: COMPONENT_REGEX, - letter: LETTER_REGEX, - source_package: PACKAGE_REGEX - }.freeze - FILE_NAME_REQUIREMENTS = { - file_name: API::NO_SLASH_URL_PART_REGEX - }.freeze - - included do - feature_category :package_registry - - helpers ::API::Helpers::PackagesHelpers - helpers ::API::Helpers::Packages::BasicAuthHelpers - - format :txt - content_type :txt, 'text/plain' - - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end - - before do - require_packages_enabled! - end - - params do - requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex - end - - namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg - desc 'The Release file signature' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'Release.gpg' do - not_found! - end - - # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release - desc 'The unsigned Release file' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'Release' do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO Release' - end - - # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease - desc 'The signed Release file' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'InRelease' do - not_found! - end - - params do - requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex - end - - namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages - desc 'The binary files index' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'Packages' do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO Packages' - end - end - end - - params do - requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' - requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex - end - - namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name - params do - requires :file_name, type: String, desc: 'The Debian File Name' - end - desc 'The package' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get ':file_name', requirements: FILE_NAME_REQUIREMENTS do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO File' - end - end - end - end -end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 8c0db42a448..0ed828fd639 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -15,14 +15,14 @@ module API authorize_read_package! end - namespace ':id/packages/debian' do - include DebianPackageEndpoints + namespace ':id' do + include ::API::Concerns::Packages::DebianEndpoints params do requires :file_name, type: String, desc: 'The file name' end - namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do + namespace 'packages/debian/:file_name', requirements: FILE_NAME_REQUIREMENTS do content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE # PUT {projects|groups}/:id/packages/debian/:file_name diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 30ec4e52b2a..e9beeb18d62 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -18,6 +18,10 @@ module API result_hash[:read_repository] = scopes.include?('read_repository') result_hash end + + params :filter_params do + optional :active, type: Boolean, desc: 'Limit by active status' + end end desc 'Return all deploy tokens' do @@ -26,11 +30,18 @@ module API end params do use :pagination + use :filter_params end get 'deploy_tokens' do authenticated_as_admin! - present paginate(DeployToken.all), with: Entities::DeployToken + deploy_tokens = ::DeployTokens::TokensFinder.new( + current_user, + :all, + declared_params + ).execute + + present paginate(deploy_tokens), with: Entities::DeployToken end params do @@ -39,6 +50,7 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do use :pagination + use :filter_params end desc 'List deploy tokens for a project' do detail 'This feature was introduced in GitLab 12.9' @@ -47,7 +59,13 @@ module API get ':id/deploy_tokens' do authorize!(:read_deploy_token, user_project) - present paginate(user_project.deploy_tokens), with: Entities::DeployToken + deploy_tokens = ::DeployTokens::TokensFinder.new( + current_user, + user_project, + declared_params + ).execute + + present paginate(deploy_tokens), with: Entities::DeployToken end params do @@ -98,6 +116,7 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do use :pagination + use :filter_params end desc 'List deploy tokens for a group' do detail 'This feature was introduced in GitLab 12.9' @@ -106,7 +125,13 @@ module API get ':id/deploy_tokens' do authorize!(:read_deploy_token, user_group) - present paginate(user_group.deploy_tokens), with: Entities::DeployToken + deploy_tokens = ::DeployTokens::TokensFinder.new( + current_user, + user_group, + declared_params + ).execute + + present paginate(deploy_tokens), with: Entities::DeployToken end params do diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 0a6ecf2919c..80a50ded522 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -41,6 +41,8 @@ module API .execute.with_api_entity_associations present paginate(deployments), with: Entities::Deployment + rescue DeploymentsFinder::InefficientQueryError => e + bad_request!(e.message) end desc 'Gets a specific deployment' do diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index 2468c1f9b18..f23fce40468 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -36,4 +36,4 @@ module API end end -API::Entities::ApplicationSetting.prepend_if_ee('EE::API::Entities::ApplicationSetting') +API::Entities::ApplicationSetting.prepend_mod_with('API::Entities::ApplicationSetting') diff --git a/lib/api/entities/board.rb b/lib/api/entities/board.rb index fe0182ad772..ee0bea466e0 100644 --- a/lib/api/entities/board.rb +++ b/lib/api/entities/board.rb @@ -16,4 +16,4 @@ module API end end -API::Entities::Board.prepend_if_ee('EE::API::Entities::Board') +API::Entities::Board.prepend_mod_with('API::Entities::Board') diff --git a/lib/api/entities/bulk_imports/export_status.rb b/lib/api/entities/bulk_imports/export_status.rb new file mode 100644 index 00000000000..c9c7f34a16a --- /dev/null +++ b/lib/api/entities/bulk_imports/export_status.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module BulkImports + class ExportStatus < Grape::Entity + expose :relation + expose :status + expose :error + expose :updated_at + end + end + end +end diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb index a29788c7abf..c31340f1ff0 100644 --- a/lib/api/entities/ci/job_basic.rb +++ b/lib/api/entities/ci/job_basic.rb @@ -6,7 +6,10 @@ module API class JobBasic < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage, :allow_failure expose :created_at, :started_at, :finished_at - expose :duration + expose :duration, + documentation: { type: 'Floating', desc: 'Time spent running' } + expose :queued_duration, + documentation: { type: 'Floating', desc: 'Time spent enqueued' } expose :user, with: ::API::Entities::User expose :commit, with: ::API::Entities::Commit expose :pipeline, with: ::API::Entities::Ci::PipelineBasic diff --git a/lib/api/entities/ci/pipeline.rb b/lib/api/entities/ci/pipeline.rb index 3dd3b9c9eff..11336ae070d 100644 --- a/lib/api/entities/ci/pipeline.rb +++ b/lib/api/entities/ci/pipeline.rb @@ -9,6 +9,7 @@ module API expose :user, with: Entities::UserBasic expose :created_at, :updated_at, :started_at, :finished_at, :committed_at expose :duration + expose :queued_duration expose :coverage expose :detailed_status, using: DetailedStatusEntity do |pipeline, options| pipeline.detailed_status(options[:current_user]) diff --git a/lib/api/entities/deploy_token.rb b/lib/api/entities/deploy_token.rb index 9c5bf54e299..daee104ba6b 100644 --- a/lib/api/entities/deploy_token.rb +++ b/lib/api/entities/deploy_token.rb @@ -4,7 +4,8 @@ module API module Entities class DeployToken < Grape::Entity # exposing :token is a security risk and should be avoided - expose :id, :name, :username, :expires_at, :scopes + expose :id, :name, :username, :expires_at, :scopes, :revoked + expose :expired?, as: :expired end end end diff --git a/lib/api/entities/environment.rb b/lib/api/entities/environment.rb index cb39ce1b13a..91867f3403d 100644 --- a/lib/api/entities/environment.rb +++ b/lib/api/entities/environment.rb @@ -3,9 +3,48 @@ module API module Entities class Environment < Entities::EnvironmentBasic + include RequestAwareEntity + include Gitlab::Utils::StrongMemoize + expose :project, using: Entities::BasicProjectDetails expose :last_deployment, using: Entities::Deployment, if: { last_deployment: true } expose :state + + expose :enable_advanced_logs_querying, if: -> (*) { can_read_pod_logs? } do |environment| + environment.elastic_stack_available? + end + + expose :logs_api_path, if: -> (*) { can_read_pod_logs? } do |environment| + if environment.elastic_stack_available? + elasticsearch_project_logs_path(environment.project, environment_name: environment.name, format: :json) + else + k8s_project_logs_path(environment.project, environment_name: environment.name, format: :json) + end + end + + expose :gitlab_managed_apps_logs_path, if: -> (*) { can_read_pod_logs? && cluster } do |environment| + ::Clusters::ClusterPresenter.new(cluster, current_user: current_user).gitlab_managed_apps_logs_path # rubocop: disable CodeReuse/Presenter + end + + private + + alias_method :environment, :object + + def can_read_pod_logs? + strong_memoize(:can_read_pod_logs) do + current_user&.can?(:read_pod_logs, environment.project) + end + end + + def cluster + strong_memoize(:cluster) do + environment&.last_deployment&.cluster + end + end + + def current_user + options[:current_user] + end end end end diff --git a/lib/api/entities/group.rb b/lib/api/entities/group.rb index e430eba4880..048b7a3c15a 100644 --- a/lib/api/entities/group.rb +++ b/lib/api/entities/group.rb @@ -38,4 +38,4 @@ module API end end -API::Entities::Group.prepend_if_ee('EE::API::Entities::Group', with_descendants: true) +API::Entities::Group.prepend_mod_with('API::Entities::Group', with_descendants: true) diff --git a/lib/api/entities/group_detail.rb b/lib/api/entities/group_detail.rb index 2d9d4ca7992..e63a3fc1334 100644 --- a/lib/api/entities/group_detail.rb +++ b/lib/api/entities/group_detail.rb @@ -39,4 +39,4 @@ module API end end -API::Entities::GroupDetail.prepend_if_ee('EE::API::Entities::GroupDetail') +API::Entities::GroupDetail.prepend_mod_with('API::Entities::GroupDetail') diff --git a/lib/api/entities/identity.rb b/lib/api/entities/identity.rb index 52045b6250a..7c8cda8f9c2 100644 --- a/lib/api/entities/identity.rb +++ b/lib/api/entities/identity.rb @@ -8,4 +8,4 @@ module API end end -API::Entities::Identity.prepend_if_ee('EE::API::Entities::Identity') +API::Entities::Identity.prepend_mod_with('API::Entities::Identity') diff --git a/lib/api/entities/issuable_entity.rb b/lib/api/entities/issuable_entity.rb index e2c674c0b8b..fd5d6c8137f 100644 --- a/lib/api/entities/issuable_entity.rb +++ b/lib/api/entities/issuable_entity.rb @@ -24,7 +24,7 @@ module API # entity according to the current top-level entity options, such # as the current_user. def lazy_issuable_metadata - BatchLoader.for(object).batch(key: [current_user, :issuable_metadata]) do |models, loader, args| + BatchLoader.for(object).batch(key: [current_user, :issuable_metadata], replace_methods: false) do |models, loader, args| current_user = args[:key].first issuable_metadata = Gitlab::IssuableMetadata.new(current_user, models) diff --git a/lib/api/entities/issue.rb b/lib/api/entities/issue.rb index 82102854394..e2506cc596e 100644 --- a/lib/api/entities/issue.rb +++ b/lib/api/entities/issue.rb @@ -48,4 +48,4 @@ module API end end -API::Entities::Issue.prepend_if_ee('EE::API::Entities::Issue') +API::Entities::Issue.prepend_mod_with('API::Entities::Issue') diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb index cf96c6556ec..d27cc5498bd 100644 --- a/lib/api/entities/issue_basic.rb +++ b/lib/api/entities/issue_basic.rb @@ -3,6 +3,10 @@ module API module Entities class IssueBasic < IssuableEntity + format_with(:upcase) do |item| + item.upcase if item.respond_to?(:upcase) + end + expose :closed_at expose :closed_by, using: Entities::UserBasic @@ -16,6 +20,10 @@ module API expose :milestone, using: Entities::Milestone expose :assignees, :author, using: Entities::UserBasic + expose :issue_type, + as: :type, + format_with: :upcase, + documentation: { type: "String", desc: "One of #{Issue.issue_types.keys.map(&:upcase)}" } expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first @@ -28,6 +36,7 @@ module API expose :due_date expose :confidential expose :discussion_locked + expose :issue_type expose :web_url do |issue| Gitlab::UrlBuilder.build(issue) @@ -42,4 +51,4 @@ module API end end -API::Entities::IssueBasic.prepend_if_ee('EE::API::Entities::IssueBasic', with_descendants: true) +API::Entities::IssueBasic.prepend_mod_with('API::Entities::IssueBasic', with_descendants: true) diff --git a/lib/api/entities/job_request/response.rb b/lib/api/entities/job_request/response.rb index bf22ea1e6e2..2e8dfc5bde0 100644 --- a/lib/api/entities/job_request/response.rb +++ b/lib/api/entities/job_request/response.rb @@ -34,4 +34,4 @@ module API end end -API::Entities::JobRequest::Response.prepend_if_ee('EE::API::Entities::JobRequest::Response') +API::Entities::JobRequest::Response.prepend_mod_with('API::Entities::JobRequest::Response') diff --git a/lib/api/entities/label_basic.rb b/lib/api/entities/label_basic.rb index ed52688638e..00ecea26ec3 100644 --- a/lib/api/entities/label_basic.rb +++ b/lib/api/entities/label_basic.rb @@ -3,7 +3,7 @@ module API module Entities class LabelBasic < Grape::Entity - expose :id, :name, :color, :description, :description_html, :text_color + expose :id, :name, :color, :description, :description_html, :text_color, :remove_on_close end end end diff --git a/lib/api/entities/list.rb b/lib/api/entities/list.rb index 480e722c22c..e9d31827e2f 100644 --- a/lib/api/entities/list.rb +++ b/lib/api/entities/list.rb @@ -10,4 +10,4 @@ module API end end -API::Entities::List.prepend_if_ee('EE::API::Entities::List') +API::Entities::List.prepend_mod_with('API::Entities::List') diff --git a/lib/api/entities/member.rb b/lib/api/entities/member.rb index ad62f92e5a0..87f03adba31 100644 --- a/lib/api/entities/member.rb +++ b/lib/api/entities/member.rb @@ -11,4 +11,4 @@ module API end end -API::Entities::Member.prepend_if_ee('EE::API::Entities::Member', with_descendants: true) +API::Entities::Member.prepend_mod_with('API::Entities::Member', with_descendants: true) diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index 88c84c494e2..cf8d03bf176 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -89,4 +89,4 @@ module API end end -API::Entities::MergeRequestBasic.prepend_if_ee('EE::API::Entities::MergeRequestBasic', with_descendants: true) +API::Entities::MergeRequestBasic.prepend_mod_with('API::Entities::MergeRequestBasic', with_descendants: true) diff --git a/lib/api/entities/namespace.rb b/lib/api/entities/namespace.rb index a7e06cc3e02..f11303d41a6 100644 --- a/lib/api/entities/namespace.rb +++ b/lib/api/entities/namespace.rb @@ -14,4 +14,4 @@ module API end end -API::Entities::Namespace.prepend_if_ee('EE::API::Entities::Namespace') +API::Entities::Namespace.prepend_mod_with('API::Entities::Namespace') diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index e7153f9bebb..2f60a0bf6bd 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -22,6 +22,7 @@ module API expose :version expose :package_type + expose :status expose :_links do expose :web_path do |package| diff --git a/lib/api/entities/package_file.rb b/lib/api/entities/package_file.rb index 2cc2f62a948..e34a6a7aa1d 100644 --- a/lib/api/entities/package_file.rb +++ b/lib/api/entities/package_file.rb @@ -5,7 +5,7 @@ module API class PackageFile < Grape::Entity expose :id, :package_id, :created_at expose :file_name, :size - expose :file_md5, :file_sha1 + expose :file_md5, :file_sha1, :file_sha256 expose :pipelines, if: ->(package_file) { package_file.pipelines.present? }, using: Package::Pipeline end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 690bc5d419d..442013c07dd 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -147,4 +147,4 @@ module API end end -API::Entities::Project.prepend_if_ee('EE::API::Entities::Project', with_descendants: true) +API::Entities::Project.prepend_mod_with('API::Entities::Project', with_descendants: true) diff --git a/lib/api/entities/protected_branch.rb b/lib/api/entities/protected_branch.rb index e5dbaffb591..ac44d06e69c 100644 --- a/lib/api/entities/protected_branch.rb +++ b/lib/api/entities/protected_branch.rb @@ -12,4 +12,4 @@ module API end end -API::Entities::ProtectedBranch.prepend_if_ee('EE::API::Entities::ProtectedBranch') +API::Entities::ProtectedBranch.prepend_mod_with('API::Entities::ProtectedBranch') diff --git a/lib/api/entities/protected_ref_access.rb b/lib/api/entities/protected_ref_access.rb index f0185705b06..443277e23cf 100644 --- a/lib/api/entities/protected_ref_access.rb +++ b/lib/api/entities/protected_ref_access.rb @@ -11,4 +11,4 @@ module API end end -API::Entities::ProtectedRefAccess.prepend_if_ee('EE::API::Entities::ProtectedRefAccess') +API::Entities::ProtectedRefAccess.prepend_mod_with('API::Entities::ProtectedRefAccess') diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index f6c3dd5a509..94124352298 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -8,7 +8,7 @@ module API expose :name expose :tag, as: :tag_name, if: ->(_, _) { can_download_code? } expose :description - expose :description_html do |entity| + expose :description_html, unless: ->(_, _) { remove_description_html? } do |entity| MarkupHelper.markdown_field(entity, :description, current_user: options[:current_user]) end expose :created_at @@ -28,9 +28,7 @@ module API expose :assets do expose :assets_count, as: :count expose :sources, using: Entities::Releases::Source, if: ->(_, _) { can_download_code? } - expose :links, using: Entities::Releases::Link do |release, options| - release.links.sorted - end + expose :sorted_links, as: :links, using: Entities::Releases::Link end expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_download_code? } expose :_links do @@ -47,6 +45,11 @@ module API def can_read_milestone? Ability.allowed?(options[:current_user], :read_milestone, object.project) end + + def remove_description_html? + ::Feature.enabled?(:remove_description_html_in_release_api, object.project, default_enabled: :yaml) && + ::Feature.disabled?(:remove_description_html_in_release_api_override, object.project) + end end end end diff --git a/lib/api/entities/terraform/module_versions.rb b/lib/api/entities/terraform/module_versions.rb new file mode 100644 index 00000000000..75037039117 --- /dev/null +++ b/lib/api/entities/terraform/module_versions.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Terraform + class ModuleVersions < Grape::Entity + expose :modules + end + end + end +end diff --git a/lib/api/entities/todo.rb b/lib/api/entities/todo.rb index 0acbb4cb704..8d222db488a 100644 --- a/lib/api/entities/todo.rb +++ b/lib/api/entities/todo.rb @@ -58,4 +58,4 @@ module API end end -API::Entities::Todo.prepend_if_ee('EE::API::Entities::Todo') +API::Entities::Todo.prepend_mod_with('API::Entities::Todo') diff --git a/lib/api/entities/user_basic.rb b/lib/api/entities/user_basic.rb index 80f3ee7b502..b8ee4e5a6e0 100644 --- a/lib/api/entities/user_basic.rb +++ b/lib/api/entities/user_basic.rb @@ -19,4 +19,4 @@ module API end end -API::Entities::UserBasic.prepend_if_ee('EE::API::Entities::UserBasic') +API::Entities::UserBasic.prepend_mod_with('API::Entities::UserBasic') diff --git a/lib/api/entities/user_credit_card_validations.rb b/lib/api/entities/user_credit_card_validations.rb new file mode 100644 index 00000000000..fcd42388b16 --- /dev/null +++ b/lib/api/entities/user_credit_card_validations.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class UserCreditCardValidations < Grape::Entity + expose :user_id, :credit_card_validated_at + end + end +end diff --git a/lib/api/entities/user_details_with_admin.rb b/lib/api/entities/user_details_with_admin.rb index e48b1da7859..3572b677646 100644 --- a/lib/api/entities/user_details_with_admin.rb +++ b/lib/api/entities/user_details_with_admin.rb @@ -11,4 +11,4 @@ module API end end -API::Entities::UserDetailsWithAdmin.prepend_if_ee('EE::API::Entities::UserDetailsWithAdmin') +API::Entities::UserDetailsWithAdmin.prepend_mod_with('API::Entities::UserDetailsWithAdmin') diff --git a/lib/api/entities/user_path.rb b/lib/api/entities/user_path.rb index 3f007659813..ed54857d041 100644 --- a/lib/api/entities/user_path.rb +++ b/lib/api/entities/user_path.rb @@ -13,4 +13,4 @@ module API end end -API::Entities::UserPath.prepend_if_ee('EE::API::Entities::UserPath') +API::Entities::UserPath.prepend_mod_with('API::Entities::UserPath') diff --git a/lib/api/entities/user_public.rb b/lib/api/entities/user_public.rb index 685adb1dd10..78f088d3c1a 100644 --- a/lib/api/entities/user_public.rb +++ b/lib/api/entities/user_public.rb @@ -19,4 +19,4 @@ module API end end -API::Entities::UserPublic.prepend_if_ee('EE::API::Entities::UserPublic', with_descendants: true) +API::Entities::UserPublic.prepend_mod_with('API::Entities::UserPublic', with_descendants: true) diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb index ab7bc738ff8..e148a5c45b5 100644 --- a/lib/api/entities/user_with_admin.rb +++ b/lib/api/entities/user_with_admin.rb @@ -9,4 +9,4 @@ module API end end -API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin') +API::Entities::UserWithAdmin.prepend_mod_with('API::Entities::UserWithAdmin') diff --git a/lib/api/environments.rb b/lib/api/environments.rb index b606b2e814d..57e548183b0 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -26,7 +26,7 @@ module API get ':id/environments' do authorize! :read_environment, user_project - environments = ::EnvironmentsFinder.new(user_project, current_user, params).execute + environments = ::Environments::EnvironmentsFinder.new(user_project, current_user, params).execute present paginate(environments), with: Entities::Environment, current_user: current_user end diff --git a/lib/api/features.rb b/lib/api/features.rb index 57bd7c38ad2..2ce2f7c518f 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -120,4 +120,4 @@ module API end end -API::Features.prepend_if_ee('EE::API::Features') +API::Features.prepend_mod_with('API::Features') diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index cce55fa92d9..d0680ad7bc5 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -74,6 +74,8 @@ module API Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id }) forbidden! + rescue ::Packages::DuplicatePackageError + bad_request!('Duplicate package is not allowed') end desc 'Download package file' do diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 90632048354..92869f8fbba 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -5,7 +5,7 @@ module API include BoardsResponses include PaginationParams - prepend_if_ee('EE::API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule feature_category :boards diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 29ffbea687a..6134515032f 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -43,6 +43,43 @@ module API render_api_error!(message: 'Group export could not be started.') end end + + desc 'Start relations export' do + detail 'This feature was introduced in GitLab 13.12' + end + post ':id/export_relations' do + response = ::BulkImports::ExportService.new(portable: user_group, user: current_user).execute + + if response.success? + accepted! + else + render_api_error!(message: 'Group relations export could not be started.') + end + end + + desc 'Download relations export' do + detail 'This feature was introduced in GitLab 13.12' + end + params do + requires :relation, type: String, desc: 'Group relation name' + end + get ':id/export_relations/download' do + export = user_group.bulk_import_exports.find_by_relation(params[:relation]) + file = export&.upload&.export_file + + if file + present_carrierwave_file!(file) + else + render_api_error!('404 Not found', 404) + end + end + + desc 'Relations export status' do + detail 'This feature was introduced in GitLab 13.12' + end + get ':id/export_relations/status' do + present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus + end end end end diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index dfffd3b1209..061d0410a9c 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -96,4 +96,4 @@ module API end end -API::GroupMilestones.prepend_if_ee('EE::API::GroupMilestones') +API::GroupMilestones.prepend_mod_with('API::GroupMilestones') diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 912813d5bb7..1a604e70bf1 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -401,4 +401,4 @@ module API end end -API::Groups.prepend_if_ee('EE::API::Groups') +API::Groups.prepend_mod_with('API::Groups') diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2d8a4f60e2a..632717e1b73 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -124,12 +124,22 @@ module API def find_project!(id) project = find_project(id) + return forbidden! unless authorized_project_scope?(project) + return project if can?(current_user, :read_project, project) return unauthorized! if authenticate_non_public? not_found!('Project') end + def authorized_project_scope?(project) + return true unless job_token_authentication? + return true unless route_authentication_setting[:job_token_scope] == :project + + ::Feature.enabled?(:ci_job_token_scope, project, default_enabled: :yaml) && + current_authenticated_job.project == project + end + # rubocop: disable CodeReuse/ActiveRecord def find_group(id) if id.to_s =~ /^\d+$/ @@ -308,7 +318,7 @@ module API def verify_workhorse_api! Gitlab::Workhorse.verify_api_request!(request.headers) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e) forbidden! @@ -549,7 +559,7 @@ module API return unless Feature.enabled?(feature_name) Gitlab::UsageDataCounters.count(event_name) - rescue => error + rescue StandardError => error Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end @@ -559,7 +569,7 @@ module API return unless values.present? Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: values) - rescue => error + rescue StandardError => error Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end @@ -582,18 +592,26 @@ module API def project_finder_params_ce finder_params = project_finder_params_visibility_ce + + finder_params.merge!( + params + .slice(:search, + :custom_attributes, + :last_activity_after, + :last_activity_before, + :repository_storage) + .symbolize_keys + .compact + ) + finder_params[:with_issues_enabled] = true if params[:with_issues_enabled].present? finder_params[:with_merge_requests_enabled] = true if params[:with_merge_requests_enabled].present? finder_params[:without_deleted] = true - finder_params[:search] = params[:search] if params[:search] finder_params[:search_namespaces] = true if params[:search_namespaces].present? finder_params[:user] = params.delete(:user) if params[:user] - finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes] finder_params[:id_after] = sanitize_id_param(params[:id_after]) if params[:id_after] finder_params[:id_before] = sanitize_id_param(params[:id_before]) if params[:id_before] - finder_params[:last_activity_after] = params[:last_activity_after] if params[:last_activity_after] - finder_params[:last_activity_before] = params[:last_activity_before] if params[:last_activity_before] - finder_params[:repository_storage] = params[:repository_storage] if params[:repository_storage] + finder_params[:tag] = params[:topic] if params[:topic].present? finder_params end @@ -700,4 +718,4 @@ module API end end -API::Helpers.prepend_if_ee('EE::API::Helpers') +API::Helpers.prepend_mod_with('API::Helpers') diff --git a/lib/api/helpers/award_emoji.rb b/lib/api/helpers/award_emoji.rb new file mode 100644 index 00000000000..5b659c4dde7 --- /dev/null +++ b/lib/api/helpers/award_emoji.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module API + module Helpers + module AwardEmoji + def self.awardables + [ + { type: 'issue', resource: :projects, find_by: :iid, feature_category: :issue_tracking }, + { type: 'merge_request', resource: :projects, find_by: :iid, feature_category: :code_review }, + { type: 'snippet', resource: :projects, find_by: :id, feature_category: :snippets } + ] + end + + def self.awardable_id_desc + "The ID of an Issue, Merge Request or Snippet" + end + + # rubocop: disable CodeReuse/ActiveRecord + def awardable + @awardable ||= + begin + if params.include?(:note_id) + note_id = params.delete(:note_id) + + awardable.notes.find(note_id) + elsif params.include?(:issue_iid) + user_project.issues.find_by!(iid: params[:issue_iid]) + elsif params.include?(:merge_request_iid) + user_project.merge_requests.find_by!(iid: params[:merge_request_iid]) + elsif params.include?(:snippet_id) + user_project.snippets.find(params[:snippet_id]) + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end + +API::Helpers::AwardEmoji.prepend_mod_with('API::Helpers::AwardEmoji') diff --git a/lib/api/helpers/caching.rb b/lib/api/helpers/caching.rb index d0f22109879..f24ac7302c1 100644 --- a/lib/api/helpers/caching.rb +++ b/lib/api/helpers/caching.rb @@ -11,6 +11,11 @@ module API # @return [ActiveSupport::Duration] DEFAULT_EXPIRY = 1.day + # @return [Hash] + DEFAULT_CACHE_OPTIONS = { + race_condition_ttl: 5.seconds + }.freeze + # @return [ActiveSupport::Cache::Store] def cache Rails.cache @@ -40,7 +45,7 @@ module API # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @param presenter_args [Hash] keyword arguments to be passed to the entity # @return [Gitlab::Json::PrecompiledJson] - def present_cached(obj_or_collection, with:, cache_context: -> (_) { current_user.cache_key }, expires_in: DEFAULT_EXPIRY, **presenter_args) + def present_cached(obj_or_collection, with:, cache_context: -> (_) { current_user&.cache_key }, expires_in: DEFAULT_EXPIRY, **presenter_args) json = if obj_or_collection.is_a?(Enumerable) cached_collection( @@ -63,8 +68,59 @@ module API body Gitlab::Json::PrecompiledJson.new(json) end + # Action caching implementation + # + # This allows you to wrap an entire API endpoint call in a cache, useful + # for short TTL caches to effectively rate-limit an endpoint. The block + # will be converted to JSON and cached, and returns a + # `Gitlab::Json::PrecompiledJson` object which will be exported without + # secondary conversion. + # + # @param key [Object] any object that can be converted into a cache key + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [Gitlab::Json::PrecompiledJson] + def cache_action(key, **cache_opts) + json = cache.fetch(key, **apply_default_cache_options(cache_opts)) do + response = yield + + if response.is_a?(Gitlab::Json::PrecompiledJson) + response.to_s + else + Gitlab::Json.dump(response.as_json) + end + end + + body Gitlab::Json::PrecompiledJson.new(json) + end + + # Conditionally cache an action + # + # Perform a `cache_action` only if the conditional passes + def cache_action_if(conditional, *opts, **kwargs) + if conditional + cache_action(*opts, **kwargs) do + yield + end + else + yield + end + end + + # Conditionally cache an action + # + # Perform a `cache_action` unless the conditional passes + def cache_action_unless(conditional, *opts, **kwargs) + cache_action_if(!conditional, *opts, **kwargs) do + yield + end + end + private + def apply_default_cache_options(opts = {}) + DEFAULT_CACHE_OPTIONS.merge(opts) + end + # Optionally uses a `Proc` to add context to a cache key # # @param object [Object] must respond to #cache_key @@ -119,8 +175,11 @@ module API objs.flatten! map = multi_key_map(objs, context: context) - cache.fetch_multi(*map.keys, **kwargs) do |key| - yield map[key] + # TODO: `contextual_cache_key` should be constructed based on the guideline https://docs.gitlab.com/ee/development/redis.html#multi-key-commands. + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + cache.fetch_multi(*map.keys, **kwargs) do |key| + yield map[key] + end end end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index 8940cf87f82..02942820982 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -40,4 +40,4 @@ module API end end -API::Helpers::CommonHelpers.prepend_if_ee('EE::API::Helpers::CommonHelpers') +API::Helpers::CommonHelpers.prepend_mod_with('API::Helpers::CommonHelpers') diff --git a/lib/api/helpers/discussions_helpers.rb b/lib/api/helpers/discussions_helpers.rb index 3c0db1d0ea9..cb2feeda1e1 100644 --- a/lib/api/helpers/discussions_helpers.rb +++ b/lib/api/helpers/discussions_helpers.rb @@ -17,4 +17,4 @@ module API end end -API::Helpers::DiscussionsHelpers.prepend_if_ee('EE::API::Helpers::DiscussionsHelpers') +API::Helpers::DiscussionsHelpers.prepend_mod_with('API::Helpers::DiscussionsHelpers') diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index ba07a70ee32..5c5109f3d21 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -48,4 +48,4 @@ module API end end -API::Helpers::GroupsHelpers.prepend_if_ee('EE::API::Helpers::GroupsHelpers') +API::Helpers::GroupsHelpers.prepend_mod_with('API::Helpers::GroupsHelpers') diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb index 908c57bb04e..56445ccbd0d 100644 --- a/lib/api/helpers/headers_helpers.rb +++ b/lib/api/helpers/headers_helpers.rb @@ -8,7 +8,7 @@ module API def set_http_headers(header_data) header_data.each do |key, value| if value.is_a?(Enumerable) - raise ArgumentError.new("Header value should be a string") + raise ArgumentError, "Header value should be a string" end header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value.to_s diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 9a1ff2ba8ce..e03f029a6ef 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -65,7 +65,7 @@ module API result = Gitlab::Redis::SharedState.with { |redis| redis.ping } result == 'PONG' - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("GitLab: An unexpected error occurred in pinging to Redis: #{e}") false end diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index 2b1ed479692..b1954f8ece9 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -28,7 +28,8 @@ module API :remove_labels, :milestone_id, :state_event, - :title + :title, + :issue_type ] end @@ -47,6 +48,7 @@ module API args[:not][:label_name] ||= args[:not]&.delete(:labels) args[:scope] = args[:scope].underscore if args[:scope] args[:sort] = "#{args[:order_by]}_#{args[:sort]}" + args[:issue_types] ||= args.delete(:issue_type) IssuesFinder.new(current_user, args) end @@ -74,4 +76,4 @@ module API end end -API::Helpers::IssuesHelpers.prepend_if_ee('EE::API::Helpers::IssuesHelpers') +API::Helpers::IssuesHelpers.prepend_mod_with('API::Helpers::IssuesHelpers') diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index 4018f2dec21..796b8928243 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -5,27 +5,34 @@ module API module LabelHelpers extend Grape::API::Helpers + params :optional_label_params do + optional :description, type: String, desc: 'The description of the label' + optional :remove_on_close, type: Boolean, desc: 'Whether the label should be removed from an issue when the issue is closed' + end + params :label_create_params do requires :name, type: String, desc: 'The name of the label to be created' requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" - optional :description, type: String, desc: 'The description of label to be created' + + use :optional_label_params end params :label_update_params do optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" - optional :description, type: String, desc: 'The new description of label' + + use :optional_label_params end params :project_label_update_params do use :label_update_params optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true - at_least_one_of :new_name, :color, :description, :priority + at_least_one_of :new_name, :color, :description, :priority, :remove_on_close end params :group_label_update_params do use :label_update_params - at_least_one_of :new_name, :color, :description + at_least_one_of :new_name, :color, :description, :remove_on_close end def find_label(parent, id_or_title, params = { include_ancestor_groups: true }) @@ -117,7 +124,7 @@ module API else render_api_error!('Failed to promote project label to group label', 400) end - rescue => error + rescue StandardError => error render_api_error!(error.to_s, 400) end end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 2de077b5a3b..bd0c2501220 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -18,7 +18,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def retrieve_members(source, params:, deep: false) - members = deep ? find_all_members(source) : source_members(source).where.not(user_id: nil) + members = deep ? find_all_members(source) : source_members(source).connected_to_user members = members.includes(:user) members = members.references(:user).merge(User.search(params[:query])) if params[:query].present? members = members.where(user_id: params[:user_ids]) if params[:user_ids].present? @@ -65,4 +65,4 @@ module API end end -API::Helpers::MembersHelpers.prepend_if_ee('EE::API::Helpers::MembersHelpers') +API::Helpers::MembersHelpers.prepend_mod_with('API::Helpers::MembersHelpers') diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index cb938bc8a14..356e4a98c97 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -151,4 +151,4 @@ module API end end -API::Helpers::NotesHelpers.prepend_if_ee('EE::API::Helpers::NotesHelpers') +API::Helpers::NotesHelpers.prepend_mod_with('API::Helpers::NotesHelpers') diff --git a/lib/api/helpers/performance_bar_helpers.rb b/lib/api/helpers/performance_bar_helpers.rb index 8430e889dff..0b7fb4308fc 100644 --- a/lib/api/helpers/performance_bar_helpers.rb +++ b/lib/api/helpers/performance_bar_helpers.rb @@ -4,17 +4,17 @@ module API module Helpers module PerformanceBarHelpers def set_peek_enabled_for_current_request - Gitlab::SafeRequestStore.fetch(:peek_enabled) { perf_bar_cookie_enabled? && perf_bar_enabled_for_user? } + Gitlab::SafeRequestStore.fetch(:peek_enabled) { perf_bar_cookie_enabled? && perf_bar_allowed_for_user? } end def perf_bar_cookie_enabled? cookies[:perf_bar_enabled] == 'true' end - def perf_bar_enabled_for_user? + def perf_bar_allowed_for_user? # We cannot use `current_user` here because that method raises an exception when the user # is unauthorized and some API endpoints require that `current_user` is not called. - Gitlab::PerformanceBar.enabled_for_user?(find_user_from_sources) + Gitlab::PerformanceBar.allowed_for_user?(find_user_from_sources) end end end diff --git a/lib/api/helpers/project_snapshots_helpers.rb b/lib/api/helpers/project_snapshots_helpers.rb index e708dbf0156..0b10641571a 100644 --- a/lib/api/helpers/project_snapshots_helpers.rb +++ b/lib/api/helpers/project_snapshots_helpers.rb @@ -3,7 +3,7 @@ module API module Helpers module ProjectSnapshotsHelpers - prepend_if_ee('::EE::API::Helpers::ProjectSnapshotsHelpers') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::Helpers::ProjectSnapshotsHelpers') # rubocop: disable Cop/InjectEnterpriseEditionModule def authorize_read_git_snapshot! authenticated_with_can_read_all_resources! diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index cf2bcace33b..d9c0b4f67c8 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -170,4 +170,4 @@ module API end end -API::Helpers::ProjectsHelpers.prepend_if_ee('EE::API::Helpers::ProjectsHelpers') +API::Helpers::ProjectsHelpers.prepend_mod_with('API::Helpers::ProjectsHelpers') diff --git a/lib/api/helpers/protected_branches_helpers.rb b/lib/api/helpers/protected_branches_helpers.rb index 970a3687214..4a968ad1d60 100644 --- a/lib/api/helpers/protected_branches_helpers.rb +++ b/lib/api/helpers/protected_branches_helpers.rb @@ -12,4 +12,4 @@ module API end end -API::Helpers::ProtectedBranchesHelpers.prepend_if_ee('EE::API::Helpers::ProtectedBranchesHelpers') +API::Helpers::ProtectedBranchesHelpers.prepend_mod_with('API::Helpers::ProtectedBranchesHelpers') diff --git a/lib/api/helpers/related_resources_helpers.rb b/lib/api/helpers/related_resources_helpers.rb index 9cdde25fe4e..d0eda68bf52 100644 --- a/lib/api/helpers/related_resources_helpers.rb +++ b/lib/api/helpers/related_resources_helpers.rb @@ -23,10 +23,10 @@ module API # Using a blank component at the beginning of the join we ensure # that the resulted path will start with '/'. If the resulted path - # does not start with '/', URI::Generic#build will fail + # does not start with '/', URI::Generic#new will fail path_with_script_name = File.join('', [script_name, path].select(&:present?)) - URI::Generic.build(scheme: protocol, host: host, port: port, path: path_with_script_name).to_s + URI::Generic.new(protocol, nil, host, port, nil, path_with_script_name, nil, nil, nil, URI::RFC3986_PARSER, true).to_s end private diff --git a/lib/api/helpers/resource_label_events_helpers.rb b/lib/api/helpers/resource_label_events_helpers.rb index ad2733baffc..7e641130062 100644 --- a/lib/api/helpers/resource_label_events_helpers.rb +++ b/lib/api/helpers/resource_label_events_helpers.rb @@ -15,4 +15,4 @@ module API end end -API::Helpers::ResourceLabelEventsHelpers.prepend_if_ee('EE::API::Helpers::ResourceLabelEventsHelpers') +API::Helpers::ResourceLabelEventsHelpers.prepend_mod_with('API::Helpers::ResourceLabelEventsHelpers') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 688cd2da994..6f25cf507bc 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -5,7 +5,7 @@ module API module Runner include Gitlab::Utils::StrongMemoize - prepend_if_ee('EE::API::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' JOB_TOKEN_PARAM = :token @@ -87,6 +87,10 @@ module API project: -> { current_job.project } ) end + + def track_ci_minutes_usage!(_build, _runner) + # noop: overridden in EE + end end end end diff --git a/lib/api/helpers/search_helpers.rb b/lib/api/helpers/search_helpers.rb index cb5f92fa62a..66321306496 100644 --- a/lib/api/helpers/search_helpers.rb +++ b/lib/api/helpers/search_helpers.rb @@ -25,4 +25,4 @@ module API end end -API::Helpers::SearchHelpers.prepend_if_ee('EE::API::Helpers::SearchHelpers') +API::Helpers::SearchHelpers.prepend_mod_with('API::Helpers::SearchHelpers') diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 2f2ad88c942..d123db8e3df 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -420,44 +420,6 @@ module API }, chat_notification_events ].flatten, - 'hipchat' => [ - { - required: true, - name: :token, - type: String, - desc: 'The room token' - }, - { - required: false, - name: :room, - type: String, - desc: 'The room name or ID' - }, - { - required: false, - name: :color, - type: String, - desc: 'The room color' - }, - { - required: false, - name: :notify, - type: Boolean, - desc: 'Enable notifications' - }, - { - required: false, - name: :api_version, - type: String, - desc: 'Leave blank for default (v2)' - }, - { - required: false, - name: :server, - type: String, - desc: 'Leave blank for default. https://hipchat.example.com' - } - ], 'irker' => [ { required: true, @@ -803,7 +765,7 @@ module API required: true, name: :webhook, type: String, - desc: 'The Webex Teams webhook. e.g. https://api.ciscospark.com/v1/webhooks/incoming/…' + desc: 'The Webex Teams webhook. For example, https://api.ciscospark.com/v1/webhooks/incoming/...' }, chat_notification_events ].flatten @@ -812,23 +774,22 @@ module API def self.service_classes [ - ::AsanaService, - ::AssemblaService, - ::BambooService, + ::Integrations::Asana, + ::Integrations::Assembla, + ::Integrations::Bamboo, + ::Integrations::Campfire, + ::Integrations::Confluence, + ::Integrations::Datadog, + ::Integrations::EmailsOnPush, ::BugzillaService, ::BuildkiteService, - ::ConfluenceService, - ::CampfireService, ::CustomIssueTrackerService, - ::DatadogService, ::DiscordService, ::DroneCiService, - ::EmailsOnPushService, ::EwmService, ::ExternalWikiService, ::FlowdockService, ::HangoutsChatService, - ::HipchatService, ::IrkerService, ::JenkinsService, ::JiraService, @@ -858,4 +819,4 @@ module API end end -API::Helpers::ServicesHelpers.prepend_if_ee('EE::API::Helpers::ServicesHelpers') +API::Helpers::ServicesHelpers.prepend_mod_with('API::Helpers::ServicesHelpers') diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb index 451e578fdd6..a3ea1057bc8 100644 --- a/lib/api/helpers/settings_helpers.rb +++ b/lib/api/helpers/settings_helpers.rb @@ -19,4 +19,4 @@ module API end end -API::Helpers::SettingsHelpers.prepend_if_ee('EE::API::Helpers::SettingsHelpers') +API::Helpers::SettingsHelpers.prepend_mod_with('API::Helpers::SettingsHelpers') diff --git a/lib/api/helpers/users_helpers.rb b/lib/api/helpers/users_helpers.rb index 2d7b22e66b3..1a019283bc6 100644 --- a/lib/api/helpers/users_helpers.rb +++ b/lib/api/helpers/users_helpers.rb @@ -22,4 +22,4 @@ module API end end -API::Helpers::UsersHelpers.prepend_if_ee('EE::API::Helpers::UsersHelpers') +API::Helpers::UsersHelpers.prepend_mod_with('API::Helpers::UsersHelpers') diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb index e2b3372fc33..edbdcb257e7 100644 --- a/lib/api/helpers/variables_helpers.rb +++ b/lib/api/helpers/variables_helpers.rb @@ -24,4 +24,4 @@ module API end end -API::Helpers::VariablesHelpers.prepend_if_ee('EE::API::Helpers::VariablesHelpers') +API::Helpers::VariablesHelpers.prepend_mod_with('API::Helpers::VariablesHelpers') diff --git a/lib/api/helpers/wikis_helpers.rb b/lib/api/helpers/wikis_helpers.rb index 49da1e317ab..4a14dc1f40a 100644 --- a/lib/api/helpers/wikis_helpers.rb +++ b/lib/api/helpers/wikis_helpers.rb @@ -32,4 +32,4 @@ module API end end -API::Helpers::WikisHelpers.prepend_if_ee('EE::API::Helpers::WikisHelpers') +API::Helpers::WikisHelpers.prepend_mod_with('API::Helpers::WikisHelpers') diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 4dcfc0cf7eb..e16149185c9 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -158,7 +158,7 @@ module API status 200 unless actor.key_or_user - raise ActiveRecord::RecordNotFound.new('User not found!') + raise ActiveRecord::RecordNotFound, 'User not found!' end actor.update_last_used_at! @@ -336,4 +336,4 @@ module API end end -API::Internal::Base.prepend_if_ee('EE::API::Internal::Base') +API::Internal::Base.prepend_mod_with('API::Internal::Base') diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index af2c53dd778..c28e2181873 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -107,18 +107,18 @@ module API detail 'Updates usage metrics for agent' end params do - requires :gitops_sync_count, type: Integer, desc: 'The count to increment the gitops_sync metric by' + optional :gitops_sync_count, type: Integer, desc: 'The count to increment the gitops_sync metric by' + optional :k8s_api_proxy_request_count, type: Integer, desc: 'The count to increment the k8s_api_proxy_request_count metric by' end post '/' do - gitops_sync_count = params[:gitops_sync_count] + events = params.slice(:gitops_sync_count, :k8s_api_proxy_request_count) + events.transform_keys! { |event| event.to_s.chomp('_count') } - if gitops_sync_count < 0 - bad_request!('gitops_sync_count must be greater than or equal to zero') - else - Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_gitops_sync(gitops_sync_count) + Gitlab::UsageDataCounters::KubernetesAgentCounter.increment_event_counts(events) - no_content! - end + no_content! + rescue ArgumentError => e + bad_request!(e.message) end end end @@ -126,4 +126,4 @@ module API end end -API::Internal::Kubernetes.prepend_if_ee('EE::API::Internal::Kubernetes') +API::Internal::Kubernetes.prepend_mod_with('API::Internal::Kubernetes') diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 1cd5bde224b..0b4f4e06d0b 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -21,12 +21,12 @@ module API related_issues = source_issue.related_issues(current_user) do |issues| issues.with_api_entity_associations.preload_awardable end - related_issues.each { |issue| issue.lazy_subscription(current_user, user_project) } # preload subscriptions present related_issues, with: Entities::RelatedIssue, current_user: current_user, - project: user_project + project: user_project, + include_subscribed: false end desc 'Relate issues' do diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c844655f0b3..355b5ed3a1f 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -74,6 +74,7 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '', desc: 'Return issues that have no due date (`0`), or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`, `0`' + optional :issue_type, type: String, values: Issue.issue_types.keys, desc: "The type of the issue. Accepts: #{Issue.issue_types.keys.join(', ')}" use :issues_stats_params use :pagination @@ -90,6 +91,7 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" + optional :issue_type, type: String, values: Issue.issue_types.keys, desc: "The type of the issue. Accepts: #{Issue.issue_types.keys.join(', ')}" use :optional_issue_params_ee end @@ -253,9 +255,9 @@ module API issue_params = convert_parameters_from_legacy_format(issue_params) begin - issue = ::Issues::CreateService.new(user_project, - current_user, - issue_params.merge(request: request, api: true)).execute + issue = ::Issues::CreateService.new(project: user_project, + current_user: current_user, + params: issue_params.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) @@ -296,9 +298,9 @@ module API update_params = convert_parameters_from_legacy_format(update_params) - issue = ::Issues::UpdateService.new(user_project, - current_user, - update_params).execute(issue) + issue = ::Issues::UpdateService.new(project: user_project, + current_user: current_user, + params: update_params).execute(issue) render_spam_error! if issue.spam? @@ -326,7 +328,7 @@ module API authorize! :update_issue, issue - if ::Issues::ReorderService.new(user_project, current_user, params).execute(issue) + if ::Issues::ReorderService.new(project: user_project, current_user: current_user, params: params).execute(issue) present issue, with: Entities::Issue, current_user: current_user, project: user_project else render_api_error!({ error: 'Unprocessable Entity' }, 422) @@ -352,7 +354,7 @@ module API not_found!('Project') unless new_project begin - issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + issue = ::Issues::MoveService.new(project: user_project, current_user: current_user).execute(issue, new_project) present issue, with: Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) @@ -372,7 +374,7 @@ module API authorize!(:destroy_issue, issue) destroy_conditionally!(issue) do |issue| - Issuable::DestroyService.new(user_project, current_user).execute(issue) + Issuable::DestroyService.new(project: user_project, current_user: current_user).execute(issue) end end # rubocop: enable CodeReuse/ActiveRecord @@ -386,7 +388,7 @@ module API get ':id/issues/:issue_iid/related_merge_requests' do issue = find_project_issue(params[:issue_iid]) - merge_requests = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) + merge_requests = ::Issues::ReferencedMergeRequestsService.new(project: user_project, current_user: current_user) .execute(issue) .first @@ -446,4 +448,4 @@ module API end end -API::Issues.prepend_if_ee('EE::API::Issues') +API::Issues.prepend_mod_with('API::Issues') diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 3dec0a29181..37199279205 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -13,7 +13,7 @@ module API end end - prepend_if_ee('EE::API::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 54951f9bd01..cf65bfdfd0e 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -202,4 +202,4 @@ module API end end -API::Jobs.prepend_if_ee('EE::API::Jobs') +API::Jobs.prepend_mod_with('API::Jobs') diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index bd1d984719e..22f7b07809b 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -25,7 +25,7 @@ module API helpers do def path_exists?(path) # return true when FF disabled so that processing the request is not stopped - return true unless Feature.enabled?(:check_maven_path_first) + return true unless Feature.enabled?(:check_maven_path_first, default_enabled: :yaml) return false if path.blank? Packages::Maven::Metadatum.with_path(path) @@ -88,17 +88,13 @@ module API end def fetch_package(file_name:, project: nil, group: nil) - order_by_package_file = false - if Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml) - order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) && - !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) - end + order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) && + !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) ::Packages::Maven::PackageFinder.new( - params[:path], current_user, - project: project, - group: group, + project || group, + path: params[:path], order_by_package_file: order_by_package_file ).execute! end diff --git a/lib/api/members.rb b/lib/api/members.rb index aaf0e3e1927..a1a733ea7ae 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -175,4 +175,4 @@ module API end end -API::Members.prepend_if_ee('EE::API::Members') +API::Members.prepend_mod_with('API::Members') diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 0cdfd8f94b4..83150bb51ca 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -54,7 +54,7 @@ module API success = ::MergeRequests::ApprovalService - .new(user_project, current_user, params) + .new(project: user_project, current_user: current_user, params: params) .execute(merge_request) unauthorized! unless success @@ -67,7 +67,7 @@ module API merge_request = find_merge_request_with_access(params[:merge_request_iid], :approve_merge_request) success = ::MergeRequests::RemoveApprovalService - .new(user_project, current_user) + .new(project: user_project, current_user: current_user) .execute(merge_request) not_found! unless success @@ -79,4 +79,4 @@ module API end end -API::MergeRequestApprovals.prepend_if_ee('EE::API::MergeRequestApprovals') +API::MergeRequestApprovals.prepend_mod_with('API::MergeRequestApprovals') diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 613de514ffa..931d2322c98 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -52,7 +52,7 @@ module API ] end - prepend_if_ee('EE::API::MergeRequests') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::MergeRequests') # rubocop: disable Cop/InjectEnterpriseEditionModule helpers do # rubocop: disable CodeReuse/ActiveRecord @@ -201,7 +201,11 @@ module API options = serializer_options_for(merge_requests).merge(project: user_project) options[:project] = user_project - present merge_requests, options + if Feature.enabled?(:api_caching_merge_requests, user_project, type: :development, default_enabled: :yaml) + present_cached merge_requests, expires_in: 10.minutes, **options + else + present merge_requests, options + end end desc 'Create a merge request' do @@ -224,7 +228,7 @@ module API mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) mr_params = convert_parameters_from_legacy_format(mr_params) - merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute + merge_request = ::MergeRequests::CreateService.new(project: user_project, current_user: current_user, params: mr_params).execute handle_merge_request_errors!(merge_request) @@ -243,7 +247,7 @@ module API authorize!(:destroy_merge_request, merge_request) destroy_conditionally!(merge_request) do |merge_request| - Issuable::DestroyService.new(user_project, current_user).execute(merge_request) + Issuable::DestroyService.new(project: user_project, current_user: current_user).execute(merge_request) end end @@ -335,7 +339,7 @@ module API authorize!(:update_merge_request, merge_request) project = merge_request.target_project - result = ::MergeRequests::AddContextService.new(project, current_user, merge_request: merge_request, commits: commit_ids).execute + result = ::MergeRequests::AddContextService.new(project: project, current_user: current_user, params: { merge_request: merge_request, commits: commit_ids }).execute if result.instance_of?(Array) present result, with: Entities::Commit @@ -398,7 +402,7 @@ module API end post ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do pipeline = ::MergeRequests::CreatePipelineService - .new(user_project, current_user, allow_duplicate: true) + .new(project: user_project, current_user: current_user, params: { allow_duplicate: true }) .execute(find_merge_request_with_access(params[:merge_request_iid])) if pipeline.nil? @@ -439,7 +443,7 @@ module API ::MergeRequests::UpdateService end - merge_request = service.new(user_project, current_user, mr_params).execute(merge_request) + merge_request = service.new(project: user_project, current_user: current_user, params: mr_params).execute(merge_request) handle_merge_request_errors!(merge_request) @@ -489,7 +493,7 @@ module API if immediately_mergeable ::MergeRequests::MergeService - .new(merge_request.target_project, current_user, merge_params) + .new(project: merge_request.target_project, current_user: current_user, params: merge_params) .execute(merge_request) elsif automatically_mergeable AutoMergeService.new(merge_request.target_project, current_user, merge_params) diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 465d2f23e9d..9d41c2f148f 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -19,7 +19,7 @@ module API end end - prepend_if_ee('EE::API::Namespaces') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('API::Namespaces') # rubocop: disable Cop/InjectEnterpriseEditionModule resource :namespaces do desc 'Get a namespaces list' do diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 4a33f3e8af2..6d0c1f44a36 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -30,6 +30,29 @@ module API present paginate(package.package_files), with: ::API::Entities::PackageFile end + + desc 'Remove a package file' do + detail 'This feature was introduced in GitLab 13.12' + end + params do + requires :package_file_id, type: Integer, desc: 'The ID of a package file' + end + delete ':id/packages/:package_id/package_files/:package_file_id' do + authorize_destroy_package!(user_project) + + # We want to make sure the file belongs to the declared package + # so we look up the package before looking up the file. + package = ::Packages::PackageFinder + .new(user_project, params[:package_id]).execute + + not_found! unless package + + package_file = package.package_files.find_by_id(params[:package_file_id]) + + not_found! unless package_file + + destroy_conditionally!(package_file) + end end end end diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 3125de88de5..2580f7adbc9 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -15,6 +15,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end + route_setting :authentication, job_token_allowed: true, job_token_scope: :project resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project container repositories' do detail 'This feature was introduced in GitLab 11.8.' diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 5f3a574eeee..039f7b4be41 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -14,6 +14,21 @@ module API def import_params declared_params(include_missing: false) end + + def namespace_from(params, current_user) + if params[:namespace] + find_namespace!(params[:namespace]) + else + current_user.namespace + end + end + + def filtered_override_params(params) + override_params = params.delete(:override_params) + filter_attributes_using_license!(override_params) if override_params + + override_params + end end before do @@ -67,34 +82,25 @@ module API check_rate_limit! :project_import, [current_user, :project_import] - Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21041') validate_file! - namespace = if import_params[:namespace] - find_namespace!(import_params[:namespace]) - else - current_user.namespace - end - - project_params = { - path: import_params[:path], - namespace_id: namespace.id, - name: import_params[:name], - file: import_params[:file], - overwrite: import_params[:overwrite] - } - - override_params = import_params.delete(:override_params) - filter_attributes_using_license!(override_params) if override_params - - project = ::Projects::GitlabProjectsImportService.new( - current_user, project_params, override_params + response = ::Import::GitlabProjects::CreateProjectFromUploadedFileService.new( + current_user, + path: import_params[:path], + namespace: namespace_from(import_params, current_user), + name: import_params[:name], + file: import_params[:file], + overwrite: import_params[:overwrite], + override: filtered_override_params(import_params) ).execute - render_api_error!(project.errors.full_messages&.first, 400) unless project.saved? - - present project, with: Entities::ProjectImportStatus + if response.success? + present(response.payload, with: Entities::ProjectImportStatus) + else + render_api_error!(response.message, response.http_status) + end end params do @@ -107,6 +113,44 @@ module API get ':id/import' do present user_project, with: Entities::ProjectImportStatus end + + params do + requires :url, type: String, desc: 'The URL for the file.' + requires :path, type: String, desc: 'The new project path and name' + optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.' + optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." + optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' + optional :override_params, + type: Hash, + desc: 'New project params to override values in the export' do + use :optional_project_params + end + end + desc 'Create a new project import using a remote object storage path' do + detail 'This feature was introduced in GitLab 13.2.' + success Entities::ProjectImportStatus + end + post 'remote-import' do + not_found! unless ::Feature.enabled?(:import_project_from_remote_file) + + check_rate_limit! :project_import, [current_user, :project_import] + + response = ::Import::GitlabProjects::CreateProjectFromRemoteFileService.new( + current_user, + path: import_params[:path], + namespace: namespace_from(import_params, current_user), + name: import_params[:name], + remote_import_url: import_params[:url], + overwrite: import_params[:overwrite], + override: filtered_override_params(import_params) + ).execute + + if response.success? + present(response.payload, with: Entities::ProjectImportStatus) + else + render_api_error!(response.message, response.http_status) + end + end end end end diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 8675de33923..107311ea446 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -117,4 +117,4 @@ module API end end -API::ProjectMilestones.prepend_if_ee('EE::API::ProjectMilestones') +API::ProjectMilestones.prepend_mod_with('API::ProjectMilestones') diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index fdfdc244cbe..5d6f67ccbae 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -4,7 +4,7 @@ module API class ProjectTemplates < ::API::Base include PaginationParams - TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls gitlab_ci_syntax_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze + TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze # The regex is needed to ensure a period (e.g. agpl-3.0) # isn't confused with a format type. We also need to allow encoded # values (e.g. C%2B%2B for C++), so allow % and + as well. @@ -16,7 +16,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' - requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|gitlab_ci_syntax_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' + requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of templates available to this project' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 92f6970e6fc..4e8786fbe1f 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require_dependency 'declarative_policy' - module API class Projects < ::API::Base include PaginationParams @@ -119,6 +117,7 @@ module API optional :last_activity_after, type: DateTime, desc: 'Limit results to projects with last_activity after specified time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :last_activity_before, type: DateTime, desc: 'Limit results to projects with last_activity before specified time. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins' + optional :topic, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of topics. Limit results to projects having all topics' use :optional_filter_params_ee end @@ -619,6 +618,8 @@ module API optional :skip_groups, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Array of group ids to exclude from list' optional :with_shared, type: Boolean, default: false, desc: 'Include shared groups' + optional :shared_visible_only, type: Boolean, default: false, + desc: 'Limit to shared groups user has access to' optional :shared_min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit returned shared groups by minimum access level to the project' use :pagination @@ -663,4 +664,4 @@ module API end end -API::Projects.prepend_if_ee('EE::API::Projects') +API::Projects.prepend_mod_with('API::Projects') diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 802dfdec511..3cebc308f51 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -104,4 +104,4 @@ module API end end -API::ProtectedBranches.prepend_if_ee('EE::API::ProtectedBranches') +API::ProtectedBranches.prepend_mod_with('API::ProtectedBranches') diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 658c6d13847..73b2f658825 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -24,25 +24,6 @@ module API render_api_error!(e.message, 400) end - helpers do - def packages_finder(project = authorized_user_project) - project - .packages - .pypi - .has_version - .processed - end - - def find_package_versions - packages = packages_finder - .with_normalized_pypi_name(params[:package_name]) - - not_found!('Package') if packages.empty? - - packages - end - end - before do require_packages_enabled! end @@ -71,7 +52,7 @@ module API project = unauthorized_user_project! filename = "#{params[:file_identifier]}.#{params[:format]}" - package = packages_finder(project).by_file_name_and_sha256(filename, params[:sha256]) + package = Packages::Pypi::PackageFinder.new(current_user, project, { filename: filename, sha256: params[:sha256] }).execute package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute track_package_event('pull_package', :pypi) @@ -95,7 +76,7 @@ module API track_package_event('list_package', :pypi) - packages = find_package_versions + packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute! presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) # Adjusts grape output format diff --git a/lib/api/releases.rb b/lib/api/releases.rb index c20e618efd1..c65a23e334f 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -6,9 +6,12 @@ module API RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS .merge(tag_name: API::NO_SLASH_URL_PART_REGEX) + RELEASE_CLI_USER_AGENT = 'GitLab-release-cli' before { authorize_read_releases! } + after { track_release_event } + feature_category :release_orchestration params do @@ -17,6 +20,7 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a project releases' do detail 'This feature was introduced in GitLab 11.7.' + named 'get_releases' success Entities::Release end params do @@ -29,11 +33,22 @@ module API get ':id/releases' do releases = ::ReleasesFinder.new(user_project, current_user, declared_params.slice(:order_by, :sort)).execute - present paginate(releases), with: Entities::Release, current_user: current_user + # We cache the serialized payload per user in order to avoid repeated renderings. + # Since the cached result could contain sensitive information, + # it will expire in a short interval. + present_cached paginate(releases), + with: Entities::Release, + # `current_user` could be absent if the releases are publicly accesible. + # We should not use `cache_key` for the user because the version/updated_at + # context is unnecessary here. + cache_context: -> (_) { "user:{#{current_user&.id}}" }, + expires_in: 5.minutes, + current_user: current_user end desc 'Get a single project release' do detail 'This feature was introduced in GitLab 11.7.' + named 'get_release' success Entities::Release end params do @@ -47,6 +62,7 @@ module API desc 'Create a new release' do detail 'This feature was introduced in GitLab 11.7.' + named 'create_release' success Entities::Release end params do @@ -84,6 +100,7 @@ module API desc 'Update a release' do detail 'This feature was introduced in GitLab 11.7.' + named 'update_release' success Entities::Release end params do @@ -112,6 +129,7 @@ module API desc 'Delete a release' do detail 'This feature was introduced in GitLab 11.7.' + named 'delete_release' success Entities::Release end params do @@ -176,8 +194,23 @@ module API def log_release_milestones_updated_audit_event # extended in EE end + + def release_cli? + request.env['HTTP_USER_AGENT']&.include?(RELEASE_CLI_USER_AGENT) == true + end + + def event_context + { + release_cli: release_cli? + } + end + + def track_release_event + Gitlab::Tracking.event(options[:for].name, options[:route_options][:named], + project: user_project, user: current_user, **event_context) + end end end end -API::Releases.prepend_if_ee('EE::API::Releases') +API::Releases.prepend_mod_with('API::Releases') diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 033cc6744b0..a5234828de3 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -37,7 +37,7 @@ module API begin @blob = Gitlab::Git::Blob.raw(@repo, params[:sha]) @blob.load_all_data!(@repo) - rescue + rescue StandardError not_found! 'Blob' end @@ -106,7 +106,7 @@ module API not_acceptable! if Gitlab::HotlinkingDetector.intercept_hotlinking?(request) send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true - rescue + rescue StandardError not_found!('File') end @@ -152,7 +152,7 @@ module API get ':id/repository/contributors' do contributors = ::Kaminari.paginate_array(user_project.repository.contributors(order_by: params[:order_by], sort: params[:sort])) present paginate(contributors), with: Entities::Contributor - rescue + rescue StandardError not_found! end @@ -224,7 +224,7 @@ module API desc: 'The commit message to use when committing the changelog' end post ':id/repository/changelog' do - branch = params[:branch] || user_project.default_branch_or_master + branch = params[:branch] || user_project.default_branch_or_main access = Gitlab::UserAccess.new(current_user, container: user_project) unless access.can_push_to_branch?(branch) diff --git a/lib/api/search.rb b/lib/api/search.rb index 8fabf379d49..3c5801366a8 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -138,4 +138,4 @@ module API end end -API::Search.prepend_if_ee('EE::API::Search') +API::Search.prepend_mod_with('API::Search') diff --git a/lib/api/services.rb b/lib/api/services.rb index cfcae13e518..8a7abe721dd 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -72,7 +72,7 @@ module API success Entities::ProjectServiceBasic end get ":id/services" do - services = user_project.services.active + services = user_project.integrations.active present services, with: Entities::ProjectServiceBasic end @@ -125,15 +125,18 @@ module API requires :service_slug, type: String, values: SERVICES.keys, desc: 'The name of the service' end get ":id/services/:service_slug" do - service = user_project.find_or_initialize_service(params[:service_slug].underscore) - present service, with: Entities::ProjectService + integration = user_project.find_or_initialize_service(params[:service_slug].underscore) + + not_found!('Service') unless integration&.persisted? + + present integration, with: Entities::ProjectService end end TRIGGER_SERVICES.each do |service_slug, settings| helpers do def slash_command_service(project, service_slug, params) - project.services.active.find do |service| + project.integrations.active.find do |service| service.try(:token) == params[:token] && service.to_param == service_slug.underscore end end @@ -172,4 +175,4 @@ module API end end -API::Services.prepend_if_ee('EE::API::Services') +API::Services.prepend_mod_with('API::Services') diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 95d0c525ced..372bc7b3d8f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -154,6 +154,7 @@ module API optional :spam_check_endpoint_enabled, type: Boolean, desc: 'Enable Spam Check via external API endpoint' given spam_check_endpoint_enabled: ->(val) { val } do requires :spam_check_endpoint_url, type: String, desc: 'The URL of the external Spam Check service endpoint' + requires :spam_check_api_key, type: String, desc: 'The API key used by GitLab for accessing the Spam Check service endpoint' end optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.' @@ -169,6 +170,8 @@ module API optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute." optional :wiki_page_max_content_bytes, type: Integer, desc: "Maximum wiki page content size in bytes" optional :require_admin_approval_after_user_signup, type: Boolean, desc: 'Require explicit admin approval for new signups' + optional :whats_new_variant, type: String, values: ApplicationSetting.whats_new_variants.keys, desc: "What's new variant, possible values: `all_tiers`, `current_tier`, and `disabled`." + optional :floc_enabled, type: Grape::API::Boolean, desc: 'Enable FloC (Federated Learning of Cohorts)' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", @@ -232,4 +235,4 @@ module API end end -API::Settings.prepend_if_ee('EE::API::Settings') +API::Settings.prepend_mod_with('API::Settings') diff --git a/lib/api/templates.rb b/lib/api/templates.rb index bc1e427bcaa..b7fb35eac03 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -13,9 +13,6 @@ module API gitlab_ci_ymls: { gitlab_version: 8.9 }, - gitlab_ci_syntax_ymls: { - gitlab_version: 13.8 - }, dockerfiles: { gitlab_version: 8.15 } diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb new file mode 100644 index 00000000000..34e77e09800 --- /dev/null +++ b/lib/api/terraform/modules/v1/packages.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module API + module Terraform + module Modules + module V1 + class Packages < ::API::Base + include ::API::Helpers::Authentication + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + + SEMVER_REGEX = Gitlab::Regex.semver_regex + + TERRAFORM_MODULE_REQUIREMENTS = { + module_namespace: API::NO_SLASH_URL_PART_REGEX, + module_name: API::NO_SLASH_URL_PART_REGEX, + module_system: API::NO_SLASH_URL_PART_REGEX + }.freeze + + TERRAFORM_MODULE_VERSION_REQUIREMENTS = { + module_version: SEMVER_REGEX + }.freeze + + feature_category :package_registry + + after_validation do + require_packages_enabled! + end + + helpers do + params :module_name do + requires :module_name, type: String, desc: "", regexp: API::NO_SLASH_URL_PART_REGEX + requires :module_system, type: String, regexp: API::NO_SLASH_URL_PART_REGEX + end + + params :module_version do + requires :module_version, type: String, desc: 'Module version', regexp: SEMVER_REGEX + end + + def module_namespace + strong_memoize(:module_namespace) do + find_namespace(params[:module_namespace]) + end + end + + def finder_params + { + package_type: :terraform_module, + package_name: "#{params[:module_name]}/#{params[:module_system]}" + }.tap do |finder_params| + finder_params[:package_version] = params[:module_version] if params.has_key?(:module_version) + end + end + + def packages + strong_memoize(:packages) do + ::Packages::GroupPackagesFinder.new( + current_user, + module_namespace, + finder_params + ).execute + end + end + + def package + strong_memoize(:package) do + packages.first + end + end + + def package_file + strong_memoize(:package_file) do + package.package_files.first + end + end + end + + params do + requires :module_namespace, type: String, desc: "Group's ID or slug", regexp: API::NO_SLASH_URL_PART_REGEX + includes :module_name + end + + namespace 'packages/terraform/modules/v1/:module_namespace/:module_name/:module_system', requirements: TERRAFORM_MODULE_REQUIREMENTS do + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_bearer_token) + end + + after_validation do + authorize_read_package!(package || module_namespace) + end + + get 'versions' do + presenter = ::Terraform::ModulesPresenter.new(packages, params[:module_system]) + present presenter, with: ::API::Entities::Terraform::ModuleVersions + end + + params do + includes :module_version + end + + namespace '*module_version', requirements: TERRAFORM_MODULE_VERSION_REQUIREMENTS do + after_validation do + not_found! unless package && package_file + end + + get 'download' do + module_file_path = api_v4_packages_terraform_modules_v1_module_version_file_path( + module_namespace: params[:module_namespace], + module_name: params[:module_name], + module_system: params[:module_system], + module_version: params[:module_version] + ) + + jwt_token = Gitlab::TerraformRegistryToken.from_token(token_from_namespace_inheritable).encoded + + header 'X-Terraform-Get', module_file_path.sub(%r{module_version/file$}, "#{params[:module_version]}/file?token=#{jwt_token}&archive=tgz") + status :no_content + end + + namespace 'file' do + authenticate_with do |accept| + accept.token_types(:deploy_token_from_jwt, :job_token_from_jwt, :personal_access_token_from_jwt).sent_through(:token_param) + end + + get do + track_package_event('pull_package', :terraform_module) + + present_carrierwave_file!(package_file.file) + end + end + end + end + + params do + requires :id, type: String, desc: 'The ID or full path of a project' + includes :module_name + includes :module_version + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/terraform/modules/:module_name/:module_system/*module_version/file' do + authenticate_with do |accept| + accept.token_types(:deploy_token).sent_through(:http_deploy_token_header) + accept.token_types(:job_token).sent_through(:http_job_token_header) + accept.token_types(:personal_access_token).sent_through(:http_private_token_header) + end + + desc 'Workhorse authorize Terraform Module package file' do + detail 'This feature was introduced in GitLab 13.11' + end + + put 'authorize' do + authorize_workhorse!( + subject: authorized_user_project, + maximum_size: authorized_user_project.actual_limits.terraform_module_max_file_size + ) + end + + desc 'Upload Terraform Module package file' do + detail 'This feature was introduced in GitLab 13.11' + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + put do + authorize_upload!(authorized_user_project) + bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:terraform_module_max_file_size, params[:file].size) + + create_package_file_params = { + module_name: params['module_name'], + module_system: params['module_system'], + module_version: params['module_version'], + file: params['file'], + build: current_authenticated_job + } + + result = ::Packages::TerraformModule::CreatePackageService + .new(authorized_user_project, current_user, create_package_file_params) + .execute + + render_api_error!(result[:message], result[:http_status]) if result[:status] == :error + + track_package_event('push_package', :terraform_module) + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) + + forbidden! + end + end + end + end + end + end + end +end diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index da234fb5277..969122d7906 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -37,7 +37,7 @@ module API custom_params = declared_params(include_missing: false) custom_params.merge!(attrs) - issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) + issuable = update_service.new(project: user_project, current_user: current_user, params: custom_params).execute(load_issuable) if issuable.valid? present issuable, with: Entities::IssuableTimeStats else @@ -85,10 +85,15 @@ module API post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do authorize! admin_issuable_key, load_issuable - update_issuable(spend_time: { - duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), - user_id: current_user.id - }) + update_params = { + spend_time: { + duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + user_id: current_user.id + } + } + update_params[:use_specialized_service] = true if issuable_name == 'merge_request' + + update_issuable(update_params) end desc "Reset spent time for a project #{issuable_name}" diff --git a/lib/api/todos.rb b/lib/api/todos.rb index afc1525cbe2..a001313a11f 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -79,7 +79,7 @@ module API next unless collection targets = collection.map(&:target) - options[type] = { issuable_metadata: Gitlab::IssuableMetadata.new(current_user, targets).data } + options[type] = { issuable_metadata: Gitlab::IssuableMetadata.new(current_user, targets).data, include_subscribed: false } end end end @@ -124,4 +124,4 @@ module API end end -API::Todos.prepend_if_ee('EE::API::Todos') +API::Todos.prepend_mod_with('API::Todos') diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 84c51e5aeac..a359083a9d2 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -37,7 +37,7 @@ module API result = ::Ci::PipelineTriggerService.new(project, nil, params).execute not_found! unless result - if result[:http_status] + if result.error? render_api_error!(result[:message], result[:http_status]) else present result[:pipeline], with: Entities::Ci::Pipeline diff --git a/lib/api/users.rb b/lib/api/users.rb index 078ba7542a3..565a3544da2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -996,6 +996,30 @@ module API present paginate(current_user.emails), with: Entities::Email end + desc "Update a user's credit_card_validation" do + success Entities::UserCreditCardValidations + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + requires :credit_card_validated_at, type: DateTime, desc: 'The time when the user\'s credit card was validated' + end + put ":user_id/credit_card_validation", feature_category: :users do + authenticated_as_admin! + + user = find_user(params[:user_id]) + not_found!('User') unless user + + attrs = declared_params(include_missing: false) + + service = ::Users::UpsertCreditCardValidationService.new(attrs).execute + + if service.success? + present user.credit_card_validation, with: Entities::UserCreditCardValidations + else + render_api_error!('400 Bad Request', 400) + end + end + desc "Update the current user's preferences" do success Entities::UserPreferences detail 'This feature was introduced in GitLab 13.10.' diff --git a/lib/api/validations/validators/check_assignees_count.rb b/lib/api/validations/validators/check_assignees_count.rb index 92ada159b46..15f48c09a4f 100644 --- a/lib/api/validations/validators/check_assignees_count.rb +++ b/lib/api/validations/validators/check_assignees_count.rb @@ -34,4 +34,4 @@ module API end end -API::Validations::Validators::CheckAssigneesCount.prepend_if_ee('EE::API::Validations::Validators::CheckAssigneesCount') +API::Validations::Validators::CheckAssigneesCount.prepend_mod_with('API::Validations::Validators::CheckAssigneesCount') diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb index a6a3c692fd6..246c445658f 100644 --- a/lib/api/validations/validators/file_path.rb +++ b/lib/api/validations/validators/file_path.rb @@ -10,7 +10,7 @@ module API path = params[attr_name] path = Gitlab::Utils.check_path_traversal!(path) Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist) - rescue + rescue StandardError raise Grape::Exceptions::Validation.new( params: [@scope.full_name(attr_name)], message: "should be a valid file path" diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 2b28b30fd74..522a034a283 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -94,7 +94,7 @@ module Backup begin FileUtils.rm(file) removed += 1 - rescue => e + rescue StandardError => e progress.puts "Deleting #{file} failed: #{e.message}".color(:red) end end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 627bb44331b..b1231eebfcc 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -26,7 +26,7 @@ module Backup Thread.new do Rails.application.executor.wrap do dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) - rescue => e + rescue StandardError => e errors << e end end @@ -115,7 +115,7 @@ module Backup begin dump_container(container) - rescue => e + rescue StandardError => e errors << e break ensure @@ -260,7 +260,7 @@ module Backup progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - rescue => e + rescue StandardError => e progress.puts "[Failed] backing up #{display_repo_path}".color(:red) progress.puts "Error #{e}".color(:red) end @@ -279,7 +279,7 @@ module Backup progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) - rescue => e + rescue StandardError => e progress.puts "[Failed] restoring #{display_repo_path}".color(:red) progress.puts "Error #{e}".color(:red) end @@ -329,4 +329,4 @@ module Backup end end -Backup::Repositories.prepend_if_ee('EE::Backup::Repositories') +Backup::Repositories.prepend_mod_with('Backup::Repositories') diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index b7344808989..1bf6cf11526 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -17,7 +17,12 @@ module Banzai return context[:project] || context[:group] unless ref return context[:project] if context[:project]&.full_path == ref - Project.find_by_full_path(ref) + if reference_cache.cache_loaded? + # optimization to reuse the parent_per_reference query information + reference_cache.parent_per_reference[ref || reference_cache.current_parent_path] + else + Project.find_by_full_path(ref) + end end end end diff --git a/lib/banzai/filter/base_relative_link_filter.rb b/lib/banzai/filter/base_relative_link_filter.rb index fd526df4c48..84a6e18e77b 100644 --- a/lib/banzai/filter/base_relative_link_filter.rb +++ b/lib/banzai/filter/base_relative_link_filter.rb @@ -10,19 +10,16 @@ module Banzai protected def linkable_attributes - strong_memoize(:linkable_attributes) do - attrs = [] - - attrs += doc.search('a:not(.gfm)').map do |el| - el.attribute('href') - end - - attrs += doc.search('img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| - [el.attribute('src'), el.attribute('data-src')] - end - - attrs.reject do |attr| - attr.blank? || attr.value.start_with?('//') + if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) + # Nokorigi Nodeset#search performs badly for documents with many nodes + # + # Here we store fetched attributes in the shared variable "result" + # This variable is passed through the chain of filters and can be + # accessed by them + result[:linkable_attributes] ||= fetch_linkable_attributes + else + strong_memoize(:linkable_attributes) do + fetch_linkable_attributes end end end @@ -40,6 +37,16 @@ module Banzai def unescape_and_scrub_uri(uri) Addressable::URI.unescape(uri).scrub.delete("\0") end + + def fetch_linkable_attributes + attrs = [] + + attrs += doc.search('a:not(.gfm), img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| + [el.attribute('href'), el.attribute('src'), el.attribute('data-src')] + end + + attrs.reject { |attr| attr.blank? || attr.value.start_with?('//') } + end end end end diff --git a/lib/banzai/filter/custom_emoji_filter.rb b/lib/banzai/filter/custom_emoji_filter.rb index 1ee8f4e31e8..3171231dc9b 100644 --- a/lib/banzai/filter/custom_emoji_filter.rb +++ b/lib/banzai/filter/custom_emoji_filter.rb @@ -3,6 +3,8 @@ module Banzai module Filter class CustomEmojiFilter < HTML::Pipeline::Filter + include Gitlab::Utils::StrongMemoize + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set def call @@ -14,7 +16,7 @@ module Banzai next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) next unless content.include?(':') - next unless namespace && namespace.custom_emoji.any? + next unless has_custom_emoji? html = custom_emoji_name_element_filter(content) @@ -46,6 +48,12 @@ module Banzai private + def has_custom_emoji? + strong_memoize(:has_custom_emoji) do + namespace&.custom_emoji&.any? + end + end + def namespace context[:project].namespace.root_ancestor end diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb index bedc2d0fd04..0c53444681d 100644 --- a/lib/banzai/filter/markdown_pre_escape_filter.rb +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -26,7 +26,7 @@ module Banzai class MarkdownPreEscapeFilter < HTML::Pipeline::TextFilter # We just need to target those that are special GitLab references REFERENCE_CHARACTERS = '@#!$&~%^' - ASCII_PUNCTUATION = %r{([\\][#{REFERENCE_CHARACTERS}])}.freeze + ASCII_PUNCTUATION = %r{(\\[#{REFERENCE_CHARACTERS}])}.freeze LITERAL_KEYWORD = 'cmliteral' def call diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index 7109373dbce..08014ccdcce 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -8,6 +8,12 @@ module Banzai class AbstractReferenceFilter < ReferenceFilter include CrossProjectReference + def initialize(doc, context = nil, result = nil) + super + + @reference_cache = ReferenceCache.new(self, context) + end + # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found # reference (which we replace with placeholder during re-scaping). The # random number helps ensure it's pretty close to unique. Since it's a @@ -16,22 +22,9 @@ module Banzai REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_" REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze - def self.object_class - # Implement in child class - # Example: MergeRequest - end - - def self.object_name - @object_name ||= object_class.name.underscore - end - - def self.object_sym - @object_sym ||= object_name.to_sym - end - # Public: Find references in text (like `!123` for merge requests) # - # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| + # references_in(text) do |match, id, project_ref, matches| # object = find_object(project_ref, id) # "<a href=...>#{object.to_reference}</a>" # end @@ -42,7 +35,7 @@ module Banzai # of the external project reference, and all of the matchdata. # # Returns a String replaced with the return of the block. - def self.references_in(text, pattern = object_class.reference_pattern) + def references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| if ident = identifier($~) yield match, ident, $~[:project], $~[:namespace], $~ @@ -52,17 +45,13 @@ module Banzai end end - def self.identifier(match_data) + def identifier(match_data) symbol = symbol_from_match(match_data) parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol) end - def identifier(match_data) - self.class.identifier(match_data) - end - - def self.symbol_from_match(match) + def symbol_from_match(match) key = object_sym match[key] if match.names.include?(key.to_s) end @@ -72,7 +61,7 @@ module Banzai # # This method has the contract that if a string `ref` refers to a # record `record`, then `parse_symbol(ref) == record_identifier(record)`. - def self.parse_symbol(symbol, match_data) + def parse_symbol(symbol, match_data) symbol.to_i end @@ -84,21 +73,10 @@ module Banzai record.id end - def object_class - self.class.object_class - end - - def object_sym - self.class.object_sym - end - - def references_in(*args, &block) - self.class.references_in(*args, &block) - end - # Implement in child class # Example: project.merge_requests.find def find_object(parent_object, id) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" end # Override if the link reference pattern produces a different ID (global @@ -110,6 +88,7 @@ module Banzai # Implement in child class # Example: project_merge_request_url def url_for_object(object, parent_object) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" end def find_object_cached(parent_object, id) @@ -139,7 +118,9 @@ module Banzai def call return doc unless project || group || user - ref_pattern = object_class.reference_pattern + reference_cache.load_reference_cache(nodes) if respond_to?(:parent_records) + + ref_pattern = object_reference_pattern link_pattern = object_class.link_reference_pattern # Compile often used regexps only once outside of the loop @@ -201,9 +182,9 @@ module Banzai def object_link_filter(text, pattern, link_content: nil, link_reference: false) references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| parent_path = if parent_type == :group - full_group_path(namespace_ref) + reference_cache.full_group_path(namespace_ref) else - full_project_path(namespace_ref, project_ref) + reference_cache.full_project_path(namespace_ref, project_ref) end parent = from_ref_cached(parent_path) @@ -290,127 +271,6 @@ module Banzai text end - # Returns a Hash containing all object references (e.g. issue IDs) per the - # project they belong to. - def references_per_parent - @references_per ||= {} - - @references_per[parent_type] ||= begin - refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = [ - object_class.link_reference_pattern, - object_class.reference_pattern - ].compact.reduce { |a, b| Regexp.union(a, b) } - - nodes.each do |node| - node.to_html.scan(regex) do - path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end - - if ident = identifier($~) - refs[path] << ident - end - end - end - - refs - end - end - - # Returns a Hash containing referenced projects grouped per their full - # path. - def parent_per_reference - @per_reference ||= {} - - @per_reference[parent_type] ||= begin - refs = Set.new - - references_per_parent.each do |ref, _| - refs << ref - end - - find_for_paths(refs.to_a).index_by(&:full_path) - end - end - - def relation_for_paths(paths) - klass = parent_type.to_s.camelize.constantize - result = klass.where_full_path_in(paths) - return result if parent_type == :group - - result.includes(:namespace) if parent_type == :project - end - - # Returns projects for the given paths. - def find_for_paths(paths) - if Gitlab::SafeRequestStore.active? - cache = refs_cache - to_query = paths - cache.keys - - unless to_query.empty? - records = relation_for_paths(to_query) - - found = [] - records.each do |record| - ref = record.full_path - get_or_set_cache(cache, ref) { record } - found << ref - end - - not_found = to_query - found - not_found.each do |ref| - get_or_set_cache(cache, ref) { nil } - end - end - - cache.slice(*paths).values.compact - else - relation_for_paths(paths) - end - end - - def current_parent_path - @current_parent_path ||= parent&.full_path - end - - def current_project_namespace_path - @current_project_namespace_path ||= project&.namespace&.full_path - end - - def records_per_parent - @_records_per_project ||= {} - - @_records_per_project[object_class.to_s.underscore] ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - parent_per_reference.each do |path, parent| - record_ids = references_per_parent[path] - - parent_records(parent, record_ids).each do |record| - hash[parent][record_identifier(record)] = record - end - end - - hash - end - end - - private - - def full_project_path(namespace, project_ref) - return current_parent_path unless project_ref - - namespace_ref = namespace || current_project_namespace_path - "#{namespace_ref}/#{project_ref}" - end - - def refs_cache - Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} - end - def parent_type :project end @@ -419,19 +279,9 @@ module Banzai parent_type == :project ? project : group end - def full_group_path(group_ref) - return current_parent_path unless group_ref - - group_ref - end - - def unescape_html_entities(text) - CGI.unescapeHTML(text.to_s) - end + private - def escape_html_entities(text) - CGI.escapeHTML(text.to_s) - end + attr_accessor :reference_cache def escape_with_placeholders(text, placeholder_data) escaped = escape_html_entities(text) @@ -444,5 +294,3 @@ module Banzai end end end - -Banzai::Filter::References::AbstractReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::AbstractReferenceFilter') diff --git a/lib/banzai/filter/references/alert_reference_filter.rb b/lib/banzai/filter/references/alert_reference_filter.rb index 90fef536605..512d4028520 100644 --- a/lib/banzai/filter/references/alert_reference_filter.rb +++ b/lib/banzai/filter/references/alert_reference_filter.rb @@ -5,12 +5,9 @@ module Banzai module References class AlertReferenceFilter < IssuableReferenceFilter self.reference_type = :alert + self.object_class = AlertManagement::Alert - def self.object_class - AlertManagement::Alert - end - - def self.object_sym + def object_sym :alert end diff --git a/lib/banzai/filter/references/commit_range_reference_filter.rb b/lib/banzai/filter/references/commit_range_reference_filter.rb index ad79f8a173c..df7f42eaa70 100644 --- a/lib/banzai/filter/references/commit_range_reference_filter.rb +++ b/lib/banzai/filter/references/commit_range_reference_filter.rb @@ -8,12 +8,9 @@ module Banzai # This filter supports cross-project references. class CommitRangeReferenceFilter < AbstractReferenceFilter self.reference_type = :commit_range + self.object_class = CommitRange - def self.object_class - CommitRange - end - - def self.references_in(text, pattern = CommitRange.reference_pattern) + def references_in(text, pattern = object_reference_pattern) text.gsub(pattern) do |match| yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ end diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb index 457921bd07d..157dc696cc8 100644 --- a/lib/banzai/filter/references/commit_reference_filter.rb +++ b/lib/banzai/filter/references/commit_reference_filter.rb @@ -8,12 +8,9 @@ module Banzai # This filter supports cross-project references. class CommitReferenceFilter < AbstractReferenceFilter self.reference_type = :commit + self.object_class = Commit - def self.object_class - Commit - end - - def self.references_in(text, pattern = Commit.reference_pattern) + def references_in(text, pattern = object_reference_pattern) text.gsub(pattern) do |match| yield match, $~[:commit], $~[:project], $~[:namespace], $~ end @@ -22,7 +19,7 @@ module Banzai def find_object(project, id) return unless project.is_a?(Project) && project.valid_repo? - _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + _, record = reference_cache.records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } record end @@ -31,7 +28,7 @@ module Banzai return [] unless noteable.is_a?(MergeRequest) @referenced_merge_request_commit_shas ||= begin - referenced_shas = references_per_parent.values.reduce(:|).to_a + referenced_shas = reference_cache.references_per_parent.values.reduce(:|).to_a noteable.all_commit_shas.select do |sha| referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } end @@ -39,7 +36,7 @@ module Banzai end # The default behaviour is `#to_i` - we just pass the hash through. - def self.parse_symbol(sha_hash, _match) + def parse_symbol(sha_hash, _match) sha_hash end @@ -69,12 +66,12 @@ module Banzai extras end - private - def parent_records(parent, ids) parent.commits_by(oids: ids.to_a) end + private + def noteable context[:noteable] end diff --git a/lib/banzai/filter/references/design_reference_filter.rb b/lib/banzai/filter/references/design_reference_filter.rb index 61234e61c15..01e1036dcec 100644 --- a/lib/banzai/filter/references/design_reference_filter.rb +++ b/lib/banzai/filter/references/design_reference_filter.rb @@ -33,9 +33,10 @@ module Banzai end self.reference_type = :design + self.object_class = ::DesignManagement::Design def find_object(project, identifier) - records_per_parent[project][identifier] + reference_cache.records_per_parent[project][identifier] end def parent_records(project, identifiers) @@ -58,15 +59,6 @@ module Banzai super.includes(:route, :namespace, :group) end - def parent_type - :project - end - - # optimisation to reuse the parent_per_reference query information - def parent_from_ref(ref) - parent_per_reference[ref || current_parent_path] - end - def url_for_object(design, project) path_options = { vueroute: design.filename } Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options) @@ -76,15 +68,11 @@ module Banzai super.merge(issue: design.issue_id) end - def self.object_class - ::DesignManagement::Design - end - - def self.object_sym + def object_sym :design end - def self.parse_symbol(raw, match_data) + def parse_symbol(raw, match_data) filename = match_data[:url_filename] iid = match_data[:issue].to_i Identifier.new(filename: CGI.unescape(filename), issue_iid: iid) diff --git a/lib/banzai/filter/references/epic_reference_filter.rb b/lib/banzai/filter/references/epic_reference_filter.rb index 4ee446e5317..e25734c8b0f 100644 --- a/lib/banzai/filter/references/epic_reference_filter.rb +++ b/lib/banzai/filter/references/epic_reference_filter.rb @@ -21,4 +21,4 @@ module Banzai end end -Banzai::Filter::References::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::EpicReferenceFilter') +Banzai::Filter::References::EpicReferenceFilter.prepend_mod_with('Banzai::Filter::References::EpicReferenceFilter') diff --git a/lib/banzai/filter/references/external_issue_reference_filter.rb b/lib/banzai/filter/references/external_issue_reference_filter.rb index 247e20967df..1061a9917dd 100644 --- a/lib/banzai/filter/references/external_issue_reference_filter.rb +++ b/lib/banzai/filter/references/external_issue_reference_filter.rb @@ -10,10 +10,11 @@ module Banzai # This filter does not support cross-project references. class ExternalIssueReferenceFilter < ReferenceFilter self.reference_type = :external_issue + self.object_class = ExternalIssue # Public: Find `JIRA-123` issue references in text # - # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| + # references_in(text, pattern) do |match, issue| # "<a href=...>##{issue}</a>" # end # @@ -22,7 +23,7 @@ module Banzai # Yields the String match and the String issue reference. # # Returns a String replaced with the return of the block. - def self.references_in(text, pattern) + def references_in(text, pattern = object_reference_pattern) text.gsub(pattern) do |match| yield match, $~[:issue] end @@ -32,27 +33,7 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? - ref_pattern = issue_reference_pattern - ref_start_pattern = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - issue_link_filter(content) - end - - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_start_pattern - replace_link_node_with_href(node, index, link) do - issue_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc + super end private @@ -65,8 +46,8 @@ module Banzai # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text, link_content: nil) - self.class.references_in(text, issue_reference_pattern) do |match, id| + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text) do |match, id| url = url_for_issue(id) klass = reference_class(:issue) data = data_attribute(project: project.id, external_issue: id) @@ -97,14 +78,10 @@ module Banzai external_issues_cached(:default_issues_tracker?) end - def issue_reference_pattern + def object_reference_pattern external_issues_cached(:external_issue_reference_pattern) end - def project - context[:project] - end - def issue_title "Issue in #{project.external_issue_tracker.title}" end diff --git a/lib/banzai/filter/references/feature_flag_reference_filter.rb b/lib/banzai/filter/references/feature_flag_reference_filter.rb index be9ded1ff43..0fb2b1b3b24 100644 --- a/lib/banzai/filter/references/feature_flag_reference_filter.rb +++ b/lib/banzai/filter/references/feature_flag_reference_filter.rb @@ -5,12 +5,9 @@ module Banzai module References class FeatureFlagReferenceFilter < IssuableReferenceFilter self.reference_type = :feature_flag + self.object_class = Operations::FeatureFlag - def self.object_class - Operations::FeatureFlag - end - - def self.object_sym + def object_sym :feature_flag end diff --git a/lib/banzai/filter/references/issuable_reference_filter.rb b/lib/banzai/filter/references/issuable_reference_filter.rb index b8ccb926ae9..6349f8542ca 100644 --- a/lib/banzai/filter/references/issuable_reference_filter.rb +++ b/lib/banzai/filter/references/issuable_reference_filter.rb @@ -9,11 +9,7 @@ module Banzai end def find_object(parent, iid) - records_per_parent[parent][iid] - end - - def parent_from_ref(ref) - parent_per_reference[ref || current_parent_path] + reference_cache.records_per_parent[parent][iid] end end end diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb index eacf261b15f..1053501de7b 100644 --- a/lib/banzai/filter/references/issue_reference_filter.rb +++ b/lib/banzai/filter/references/issue_reference_filter.rb @@ -13,10 +13,7 @@ module Banzai # to reference issues from other GitLab projects. class IssueReferenceFilter < IssuableReferenceFilter self.reference_type = :issue - - def self.object_class - Issue - end + self.object_class = Issue def url_for_object(issue, project) return issue_path(issue, project) if only_path? diff --git a/lib/banzai/filter/references/iteration_reference_filter.rb b/lib/banzai/filter/references/iteration_reference_filter.rb index cf3d446147f..591e07013c3 100644 --- a/lib/banzai/filter/references/iteration_reference_filter.rb +++ b/lib/banzai/filter/references/iteration_reference_filter.rb @@ -6,13 +6,10 @@ module Banzai # The actual filter is implemented in the EE mixin class IterationReferenceFilter < AbstractReferenceFilter self.reference_type = :iteration - - def self.object_class - Iteration - end + self.object_class = Iteration end end end end -Banzai::Filter::References::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::IterationReferenceFilter') +Banzai::Filter::References::IterationReferenceFilter.prepend_mod_with('Banzai::Filter::References::IterationReferenceFilter') diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb index a6a5eec5d9a..bf6b3e47d3b 100644 --- a/lib/banzai/filter/references/label_reference_filter.rb +++ b/lib/banzai/filter/references/label_reference_filter.rb @@ -6,10 +6,7 @@ module Banzai # HTML filter that replaces label references with links. class LabelReferenceFilter < AbstractReferenceFilter self.reference_type = :label - - def self.object_class - Label - end + self.object_class = Label def find_object(parent_object, id) find_labels(parent_object).find(id) @@ -20,7 +17,7 @@ module Banzai unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| namespace = $~[:namespace] project = $~[:project] - project_path = full_project_path(namespace, project) + project_path = reference_cache.full_project_path(namespace, project) label = find_label_cached(project_path, $~[:label_id], $~[:label_name]) if label @@ -96,7 +93,7 @@ module Banzai parent = project || group if project || full_path_ref?(matches) - project_path = full_project_path(matches[:namespace], matches[:project]) + project_path = reference_cache.full_project_path(matches[:namespace], matches[:project]) parent_from_ref = from_ref_cached(project_path) reference = parent_from_ref.to_human_reference(parent) @@ -129,4 +126,4 @@ module Banzai end end -Banzai::Filter::References::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::LabelReferenceFilter') +Banzai::Filter::References::LabelReferenceFilter.prepend_mod_with('Banzai::Filter::References::LabelReferenceFilter') diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb index 872c33f6873..6c5ad83d9ae 100644 --- a/lib/banzai/filter/references/merge_request_reference_filter.rb +++ b/lib/banzai/filter/references/merge_request_reference_filter.rb @@ -9,10 +9,7 @@ module Banzai # This filter supports cross-project references. class MergeRequestReferenceFilter < IssuableReferenceFilter self.reference_type = :merge_request - - def self.object_class - MergeRequest - end + self.object_class = MergeRequest def url_for_object(mr, project) h = Gitlab::Routing.url_helpers diff --git a/lib/banzai/filter/references/milestone_reference_filter.rb b/lib/banzai/filter/references/milestone_reference_filter.rb index 49110194ddc..31a961f3e73 100644 --- a/lib/banzai/filter/references/milestone_reference_filter.rb +++ b/lib/banzai/filter/references/milestone_reference_filter.rb @@ -8,10 +8,7 @@ module Banzai include Gitlab::Utils::StrongMemoize self.reference_type = :milestone - - def self.object_class - Milestone - end + self.object_class = Milestone # Links to project milestones contain the IID, but when we're handling # 'regular' references, we need to use the global ID to disambiguate @@ -70,7 +67,7 @@ module Banzai end def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) - project_path = full_project_path(namespace_ref, project_ref) + project_path = reference_cache.full_project_path(namespace_ref, project_ref) # Returns group if project is not found by path parent = parent_from_ref(project_path) diff --git a/lib/banzai/filter/references/project_reference_filter.rb b/lib/banzai/filter/references/project_reference_filter.rb index 522c6e0f5f3..678d2aa3468 100644 --- a/lib/banzai/filter/references/project_reference_filter.rb +++ b/lib/banzai/filter/references/project_reference_filter.rb @@ -6,10 +6,11 @@ module Banzai # HTML filter that replaces project references with links. class ProjectReferenceFilter < ReferenceFilter self.reference_type = :project + self.object_class = Project # Public: Find `namespace/project>` project references in text # - # ProjectReferenceFilter.references_in(text) do |match, project| + # references_in(text) do |match, project| # "<a href=...>#{project}></a>" # end # @@ -18,33 +19,16 @@ module Banzai # Yields the String match, and the String project name. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Project.markdown_reference_pattern) do |match| + def references_in(text, pattern = object_reference_pattern) + text.gsub(pattern) do |match| yield match, "#{$~[:namespace]}/#{$~[:project]}" end end - def call - ref_pattern = Project.markdown_reference_pattern - ref_pattern_start = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - project_link_filter(content) - end - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_pattern_start - replace_link_node_with_href(node, index, link) do - project_link_filter(link, link_content: inner_html) - end - end - end - end - end + private - doc + def object_reference_pattern + @object_reference_pattern ||= Project.markdown_reference_pattern end # Replace `namespace/project>` project references in text with links to the referenced @@ -55,8 +39,8 @@ module Banzai # # Returns a String with `namespace/project>` references replaced with links. All links # have `gfm` and `gfm-project` class names attached for styling. - def project_link_filter(text, link_content: nil) - self.class.references_in(text) do |match, project_path| + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text) do |match, project_path| cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do if project = projects_hash[project_path.downcase] link_to_project(project, link_content: link_content) || match @@ -92,8 +76,6 @@ module Banzai refs.to_a end - private - def urls Gitlab::Routing.url_helpers end diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb new file mode 100644 index 00000000000..ab0c74e00d9 --- /dev/null +++ b/lib/banzai/filter/references/reference_cache.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + class ReferenceCache + include Gitlab::Utils::StrongMemoize + include RequestStoreReferenceCache + + def initialize(filter, context) + @filter = filter + @context = context + end + + def load_reference_cache(nodes) + load_references_per_parent(nodes) + load_parent_per_reference + load_records_per_parent + + @cache_loaded = true + end + + # Loads all object references (e.g. issue IDs) per + # project/group they belong to. + def load_references_per_parent(nodes) + @references_per_parent ||= {} + + @references_per_parent[parent_type] ||= begin + refs = Hash.new { |hash, key| hash[key] = Set.new } + + nodes.each do |node| + node.to_html.scan(regex) do + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + + ident = filter.identifier($~) + refs[path] << ident if ident + end + end + + refs + end + end + + def references_per_parent + @references_per_parent[parent_type] + end + + # Returns a Hash containing referenced projects grouped per their full + # path. + def load_parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin + refs = references_per_parent.keys.to_set + + find_for_paths(refs.to_a).index_by(&:full_path) + end + end + + def parent_per_reference + @per_reference[parent_type] + end + + def load_records_per_parent + @_records_per_project ||= {} + + @_records_per_project[filter.object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + filter.parent_records(parent, record_ids).each do |record| + hash[parent][filter.record_identifier(record)] = record + end + end + + hash + end + end + + def records_per_parent + @_records_per_project[filter.object_class.to_s.underscore] + end + + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(namespace: :route) if parent_type == :project + end + + # Returns projects for the given paths. + def find_for_paths(paths) + if Gitlab::SafeRequestStore.active? + cache = refs_cache + to_query = paths - cache.keys + + unless to_query.empty? + records = relation_for_paths(to_query) + + found = [] + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } + found << ref + end + + not_found = to_query - found + not_found.each do |ref| + get_or_set_cache(cache, ref) { nil } + end + end + + cache.slice(*paths).values.compact + else + relation_for_paths(paths) + end + end + + def current_parent_path + strong_memoize(:current_parent_path) do + parent&.full_path + end + end + + def current_project_namespace_path + strong_memoize(:current_project_namespace_path) do + project&.namespace&.full_path + end + end + + def full_project_path(namespace, project_ref) + return current_parent_path unless project_ref + + namespace_ref = namespace || current_project_namespace_path + "#{namespace_ref}/#{project_ref}" + end + + def full_group_path(group_ref) + return current_parent_path unless group_ref + + group_ref + end + + def cache_loaded? + !!@cache_loaded + end + + private + + attr_accessor :filter, :context + + delegate :project, :group, :parent, :parent_type, to: :filter + + def regex + strong_memoize(:regex) do + [ + filter.object_class.link_reference_pattern, + filter.object_class.reference_pattern + ].compact.reduce { |a, b| Regexp.union(a, b) } + end + end + + def refs_cache + Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + end + end + end +end + +Banzai::Filter::References::ReferenceCache.prepend_mod diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index dd15c43f5d8..58436f4505e 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -16,8 +16,14 @@ module Banzai include OutputSafety class << self + # Implement in child class + # Example: self.reference_type = :merge_request attr_accessor :reference_type + # Implement in child class + # Example: self.object_class = MergeRequest + attr_accessor :object_class + def call(doc, context = nil, result = nil) new(doc, context, result).call_and_update_nodes end @@ -34,6 +40,77 @@ module Banzai with_update_nodes { call } end + def call + ref_pattern_start = /\A#{object_reference_pattern}\z/ + + nodes.each_with_index do |node, index| + if text_node?(node) + replace_text_when_pattern_matches(node, index, object_reference_pattern) do |content| + object_link_filter(content, object_reference_pattern) + end + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if link =~ ref_pattern_start + replace_link_node_with_href(node, index, link) do + object_link_filter(link, object_reference_pattern, link_content: inner_html) + end + end + end + end + end + + doc + end + + # Public: Find references in text (like `!123` for merge requests) + # + # references_in(text) do |match, id, project_ref, matches| + # object = find_object(project_ref, id) + # "<a href=...>#{object.to_reference}</a>" + # end + # + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, an optional String + # of the external project reference, and all of the matchdata. + # + # Returns a String replaced with the return of the block. + def references_in(text, pattern = object_reference_pattern) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" + end + + # Iterates over all <a> and text() nodes in a document. + # + # Nodes are skipped whenever their ancestor is one of the nodes returned + # by `ignore_ancestor_query`. Link tags are not processed if they have a + # "gfm" class or the "href" attribute is empty. + def each_node + return to_enum(__method__) unless block_given? + + doc.xpath(query).each do |node| + yield node + end + end + + # Returns an Array containing all HTML nodes. + def nodes + @nodes ||= each_node.to_a + end + + def object_class + self.class.object_class + end + + def project + context[:project] + end + + def group + context[:group] + end + + private + # Returns a data attribute String to attach to a reference link # # attributes - Hash, where the key becomes the data attribute name and the @@ -69,12 +146,11 @@ module Banzai end end - def project - context[:project] - end - - def group - context[:group] + # Ensure that a :project key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project unless skip_project_check? end def user @@ -93,31 +169,6 @@ module Banzai "#{gfm_klass} has-tooltip" end - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project unless skip_project_check? - end - - # Iterates over all <a> and text() nodes in a document. - # - # Nodes are skipped whenever their ancestor is one of the nodes returned - # by `ignore_ancestor_query`. Link tags are not processed if they have a - # "gfm" class or the "href" attribute is empty. - def each_node - return to_enum(__method__) unless block_given? - - doc.xpath(query).each do |node| - yield node - end - end - - # Returns an Array containing all HTML nodes. - def nodes - @nodes ||= each_node.to_a - end - # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. def yield_valid_link(node) link = unescape_link(node.attr('href').to_s) @@ -132,6 +183,14 @@ module Banzai CGI.unescape(href) end + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end + + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end + def replace_text_when_pattern_matches(node, index, pattern) return unless node.text =~ pattern @@ -161,7 +220,21 @@ module Banzai node.is_a?(Nokogiri::XML::Element) end - private + def object_reference_pattern + @object_reference_pattern ||= object_class.reference_pattern + end + + def object_name + @object_name ||= object_class.name.underscore + end + + def object_sym + @object_sym ||= object_name.to_sym + end + + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + raise NotImplementedError, "#{self.class} must implement method: #{__callee__}" + end def query @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] diff --git a/lib/banzai/filter/references/snippet_reference_filter.rb b/lib/banzai/filter/references/snippet_reference_filter.rb index bf7e0f78609..502bfca1ab7 100644 --- a/lib/banzai/filter/references/snippet_reference_filter.rb +++ b/lib/banzai/filter/references/snippet_reference_filter.rb @@ -9,15 +9,16 @@ module Banzai # This filter supports cross-project references. class SnippetReferenceFilter < AbstractReferenceFilter self.reference_type = :snippet + self.object_class = Snippet - def self.object_class - Snippet + def parent_records(project, ids) + return unless project.is_a?(Project) + + project.snippets.where(id: ids.to_a) end def find_object(project, id) - return unless project.is_a?(Project) - - project.snippets.find_by(id: id) + reference_cache.records_per_parent[project][id] end def url_for_object(snippet, project) diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb index 04665973f51..1709b607c2e 100644 --- a/lib/banzai/filter/references/user_reference_filter.rb +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -8,10 +8,11 @@ module Banzai # A special `@all` reference is also supported. class UserReferenceFilter < ReferenceFilter self.reference_type = :user + self.object_class = User # Public: Find `@user` user references in text # - # UserReferenceFilter.references_in(text) do |match, username| + # references_in(text) do |match, username| # "<a href=...>@#{user}</a>" # end # @@ -20,8 +21,8 @@ module Banzai # Yields the String match, and the String user name. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(User.reference_pattern) do |match| + def references_in(text, pattern = object_reference_pattern) + text.gsub(pattern) do |match| yield match, $~[:user] end end @@ -29,28 +30,11 @@ module Banzai def call return doc if project.nil? && group.nil? && !skip_project_check? - ref_pattern = User.reference_pattern - ref_pattern_start = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - user_link_filter(content) - end - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_pattern_start - replace_link_node_with_href(node, index, link) do - user_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc + super end + private + # Replace `@user` user references in text with links to the referenced # user's profile page. # @@ -59,8 +43,8 @@ module Banzai # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text, link_content: nil) - self.class.references_in(text) do |match, username| + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text, pattern) do |match, username| if username == 'all' && !skip_project_check? link_to_all(link_content: link_content) else @@ -100,8 +84,6 @@ module Banzai refs.to_a end - private - def urls Gitlab::Routing.url_helpers end diff --git a/lib/banzai/filter/references/vulnerability_reference_filter.rb b/lib/banzai/filter/references/vulnerability_reference_filter.rb index e5f2408eda4..aaf45304021 100644 --- a/lib/banzai/filter/references/vulnerability_reference_filter.rb +++ b/lib/banzai/filter/references/vulnerability_reference_filter.rb @@ -6,19 +6,10 @@ module Banzai # The actual filter is implemented in the EE mixin class VulnerabilityReferenceFilter < IssuableReferenceFilter self.reference_type = :vulnerability - - def self.object_class - Vulnerability - end - - private - - def project - context[:project] - end + self.object_class = Vulnerability end end end end -Banzai::Filter::References::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::VulnerabilityReferenceFilter') +Banzai::Filter::References::VulnerabilityReferenceFilter.prepend_mod_with('Banzai::Filter::References::VulnerabilityReferenceFilter') diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 06dddc74eba..1e84e7e8af3 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -65,4 +65,4 @@ module Banzai end end -Banzai::Filter::SanitizationFilter.prepend_if_ee('EE::Banzai::Filter::SanitizationFilter') +Banzai::Filter::SanitizationFilter.prepend_mod_with('Banzai::Filter::SanitizationFilter') diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 731a2bb4c77..b16ea689d2e 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -38,7 +38,7 @@ module Banzai begin code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) css_classes << " language-#{language}" if language - rescue + rescue StandardError # Gracefully handle syntax highlighter bugs/errors to ensure users can # still access an issue/comment/etc. First, retry with the plain text # filter. If that fails, then just skip this entirely, but that would diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 762371e1418..ceb7547a85d 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -15,8 +15,16 @@ module Banzai def call return doc if context[:system_note] - linkable_attributes.each do |attr| - process_link_to_upload_attr(attr) + if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) + # We exclude processed upload links from the linkable attributes to + # prevent further modifications by RepositoryLinkFilter + linkable_attributes.reject! do |attr| + process_link_to_upload_attr(attr) + end + else + linkable_attributes.each do |attr| + process_link_to_upload_attr(attr) + end end doc diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 8994cdbed60..34b6ca99e32 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -58,4 +58,4 @@ module Banzai end end -Banzai::IssuableExtractor.prepend_if_ee('EE::Banzai::IssuableExtractor') +Banzai::IssuableExtractor.prepend_mod_with('Banzai::IssuableExtractor') diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 028e3c44dc3..df8151b3296 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -76,4 +76,4 @@ module Banzai end end -Banzai::Pipeline::GfmPipeline.prepend_if_ee('EE::Banzai::Pipeline::GfmPipeline') +Banzai::Pipeline::GfmPipeline.prepend_mod_with('Banzai::Pipeline::GfmPipeline') diff --git a/lib/banzai/pipeline/post_process_pipeline.rb b/lib/banzai/pipeline/post_process_pipeline.rb index 32d7126c97d..889574cf6bf 100644 --- a/lib/banzai/pipeline/post_process_pipeline.rb +++ b/lib/banzai/pipeline/post_process_pipeline.rb @@ -33,4 +33,4 @@ module Banzai end end -Banzai::Pipeline::PostProcessPipeline.prepend_if_ee('EE::Banzai::Pipeline::PostProcessPipeline') +Banzai::Pipeline::PostProcessPipeline.prepend_mod_with('Banzai::Pipeline::PostProcessPipeline') diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 65a5e28b704..0031ccc7011 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -40,4 +40,4 @@ module Banzai end end -Banzai::Pipeline::SingleLinePipeline.prepend_if_ee('EE::Banzai::Pipeline::SingleLinePipeline') +Banzai::Pipeline::SingleLinePipeline.prepend_mod_with('Banzai::Pipeline::SingleLinePipeline') diff --git a/lib/banzai/reference_parser/epic_parser.rb b/lib/banzai/reference_parser/epic_parser.rb index 7e72a260839..862d09934e9 100644 --- a/lib/banzai/reference_parser/epic_parser.rb +++ b/lib/banzai/reference_parser/epic_parser.rb @@ -13,4 +13,4 @@ module Banzai end end -Banzai::ReferenceParser::EpicParser.prepend_if_ee('::EE::Banzai::ReferenceParser::EpicParser') +Banzai::ReferenceParser::EpicParser.prepend_mod_with('Banzai::ReferenceParser::EpicParser') diff --git a/lib/banzai/reference_parser/iteration_parser.rb b/lib/banzai/reference_parser/iteration_parser.rb index 45253fa1977..981354aa8e1 100644 --- a/lib/banzai/reference_parser/iteration_parser.rb +++ b/lib/banzai/reference_parser/iteration_parser.rb @@ -19,4 +19,4 @@ module Banzai end end -Banzai::ReferenceParser::IterationParser.prepend_if_ee('::EE::Banzai::ReferenceParser::IterationParser') +Banzai::ReferenceParser::IterationParser.prepend_mod_with('Banzai::ReferenceParser::IterationParser') diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index d7bf450465e..24bc1a24e09 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -3,6 +3,8 @@ module Banzai module ReferenceParser class MergeRequestParser < IssuableParser + include Gitlab::Utils::StrongMemoize + self.reference_type = :merge_request def records_for_nodes(nodes) @@ -27,6 +29,16 @@ module Banzai self.class.data_attribute ) end + + def can_read_reference?(user, merge_request) + memo = strong_memoize(:can_read_reference) { {} } + + project_id = merge_request.project_id + + return memo[project_id] if memo.key?(project_id) + + memo[project_id] = can?(user, :read_merge_request_iid, merge_request.project) + end end end end diff --git a/lib/banzai/reference_parser/project_parser.rb b/lib/banzai/reference_parser/project_parser.rb index b4e3a55b4f1..6c600508996 100644 --- a/lib/banzai/reference_parser/project_parser.rb +++ b/lib/banzai/reference_parser/project_parser.rb @@ -19,7 +19,7 @@ module Banzai def readable_project_ids_for(user) @project_ids_by_user ||= {} @project_ids_by_user[user] ||= - Project.public_or_visible_to_user(user).where("projects.id IN (?)", @projects_for_nodes.values.map(&:id)).pluck(:id) + Project.public_or_visible_to_user(user).where(projects: { id: @projects_for_nodes.values.map(&:id) }).pluck(:id) end def can_read_reference?(user, ref_project, node) diff --git a/lib/banzai/reference_parser/vulnerability_parser.rb b/lib/banzai/reference_parser/vulnerability_parser.rb index 143f2605927..86b16605d39 100644 --- a/lib/banzai/reference_parser/vulnerability_parser.rb +++ b/lib/banzai/reference_parser/vulnerability_parser.rb @@ -13,4 +13,4 @@ module Banzai end end -Banzai::ReferenceParser::VulnerabilityParser.prepend_if_ee('::EE::Banzai::ReferenceParser::VulnerabilityParser') +Banzai::ReferenceParser::VulnerabilityParser.prepend_mod_with('Banzai::ReferenceParser::VulnerabilityParser') diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index ef99122cdfd..c89679f63b5 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -28,6 +28,17 @@ module BulkImports end end + def post(resource, body = {}) + with_error_handling do + Gitlab::HTTP.post( + resource_url(resource), + headers: request_headers, + follow_redirects: false, + body: body + ) + end + end + def each_page(method, resource, query = {}, &block) return to_enum(__method__, method, resource, query) unless block_given? @@ -63,7 +74,7 @@ module BulkImports def with_error_handling response = yield - raise ConnectionError.new("Error #{response.code}") unless response.success? + raise ConnectionError, "Error #{response.code}" unless response.success? response rescue *Gitlab::HTTP::HTTP_ERRORS => e diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index b756fba3bee..8f515b571a6 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -56,7 +56,7 @@ module BulkImports pipeline_step: step, step_class: class_name ) - rescue => e + rescue StandardError => e log_import_failure(e, step) mark_as_failed if abort_on_failure? diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb new file mode 100644 index 00000000000..35b77240ea7 --- /dev/null +++ b/lib/bulk_imports/stage.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module BulkImports + class Stage + include Singleton + + CONFIG = { + group: { + pipeline: BulkImports::Groups::Pipelines::GroupPipeline, + stage: 0 + }, + subgroups: { + pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + stage: 1 + }, + members: { + pipeline: BulkImports::Groups::Pipelines::MembersPipeline, + stage: 1 + }, + labels: { + pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, + stage: 1 + }, + milestones: { + pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, + stage: 1 + }, + badges: { + pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + stage: 1 + }, + finisher: { + pipeline: BulkImports::Groups::Pipelines::EntityFinisher, + stage: 2 + } + }.freeze + + def self.pipelines + instance.pipelines + end + + def self.pipeline_exists?(name) + pipelines.any? do |(_, pipeline)| + pipeline.to_s == name.to_s + end + end + + def pipelines + @pipelines ||= config + .values + .sort_by { |entry| entry[:stage] } + .map do |entry| + [entry[:stage], entry[:pipeline]] + end + end + + private + + def config + @config ||= CONFIG + end + end +end + +::BulkImports::Stage.prepend_mod_with('BulkImports::Stage') diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 6f5acabe81f..8dea765dd11 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -250,4 +250,4 @@ module ContainerRegistry end end -ContainerRegistry::Client.prepend_if_ee('EE::ContainerRegistry::Client') +ContainerRegistry::Client.prepend_mod_with('ContainerRegistry::Client') diff --git a/lib/declarative_enum.rb b/lib/declarative_enum.rb index f3c8c181c73..8dea9d6130b 100644 --- a/lib/declarative_enum.rb +++ b/lib/declarative_enum.rb @@ -40,7 +40,7 @@ module DeclarativeEnum # This `prepended` hook will merge the enum definition # of the prepended module into the base module to be - # used by `prepend_if_ee` helper method. + # used by `prepend_mod_with` helper method. def prepended(base) base.definition.merge!(definition) end @@ -64,7 +64,7 @@ module DeclarativeEnum end def define(&block) - raise LocalJumpError.new('No block given') unless block + raise LocalJumpError, 'No block given' unless block @definition = Builder.new(definition, block).build end diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb deleted file mode 100644 index bd1c121fe79..00000000000 --- a/lib/declarative_policy.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: true - -require_dependency 'declarative_policy/cache' -require_dependency 'declarative_policy/condition' -require_dependency 'declarative_policy/delegate_dsl' -require_dependency 'declarative_policy/policy_dsl' -require_dependency 'declarative_policy/rule_dsl' -require_dependency 'declarative_policy/preferred_scope' -require_dependency 'declarative_policy/rule' -require_dependency 'declarative_policy/runner' -require_dependency 'declarative_policy/step' - -require_dependency 'declarative_policy/base' - -module DeclarativePolicy - extend PreferredScope - - CLASS_CACHE_MUTEX = Mutex.new - CLASS_CACHE_IVAR = :@__DeclarativePolicy_CLASS_CACHE - - class << self - def policy_for(user, subject, opts = {}) - cache = opts[:cache] || {} - key = Cache.policy_key(user, subject) - - cache[key] ||= - # to avoid deadlocks in multi-threaded environment when - # autoloading is enabled, we allow concurrent loads, - # https://gitlab.com/gitlab-org/gitlab-foss/issues/48263 - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - class_for(subject).new(user, subject, opts) - end - end - - def class_for(subject) - return GlobalPolicy if subject == :global - return NilPolicy if subject.nil? - - subject = find_delegate(subject) - - policy_class = class_for_class(subject.class) - raise "no policy for #{subject.class.name}" if policy_class.nil? - - policy_class - end - - def has_policy?(subject) - !class_for_class(subject.class).nil? - end - - private - - # This method is heavily cached because there are a lot of anonymous - # modules in play in a typical rails app, and #name performs quite - # slowly for anonymous classes and modules. - # - # See https://bugs.ruby-lang.org/issues/11119 - # - # if the above bug is resolved, this caching could likely be removed. - def class_for_class(subject_class) - unless subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) - CLASS_CACHE_MUTEX.synchronize do - # re-check in case of a race - break if subject_class.instance_variable_defined?(CLASS_CACHE_IVAR) - - policy_class = compute_class_for_class(subject_class) - subject_class.instance_variable_set(CLASS_CACHE_IVAR, policy_class) - end - end - - subject_class.instance_variable_get(CLASS_CACHE_IVAR) - end - - def compute_class_for_class(subject_class) - if subject_class.respond_to?(:declarative_policy_class) - return subject_class.declarative_policy_class.constantize - end - - subject_class.ancestors.each do |klass| - name = klass.name - - next unless name - - begin - policy_class = "#{name}Policy".constantize - - # NOTE: the < operator here tests whether policy_class - # inherits from Base. We can't use #is_a? because that - # tests for *instances*, not *subclasses*. - return policy_class if policy_class < Base - rescue NameError - nil - end - end - - nil - end - - def find_delegate(subject) - seen = Set.new - - while subject.respond_to?(:declarative_policy_delegate) - raise ArgumentError, "circular delegations" if seen.include?(subject.object_id) - - seen << subject.object_id - subject = subject.declarative_policy_delegate - end - - subject - end - end -end diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb deleted file mode 100644 index 49cbdd2aeb4..00000000000 --- a/lib/declarative_policy/base.rb +++ /dev/null @@ -1,354 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - class Base - # A map of ability => list of rules together with :enable - # or :prevent actions. Used to look up which rules apply to - # a given ability. See Base.ability_map - class AbilityMap - attr_reader :map - def initialize(map = {}) - @map = map - end - - # This merge behavior is different than regular hashes - if both - # share a key, the values at that key are concatenated, rather than - # overridden. - def merge(other) - conflict_proc = proc { |key, my_val, other_val| my_val + other_val } - AbilityMap.new(@map.merge(other.map, &conflict_proc)) - end - - def actions(key) - @map[key] ||= [] - end - - def enable(key, rule) - actions(key) << [:enable, rule] - end - - def prevent(key, rule) - actions(key) << [:prevent, rule] - end - end - - class << self - # The `own_ability_map` vs `ability_map` distinction is used so that - # the data structure is properly inherited - with subclasses recursively - # merging their parent class. - # - # This pattern is also used for conditions, global_actions, and delegations. - def ability_map - if self == Base - own_ability_map - else - superclass.ability_map.merge(own_ability_map) - end - end - - def own_ability_map - @own_ability_map ||= AbilityMap.new - end - - # an inheritable map of conditions, by name - def conditions - if self == Base - own_conditions - else - superclass.conditions.merge(own_conditions) - end - end - - def own_conditions - @own_conditions ||= {} - end - - # a list of global actions, generated by `prevent_all`. these aren't - # stored in `ability_map` because they aren't indexed by a particular - # ability. - def global_actions - if self == Base - own_global_actions - else - superclass.global_actions + own_global_actions - end - end - - def own_global_actions - @own_global_actions ||= [] - end - - # an inheritable map of delegations, indexed by name (which may be - # autogenerated) - def delegations - if self == Base - own_delegations - else - superclass.delegations.merge(own_delegations) - end - end - - def own_delegations - @own_delegations ||= {} - end - - # all the [rule, action] pairs that apply to a particular ability. - # we combine the specific ones looked up in ability_map with the global - # ones. - def configuration_for(ability) - ability_map.actions(ability) + global_actions - end - - ### declaration methods ### - - def delegate(name = nil, &delegation_block) - if name.nil? - @delegate_name_counter ||= 0 - @delegate_name_counter += 1 - name = :"anonymous_#{@delegate_name_counter}" - end - - name = name.to_sym - - if delegation_block.nil? - delegation_block = proc { @subject.__send__(name) } # rubocop:disable GitlabSecurity/PublicSend - end - - own_delegations[name] = delegation_block - end - - # Declare that the given abilities should not be read from delegates. - # - # This is useful if you have an ability that you want to define - # differently in a policy than in a delegated policy, but still want to - # delegate all other abilities. - # - # example: - # - # delegate { @subect.parent } - # - # overrides :drive_car, :watch_tv - # - def overrides(*names) - @overrides ||= [].to_set - @overrides.merge(names) - end - - # Declares a rule, constructed using RuleDsl, and returns - # a PolicyDsl which is used for registering the rule with - # this class. PolicyDsl will call back into Base.enable_when, - # Base.prevent_when, and Base.prevent_all_when. - def rule(&block) - rule = RuleDsl.new(self).instance_eval(&block) - PolicyDsl.new(self, rule) - end - - # A hash in which to store calls to `desc` and `with_scope`, etc. - def last_options - @last_options ||= {}.with_indifferent_access - end - - # retrieve and zero out the previously set options (used in .condition) - def last_options! - last_options.tap { @last_options = nil } - end - - # Declare a description for the following condition. Currently unused, - # but opens the potential for explaining to users why they were or were - # not able to do something. - def desc(description) - last_options[:description] = description - end - - def with_options(opts = {}) - last_options.merge!(opts) - end - - def with_scope(scope) - with_options scope: scope - end - - def with_score(score) - with_options score: score - end - - # Declares a condition. It gets stored in `own_conditions`, and generates - # a query method based on the condition's name. - def condition(name, opts = {}, &value) - name = name.to_sym - - opts = last_options!.merge(opts) - opts[:context_key] ||= self.name - - condition = Condition.new(name, opts, &value) - - self.own_conditions[name] = condition - - define_method(:"#{name}?") { condition(name).pass? } - end - - # These next three methods are mainly called from PolicyDsl, - # and are responsible for "inverting" the relationship between - # an ability and a rule. We store in `ability_map` a map of - # abilities to rules that affect them, together with a - # symbol indicating :prevent or :enable. - def enable_when(abilities, rule) - abilities.each { |a| own_ability_map.enable(a, rule) } - end - - def prevent_when(abilities, rule) - abilities.each { |a| own_ability_map.prevent(a, rule) } - end - - # we store global prevents (from `prevent_all`) separately, - # so that they can be combined into every decision made. - def prevent_all_when(rule) - own_global_actions << [:prevent, rule] - end - end - - # A policy object contains a specific user and subject on which - # to compute abilities. For this reason it's sometimes called - # "context" within the framework. - # - # It also stores a reference to the cache, so it can be used - # to cache computations by e.g. ManifestCondition. - attr_reader :user, :subject - def initialize(user, subject, opts = {}) - @user = user - @subject = subject - @cache = opts[:cache] || {} - end - - # helper for checking abilities on this and other subjects - # for the current user. - def can?(ability, new_subject = :_self) - return allowed?(ability) if new_subject == :_self - - policy_for(new_subject).allowed?(ability) - end - - # This is the main entry point for permission checks. It constructs - # or looks up a Runner for the given ability and asks it if it passes. - def allowed?(*abilities) - abilities.all? { |a| runner(a).pass? } - end - - # The inverse of #allowed?, used mainly in specs. - def disallowed?(*abilities) - abilities.all? { |a| !runner(a).pass? } - end - - # computes the given ability and prints a helpful debugging output - # showing which - def debug(ability, *args) - runner(ability).debug(*args) - end - - desc "Unknown user" - condition(:anonymous, scope: :user, score: 0) { @user.nil? } - - desc "By default" - condition(:default, scope: :global, score: 0) { true } - - def repr - subject_repr = - if @subject.respond_to?(:id) - "#{@subject.class.name}/#{@subject.id}" - else - @subject.inspect - end - - user_repr = - if @user - @user.to_reference - else - "<anonymous>" - end - - "(#{user_repr} : #{subject_repr})" - end - - def inspect - "#<#{self.class.name} #{repr}>" - end - - # returns a Runner for the given ability, capable of computing whether - # the ability is allowed. Runners are cached on the policy (which itself - # is cached on @cache), and caches its result. This is how we perform caching - # at the ability level. - def runner(ability) - ability = ability.to_sym - @runners ||= {} - @runners[ability] ||= - begin - own_runner = Runner.new(own_steps(ability)) - if self.class.overrides.include?(ability) - own_runner - else - delegated_runners = delegated_policies.values.compact.map { |p| p.runner(ability) } - delegated_runners.inject(own_runner, &:merge_runner) - end - end - end - - # Helpers for caching. Used by ManifestCondition in performing condition - # computation. - # - # NOTE we can't use ||= here because the value might be the - # boolean `false` - def cache(key) - return @cache[key] if cached?(key) - - @cache[key] = yield - end - - def cached?(key) - !@cache[key].nil? - end - - # returns a ManifestCondition capable of computing itself. The computation - # will use our own @cache. - def condition(name) - name = name.to_sym - @_conditions ||= {} - @_conditions[name] ||= - begin - raise "invalid condition #{name}" unless self.class.conditions.key?(name) - - ManifestCondition.new(self.class.conditions[name], self) - end - end - - # used in specs - returns true if there is no possible way for any action - # to be allowed, determined only by the global :prevent_all rules. - def banned? - global_steps = self.class.global_actions.map { |(action, rule)| Step.new(self, rule, action) } - !Runner.new(global_steps).pass? - end - - # A list of other policies that we've delegated to (see `Base.delegate`) - def delegated_policies - @delegated_policies ||= self.class.delegations.transform_values do |block| - new_subject = instance_eval(&block) - - # never delegate to nil, as that would immediately prevent_all - next if new_subject.nil? - - policy_for(new_subject) - end - end - - def policy_for(other_subject) - DeclarativePolicy.policy_for(@user, other_subject, cache: @cache) - end - - protected - - # constructs steps that come from this policy and not from any delegations - def own_steps(ability) - rules = self.class.configuration_for(ability) - rules.map { |(action, rule)| Step.new(self, rule, action) } - end - end -end diff --git a/lib/declarative_policy/cache.rb b/lib/declarative_policy/cache.rb deleted file mode 100644 index 13006e56454..00000000000 --- a/lib/declarative_policy/cache.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - module Cache - class << self - def user_key(user) - return '<anonymous>' if user.nil? - - id_for(user) - end - - def policy_key(user, subject) - u = user_key(user) - s = subject_key(subject) - "/dp/policy/#{u}/#{s}" - end - - def subject_key(subject) - return '<nil>' if subject.nil? - return subject.inspect if subject.is_a?(Symbol) - - "#{subject.class.name}:#{id_for(subject)}" - end - - private - - def id_for(obj) - id = - begin - obj.id - rescue NoMethodError - nil - end - - id || "##{obj.object_id}" - end - end - end -end diff --git a/lib/declarative_policy/condition.rb b/lib/declarative_policy/condition.rb deleted file mode 100644 index b77f40b1093..00000000000 --- a/lib/declarative_policy/condition.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - # A Condition is the data structure that is created by the - # `condition` declaration on DeclarativePolicy::Base. It is - # more or less just a struct of the data passed to that - # declaration. It holds on to the block to be instance_eval'd - # on a context (instance of Base) later, via #compute. - class Condition - attr_reader :name, :description, :scope - attr_reader :manual_score - attr_reader :context_key - def initialize(name, opts = {}, &compute) - @name = name - @compute = compute - @scope = opts.fetch(:scope, :normal) - @description = opts.delete(:description) - @context_key = opts[:context_key] - @manual_score = opts.fetch(:score, nil) - end - - def compute(context) - !!context.instance_eval(&@compute) - end - - def key - "#{@context_key}/#{@name}" - end - end - - # In contrast to a Condition, a ManifestCondition contains - # a Condition and a context object, and is capable of calculating - # a result itself. This is the return value of Base#condition. - class ManifestCondition - def initialize(condition, context) - @condition = condition - @context = context - end - - # The main entry point - does this condition pass? We reach into - # the context's cache here so that we can share in the global - # cache (often RequestStore or similar). - def pass? - @context.cache(cache_key) { @condition.compute(@context) } - end - - # Whether we've already computed this condition. - def cached? - @context.cached?(cache_key) - end - - # This is used to score Rule::Condition. See Rule::Condition#score - # and Runner#steps_by_score for how scores are used. - # - # The number here is intended to represent, abstractly, how - # expensive it would be to calculate this condition. - # - # See #cache_key for info about @condition.scope. - def score - # If we've been cached, no computation is necessary. - return 0 if cached? - - # Use the override from condition(score: ...) if present - return @condition.manual_score if @condition.manual_score - - # Global scope rules are cheap due to max cache sharing - return 2 if @condition.scope == :global - - # "Normal" rules can't share caches with any other policies - return 16 if @condition.scope == :normal - - # otherwise, we're :user or :subject scope, so it's 4 if - # the caller has declared a preference - return 4 if @condition.scope == DeclarativePolicy.preferred_scope - - # and 8 for all other :user or :subject scope conditions. - 8 - end - - private - - # This method controls the caching for the condition. This is where - # the condition(scope: ...) option comes into play. Notice that - # depending on the scope, we may cache only by the user or only by - # the subject, resulting in sharing across different policy objects. - def cache_key - @cache_key ||= - case @condition.scope - when :normal then "/dp/condition/#{@condition.key}/#{user_key},#{subject_key}" - when :user then "/dp/condition/#{@condition.key}/#{user_key}" - when :subject then "/dp/condition/#{@condition.key}/#{subject_key}" - when :global then "/dp/condition/#{@condition.key}" - else raise 'invalid scope' - end - end - - def user_key - Cache.user_key(@context.user) - end - - def subject_key - Cache.subject_key(@context.subject) - end - end -end diff --git a/lib/declarative_policy/delegate_dsl.rb b/lib/declarative_policy/delegate_dsl.rb deleted file mode 100644 index 67e3429b696..00000000000 --- a/lib/declarative_policy/delegate_dsl.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - # Used when the name of a delegate is mentioned in - # the rule DSL. - class DelegateDsl - def initialize(rule_dsl, delegate_name) - @rule_dsl = rule_dsl - @delegate_name = delegate_name - end - - def method_missing(msg, *args) - return super unless args.empty? && !block_given? - - @rule_dsl.delegate(@delegate_name, msg) - end - end -end diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb deleted file mode 100644 index 69a2bbcc79e..00000000000 --- a/lib/declarative_policy/policy_dsl.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - # The return value of a rule { ... } declaration. - # Can call back to register rules with the containing - # Policy class (context_class here). See Base.rule - # - # Note that the #policy method just performs an #instance_eval, - # which is useful for multiple #enable or #prevent calls. - # - # Also provides a #method_missing proxy to the context - # class's class methods, so that helper methods can be - # defined and used in a #policy { ... } block. - class PolicyDsl - def initialize(context_class, rule) - @context_class = context_class - @rule = rule - end - - def policy(&block) - instance_eval(&block) - end - - def enable(*abilities) - @context_class.enable_when(abilities, @rule) - end - - def prevent(*abilities) - @context_class.prevent_when(abilities, @rule) - end - - def prevent_all - @context_class.prevent_all_when(@rule) - end - - def method_missing(msg, *args, &block) - return super unless @context_class.respond_to?(msg) - - @context_class.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(msg) - @context_class.respond_to?(msg) || super - end - end -end diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb deleted file mode 100644 index 9e512086593..00000000000 --- a/lib/declarative_policy/preferred_scope.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - module PreferredScope - PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" - - def with_preferred_scope(scope) - old_scope = Thread.current[PREFERRED_SCOPE_KEY] - Thread.current[PREFERRED_SCOPE_KEY] = scope - yield - ensure - Thread.current[PREFERRED_SCOPE_KEY] = old_scope - end - - def preferred_scope - Thread.current[PREFERRED_SCOPE_KEY] - end - - def user_scope(&block) - with_preferred_scope(:user, &block) - end - - def subject_scope(&block) - with_preferred_scope(:subject, &block) - end - - def preferred_scope=(scope) - Thread.current[PREFERRED_SCOPE_KEY] = scope - end - end -end diff --git a/lib/declarative_policy/rule.rb b/lib/declarative_policy/rule.rb deleted file mode 100644 index 964d35cde9e..00000000000 --- a/lib/declarative_policy/rule.rb +++ /dev/null @@ -1,312 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - module Rule - # A Rule is the object that results from the `rule` declaration, - # usually built using the DSL in `RuleDsl`. It is a basic logical - # combination of building blocks, and is capable of deciding, - # given a context (instance of DeclarativePolicy::Base) whether it - # passes or not. Note that this decision doesn't by itself know - # how that affects the actual ability decision - for that, a - # `Step` is used. - class Base - def self.make(*args) - new(*args).simplify - end - - # true or false whether this rule passes. - # `context` is a policy - an instance of - # DeclarativePolicy::Base. - def pass?(context) - raise 'abstract' - end - - # same as #pass? except refuses to do any I/O, - # returning nil if the result is not yet cached. - # used for accurately scoring And/Or - def cached_pass?(context) - raise 'abstract' - end - - # abstractly, how long would it take to compute - # this rule? lower-scored rules are tried first. - def score(context) - raise 'abstract' - end - - # unwrap double negatives and nested and/or - def simplify - self - end - - # convenience combination methods - def or(other) - Or.make([self, other]) - end - - def and(other) - And.make([self, other]) - end - - def negate - Not.make(self) - end - - alias_method :|, :or - alias_method :&, :and - alias_method :~@, :negate - - def inspect - "#<Rule #{repr}>" - end - end - - # A rule that checks a condition. This is the - # type of rule that results from a basic bareword - # in the rule dsl (see RuleDsl#method_missing). - class Condition < Base - def initialize(name) - @name = name - end - - # we delegate scoring to the condition. See - # ManifestCondition#score. - def score(context) - context.condition(@name).score - end - - # Let the ManifestCondition from the context - # decide whether we pass. - def pass?(context) - context.condition(@name).pass? - end - - # returns nil unless it's already cached - def cached_pass?(context) - condition = context.condition(@name) - return unless condition.cached? - - condition.pass? - end - - def description(context) - context.class.conditions[@name].description - end - - def repr - @name.to_s - end - end - - # A rule constructed from DelegateDsl - using a condition from a - # delegated policy. - class DelegatedCondition < Base - # Internal use only - this is rescued each time it's raised. - MissingDelegate = Class.new(StandardError) - - def initialize(delegate_name, name) - @delegate_name = delegate_name - @name = name - end - - def delegated_context(context) - policy = context.delegated_policies[@delegate_name] - raise MissingDelegate if policy.nil? - - policy - end - - def score(context) - delegated_context(context).condition(@name).score - rescue MissingDelegate - 0 - end - - def cached_pass?(context) - condition = delegated_context(context).condition(@name) - return unless condition.cached? - - condition.pass? - rescue MissingDelegate - false - end - - def pass?(context) - delegated_context(context).condition(@name).pass? - rescue MissingDelegate - false - end - - def repr - "#{@delegate_name}.#{@name}" - end - end - - # A rule constructed from RuleDsl#can?. Computes a different ability - # on the same subject. - class Ability < Base - attr_reader :ability - def initialize(ability) - @ability = ability - end - - # We ask the ability's runner for a score - def score(context) - context.runner(@ability).score - end - - def pass?(context) - context.allowed?(@ability) - end - - def cached_pass?(context) - runner = context.runner(@ability) - return unless runner.cached? - - runner.pass? - end - - def description(context) - "User can #{@ability.inspect}" - end - - def repr - "can?(#{@ability.inspect})" - end - end - - # Logical `and`, containing a list of rules. Only passes - # if all of them do. - class And < Base - attr_reader :rules - def initialize(rules) - @rules = rules - end - - def simplify - simplified_rules = @rules.flat_map do |rule| - simplified = rule.simplify - case simplified - when And then simplified.rules - else [simplified] - end - end - - And.new(simplified_rules) - end - - def score(context) - return 0 unless cached_pass?(context).nil? - - # note that cached rules will have score 0 anyways. - @rules.map { |r| r.score(context) }.inject(0, :+) - end - - def pass?(context) - # try to find a cached answer before - # checking in order - cached = cached_pass?(context) - return cached unless cached.nil? - - @rules.all? { |r| r.pass?(context) } - end - - def cached_pass?(context) - @rules.each do |rule| - pass = rule.cached_pass?(context) - - return pass if pass.nil? || pass == false - end - - true - end - - def repr - "all?(#{rules.map(&:repr).join(', ')})" - end - end - - # Logical `or`. Mirrors And. - class Or < Base - attr_reader :rules - def initialize(rules) - @rules = rules - end - - def pass?(context) - cached = cached_pass?(context) - return cached unless cached.nil? - - @rules.any? { |r| r.pass?(context) } - end - - def simplify - simplified_rules = @rules.flat_map do |rule| - simplified = rule.simplify - case simplified - when Or then simplified.rules - else [simplified] - end - end - - Or.new(simplified_rules) - end - - def cached_pass?(context) - @rules.each do |rule| - pass = rule.cached_pass?(context) - - return pass if pass.nil? || pass == true - end - - false - end - - def score(context) - return 0 unless cached_pass?(context).nil? - - @rules.map { |r| r.score(context) }.inject(0, :+) - end - - def repr - "any?(#{@rules.map(&:repr).join(', ')})" - end - end - - class Not < Base - attr_reader :rule - def initialize(rule) - @rule = rule - end - - def simplify - case @rule - when And then Or.new(@rule.rules.map(&:negate)).simplify - when Or then And.new(@rule.rules.map(&:negate)).simplify - when Not then @rule.rule.simplify - else Not.new(@rule.simplify) - end - end - - def pass?(context) - !@rule.pass?(context) - end - - def cached_pass?(context) - case @rule.cached_pass?(context) - when nil then nil - when true then false - when false then true - end - end - - def score(context) - @rule.score(context) - end - - def repr - "~#{@rule.repr}" - end - end - end -end diff --git a/lib/declarative_policy/rule_dsl.rb b/lib/declarative_policy/rule_dsl.rb deleted file mode 100644 index 85da7f261fa..00000000000 --- a/lib/declarative_policy/rule_dsl.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - # The DSL evaluation context inside rule { ... } blocks. - # Responsible for creating and combining Rule objects. - # - # See Base.rule - class RuleDsl - def initialize(context_class) - @context_class = context_class - end - - def can?(ability) - Rule::Ability.new(ability) - end - - def all?(*rules) - Rule::And.make(rules) - end - - def any?(*rules) - Rule::Or.make(rules) - end - - def none?(*rules) - ~Rule::Or.new(rules) - end - - def cond(condition) - Rule::Condition.new(condition) - end - - def delegate(delegate_name, condition) - Rule::DelegatedCondition.new(delegate_name, condition) - end - - def method_missing(msg, *args) - return super unless args.empty? && !block_given? - - if @context_class.delegations.key?(msg) - DelegateDsl.new(self, msg) - else - cond(msg.to_sym) - end - end - end -end diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb deleted file mode 100644 index 59588b4d84e..00000000000 --- a/lib/declarative_policy/runner.rb +++ /dev/null @@ -1,196 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - class Runner - class State - def initialize - @enabled = false - @prevented = false - end - - def enable! - @enabled = true - end - - def enabled? - @enabled - end - - def prevent! - @prevented = true - end - - def prevented? - @prevented - end - - def pass? - !prevented? && enabled? - end - end - - # a Runner contains a list of Steps to be run. - attr_reader :steps - def initialize(steps) - @steps = steps - @state = nil - end - - # We make sure only to run any given Runner once, - # and just continue to use the resulting @state - # that's left behind. - def cached? - !!@state - end - - # used by Rule::Ability. See #steps_by_score - def score - return 0 if cached? - - steps.map(&:score).inject(0, :+) - end - - def merge_runner(other) - Runner.new(@steps + other.steps) - end - - # The main entry point, called for making an ability decision. - # See #run and DeclarativePolicy::Base#can? - def pass? - run unless cached? - - @state.pass? - end - - # see DeclarativePolicy::Base#debug - def debug(out = $stderr) - run(out) - end - - private - - def flatten_steps! - @steps = @steps.flat_map { |s| s.flattened(@steps) } - end - - # This method implements the semantic of "one enable and no prevents". - # It relies on #steps_by_score for the main loop, and updates @state - # with the result of the step. - def run(debug = nil) - @state = State.new - - steps_by_score do |step, score| - break if !debug && @state.prevented? - - passed = nil - case step.action - when :enable then - # we only check :enable actions if they have a chance of - # changing the outcome - if no other rule has enabled or - # prevented. - unless @state.enabled? || @state.prevented? - passed = step.pass? - @state.enable! if passed - end - - debug << inspect_step(step, score, passed) if debug - when :prevent then - # we only check :prevent actions if the state hasn't already - # been prevented. - unless @state.prevented? - passed = step.pass? - @state.prevent! if passed - end - - debug << inspect_step(step, score, passed) if debug - else raise "invalid action #{step.action.inspect}" - end - end - - @state - end - - # This is the core spot where all those `#score` methods matter. - # It is critical for performance to run steps in the correct order, - # so that we don't compute expensive conditions (potentially n times - # if we're called on, say, a large list of users). - # - # In order to determine the cheapest step to run next, we rely on - # Step#score, which returns a numerical rating of how expensive - # it would be to calculate - the lower the better. It would be - # easy enough to statically sort by these scores, but we can do - # a little better - the scores are cache-aware (conditions that - # are already in the cache have score 0), which means that running - # a step can actually change the scores of other steps. - # - # So! The way we sort here involves re-scoring at every step. This - # is by necessity quadratic, but most of the time the number of steps - # will be low. But just in case, if the number of steps exceeds 50, - # we print a warning and fall back to a static sort. - # - # For each step, we yield the step object along with the computed score - # for debugging purposes. - def steps_by_score - flatten_steps! - - if @steps.size > 50 - warn "DeclarativePolicy: large number of steps (#{steps.size}), falling back to static sort" - - @steps.map { |s| [s.score, s] }.sort_by { |(score, _)| score }.each do |(score, step)| - yield step, score - end - - return - end - - remaining_steps = Set.new(@steps) - remaining_enablers, remaining_preventers = remaining_steps.partition(&:enable?).map { |s| Set.new(s) } - - loop do - if @state.enabled? - # Once we set this, we never need to unset it, because a single - # prevent will stop this from being enabled - remaining_steps = remaining_preventers - else - # if the permission hasn't yet been enabled and we only have - # prevent steps left, we short-circuit the state here - @state.prevent! if remaining_enablers.empty? - end - - return if remaining_steps.empty? - - lowest_score = Float::INFINITY - next_step = nil - - remaining_steps.each do |step| - score = step.score - - if score < lowest_score - next_step = step - lowest_score = score - end - - break if lowest_score == 0 - end - - [remaining_steps, remaining_enablers, remaining_preventers].each do |set| - set.delete(next_step) - end - - yield next_step, lowest_score - end - end - - # Formatter for debugging output. - def inspect_step(step, original_score, passed) - symbol = - case passed - when true then '+' - when false then '-' - when nil then ' ' - end - - "#{symbol} [#{original_score.to_i}] #{step.repr}\n" - end - end -end diff --git a/lib/declarative_policy/step.rb b/lib/declarative_policy/step.rb deleted file mode 100644 index c289c17cc19..00000000000 --- a/lib/declarative_policy/step.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -module DeclarativePolicy - # This object represents one step in the runtime decision of whether - # an ability is allowed. It contains a Rule and a context (instance - # of DeclarativePolicy::Base), which contains the user, the subject, - # and the cache. It also contains an "action", which is the symbol - # :prevent or :enable. - class Step - attr_reader :context, :rule, :action - def initialize(context, rule, action) - @context = context - @rule = rule - @action = action - end - - # In the flattening process, duplicate steps may be generated in the - # same rule. This allows us to eliminate those (see Runner#steps_by_score - # and note its use of a Set) - def ==(other) - @context == other.context && @rule == other.rule && @action == other.action - end - - # In the runner, steps are sorted dynamically by score, so that - # we are sure to compute them in close to the optimal order. - # - # See also Rule#score, ManifestCondition#score, and Runner#steps_by_score. - def score - # we slightly prefer the preventative actions - # since they are more likely to short-circuit - case @action - when :prevent - @rule.score(@context) * (7.0 / 8) - when :enable - @rule.score(@context) - end - end - - def with_action(action) - Step.new(@context, @rule, action) - end - - def enable? - @action == :enable - end - - def prevent? - @action == :prevent - end - - # This rather complex method allows us to split rules into parts so that - # they can be sorted independently for better optimization - def flattened(roots) - case @rule - when Rule::Or - # A single `Or` step is the same as each of its elements as separate steps - @rule.rules.flat_map { |r| Step.new(@context, r, @action).flattened(roots) } - when Rule::Ability - # This looks like a weird micro-optimization but it buys us quite a lot - # in some cases. If we depend on an Ability (i.e. a `can?(...)` rule), - # and that ability *only* has :enable actions (modulo some actions that - # we already have taken care of), then its rules can be safely inlined. - steps = @context.runner(@rule.ability).steps.reject { |s| roots.include?(s) } - - if steps.all?(&:enable?) - # in the case that we are a :prevent step, each inlined step becomes - # an independent :prevent, even though it was an :enable in its initial - # context. - steps.map! { |s| s.with_action(:prevent) } if prevent? - - steps.flat_map { |s| s.flattened(roots) } - else - [self] - end - else - [self] - end - end - - def pass? - @rule.pass?(@context) - end - - def repr - "#{@action} when #{@rule.repr} (#{@context.repr})" - end - end -end diff --git a/lib/error_tracking/sentry_client.rb b/lib/error_tracking/sentry_client.rb index 68e64fba093..8d1bcec032d 100644 --- a/lib/error_tracking/sentry_client.rb +++ b/lib/error_tracking/sentry_client.rb @@ -80,7 +80,7 @@ module ErrorTracking raise_error 'Sentry returned invalid SSL data' rescue Errno::ECONNREFUSED raise_error 'Connection refused' - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e) raise_error "Sentry request failed due to #{e.class}" end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 0b5833b91ed..915ab355508 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -62,4 +62,4 @@ class EventFilter end end -EventFilter.prepend_if_ee('EE::EventFilter') +EventFilter.prepend_mod_with('EventFilter') diff --git a/lib/feature.rb b/lib/feature.rb index 709610b91be..87abd2689d0 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -249,4 +249,4 @@ class Feature end end -Feature::ActiveSupportCacheStoreAdapter.prepend_if_ee('EE::Feature::ActiveSupportCacheStoreAdapter') +Feature::ActiveSupportCacheStoreAdapter.prepend_mod_with('Feature::ActiveSupportCacheStoreAdapter') diff --git a/lib/feature/active_support_cache_store_adapter.rb b/lib/feature/active_support_cache_store_adapter.rb index ae2d623abe1..431f1169a86 100644 --- a/lib/feature/active_support_cache_store_adapter.rb +++ b/lib/feature/active_support_cache_store_adapter.rb @@ -4,18 +4,27 @@ # This class was already nested this way before moving to a separate file class Feature class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore + # This patch represents https://github.com/jnunemaker/flipper/pull/512. In + # Flipper 0.21.0 and later, we can remove this and just pass `write_through: + # true` to the constructor in `Feature.build_flipper_instance`. + + extend ::Gitlab::Utils::Override + + override :enable def enable(feature, gate, thing) result = @adapter.enable(feature, gate, thing) @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) result end + override :disable def disable(feature, gate, thing) result = @adapter.disable(feature, gate, thing) @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) result end + override :remove def remove(feature) result = @adapter.remove(feature) @cache.delete(FeaturesKey) diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index 8d9b2fa5234..cd2f5cb07a2 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -153,7 +153,7 @@ class Feature definition.deep_symbolize_keys! self.new(path, definition).tap(&:validate!) - rescue => e + rescue StandardError => e raise Feature::InvalidFeatureFlagError, "Invalid definition for `#{path}`: #{e.message}" end @@ -185,4 +185,4 @@ class Feature end end -Feature::Definition.prepend_if_ee('EE::Feature::Definition') +Feature::Definition.prepend_mod_with('Feature::Definition') diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index e9868732172..ac3210b4e98 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -62,7 +62,7 @@ class FileSizeValidator < ActiveModel::EachValidator default_message = options[MESSAGES[key]] errors_options[:message] ||= default_message if default_message - record.errors.add(attribute, MESSAGES[key], errors_options) + record.errors.add(attribute, MESSAGES[key], **errors_options) end end diff --git a/lib/flowdock/git.rb b/lib/flowdock/git.rb index f165ecfc1fa..539fd66a510 100644 --- a/lib/flowdock/git.rb +++ b/lib/flowdock/git.rb @@ -17,7 +17,7 @@ module Flowdock end def initialize(ref, from, to, options = {}) - raise TokenError.new("Flowdock API token not found") unless options[:token] + raise TokenError, "Flowdock API token not found" unless options[:token] @ref = ref @from = from diff --git a/lib/generators/gitlab/snowplow_event_definition_generator.rb b/lib/generators/gitlab/snowplow_event_definition_generator.rb new file mode 100644 index 00000000000..497d0cd512a --- /dev/null +++ b/lib/generators/gitlab/snowplow_event_definition_generator.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module Gitlab + class SnowplowEventDefinitionGenerator < Rails::Generators::Base + CE_DIR = 'config/events' + EE_DIR = 'ee/config/events' + + source_root File.expand_path('../../../generator_templates/snowplow_event_definition', __dir__) + + desc 'Generates an event definition yml file' + + class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if event is for ee' + class_option :category, type: :string, optional: false, desc: 'Category of the event' + class_option :action, type: :string, optional: false, desc: 'Action of the event' + class_option :force, type: :boolean, optional: true, default: false, desc: 'Overwrite existing definition' + + def create_event_file + raise "Event definition already exists at #{file_path}" if definition_exists? && !force_definition_override? + + template "event_definition.yml", file_path, force: force_definition_override? + end + + def distributions + (ee? ? ['- ee'] : ['- ce', '- ee']).join("\n") + end + + def event_category + options[:category] + end + + def event_action + options[:action] + end + + def milestone + Gitlab::VERSION.match('(\d+\.\d+)').captures.first + end + + def ee? + options[:ee] + end + + def force_definition_override? + options[:force] + end + + private + + def definition_exists? + File.exist?(ce_file_path) || File.exist?(ee_file_path) + end + + def file_path + ee? ? ee_file_path : ce_file_path + end + + def ce_file_path + File.join(CE_DIR, file_name) + end + + def ee_file_path + File.join(EE_DIR, file_name) + end + + def file_name + "#{event_category}_#{event_action}.yml".underscore.gsub("/", "__") + end + end +end diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb index d826c51a73d..e343e2dcf91 100644 --- a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'rails/generators' -require_relative '../usage_metric_definition_generator' module Gitlab module UsageMetricDefinition @@ -10,14 +9,24 @@ module Gitlab argument :category, type: :string, desc: "Category name" argument :event, type: :string, desc: "Event name" + class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' def create_metrics - Gitlab::UsageMetricDefinitionGenerator.start(["#{key_path}_weekly", '--dir', '7d']) - Gitlab::UsageMetricDefinitionGenerator.start(["#{key_path}_monthly", '--dir', '28d']) + weekly_params = ["#{key_path}_weekly", '--dir', '7d'] + weekly_params << '--ee' if ee? + Gitlab::UsageMetricDefinitionGenerator.start(weekly_params) + + monthly_params = ["#{key_path}_monthly", '--dir', '28d'] + monthly_params << '--ee' if ee? + Gitlab::UsageMetricDefinitionGenerator.start(monthly_params) end private + def ee? + options[:ee] + end + def key_path "redis_hll_counters.#{category}.#{event}" end diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb index cadc319a212..2d65363bf7b 100644 --- a/lib/generators/gitlab/usage_metric_definition_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -53,9 +53,11 @@ module Gitlab end def distribution - value = ['- ce'] - value << '- ee' if ee? - value.join("\n") + (ee? ? ['- ee'] : ['- ce', '- ee']).join("\n") + end + + def tier + (ee? ? ['#- premium', '- ultimate'] : ['- free', '- premium', '- ultimate']).join("\n") end def milestone diff --git a/lib/gitlab.rb b/lib/gitlab.rb index ddf08c8dc20..86bb2f662e5 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -92,6 +92,16 @@ module Gitlab Rails.env.development? || Rails.env.test? end + def self.extensions + if jh? + %w[ee jh] + elsif ee? + %w[ee] + else + %w[] + end + end + def self.ee? @is_ee ||= # We use this method when the Rails environment is not loaded. This diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 830980f0997..6afcd745d4e 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -159,4 +159,4 @@ module Gitlab end end -Gitlab::Access.prepend_if_ee('EE::Gitlab::Access') +Gitlab::Access.prepend_mod_with('Gitlab::Access') diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index a1063001330..1b67b91e839 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -49,4 +49,4 @@ module Gitlab end end -Gitlab::AlertManagement::Payload.prepend_if_ee('EE::Gitlab::AlertManagement::Payload') +Gitlab::AlertManagement::Payload.prepend_mod_with('Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 786c5bf675b..5e535ded439 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -130,7 +130,7 @@ module Gitlab strong_memoize(:environment) do next unless environment_name - EnvironmentsFinder + ::Environments::EnvironmentsFinder .new(project, nil, { name: environment_name }) .execute .first @@ -193,7 +193,7 @@ module Gitlab def parse_time(value) Time.parse(value).utc - rescue ArgumentError + rescue ArgumentError, TypeError end def parse_integer(value) diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index e2db9b62dd5..15238b5e50f 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -27,4 +27,4 @@ module Gitlab end end -Gitlab::AlertManagement::Payload::Generic.prepend_if_ee('EE::Gitlab::AlertManagement::Payload::Generic') +Gitlab::AlertManagement::Payload::Generic.prepend_mod_with('Gitlab::AlertManagement::Payload::Generic') diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb index a449b71b165..7140d31d536 100644 --- a/lib/gitlab/analytics/cycle_analytics/average.rb +++ b/lib/gitlab/analytics/cycle_analytics/average.rb @@ -7,9 +7,10 @@ module Gitlab include Gitlab::Utils::StrongMemoize include StageQueryHelpers - def initialize(stage:, query:) + def initialize(stage:, query:, params: {}) @stage = stage @query = query + @params = params end def seconds @@ -22,7 +23,7 @@ module Gitlab private - attr_reader :stage + attr_reader :stage, :params # rubocop: disable CodeReuse/ActiveRecord def select_average diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index 4dec71b35e8..c7987d63153 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -5,6 +5,7 @@ module Gitlab module CycleAnalytics class BaseQueryBuilder include Gitlab::CycleAnalytics::MetricsTables + include StageQueryHelpers delegate :subject_class, to: :stage @@ -13,17 +14,19 @@ module Gitlab Issue.to_s => IssuesFinder }.freeze + DEFAULT_END_EVENT_FILTER = :finished + def initialize(stage:, params: {}) @stage = stage @params = build_finder_params(params) + @params[:state] = :opened if in_progress? end # rubocop: disable CodeReuse/ActiveRecord def build query = finder.execute query = stage.start_event.apply_query_customization(query) - query = stage.end_event.apply_query_customization(query) - query.where(duration_condition) + apply_end_event_query_customization(query) end # rubocop: enable CodeReuse/ActiveRecord @@ -46,6 +49,7 @@ module Gitlab def build_finder_params(params) {}.tap do |finder_params| finder_params[:current_user] = params[:current_user] + finder_params[:end_event_filter] = params[:end_event_filter] || DEFAULT_END_EVENT_FILTER add_parent_model_params!(finder_params) add_time_range_params!(finder_params, params[:from], params[:to]) @@ -62,9 +66,20 @@ module Gitlab finder_params[:created_after] = from || 30.days.ago finder_params[:created_before] = to if to end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_end_event_query_customization(query) + if in_progress? + stage.end_event.apply_negated_query_customization(query) + else + query = stage.end_event.apply_query_customization(query) + query.where(duration_condition) + end + end + # rubocop: enable CodeReuse/ActiveRecord end end end end -Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder') +Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder') diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 10a008a76d5..56179533ffb 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -12,6 +12,8 @@ module Gitlab class DataCollector include Gitlab::Utils::StrongMemoize + MAX_COUNT = 1001 + delegate :serialized_records, to: :records_fetcher def initialize(stage:, params: {}) @@ -27,13 +29,19 @@ module Gitlab def median strong_memoize(:median) do - Median.new(stage: stage, query: query) + Median.new(stage: stage, query: query, params: params) end end def average strong_memoize(:average) do - Average.new(stage: stage, query: query) + Average.new(stage: stage, query: query, params: params) + end + end + + def count + strong_memoize(:count) do + limit_count end end @@ -44,9 +52,16 @@ module Gitlab def query BaseQueryBuilder.new(stage: stage, params: params).build end + + # Limiting the maximum number of records so the COUNT(*) query stays efficient for large groups. + # COUNT = 1001, show 1000+ on the UI + # COUNT < 1001, show the actual number on the UI + def limit_count + query.limit(MAX_COUNT).count + end end end end end -Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::DataCollector') +Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::DataCollector') diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb index 6c0450ac9e5..5775d0324c6 100644 --- a/lib/gitlab/analytics/cycle_analytics/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -6,9 +6,10 @@ module Gitlab class Median include StageQueryHelpers - def initialize(stage:, query:) + def initialize(stage:, query:, params: {}) @stage = stage @query = query + @params = params end # rubocop: disable CodeReuse/ActiveRecord @@ -26,7 +27,7 @@ module Gitlab private - attr_reader :stage + attr_reader :stage, :params def percentile_cont percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration) diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index b4752ed9e5b..9a37a41ff81 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -124,7 +124,7 @@ module Gitlab def time_columns [ stage.start_event.timestamp_projection.as('start_event_timestamp'), - stage.end_event.timestamp_projection.as('end_event_timestamp'), + end_event_timestamp_projection.as('end_event_timestamp'), round_duration_to_seconds.as('total_time') ] end @@ -133,4 +133,4 @@ module Gitlab end end -Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher') +Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::RecordsFetcher') diff --git a/lib/gitlab/analytics/cycle_analytics/sorting.rb b/lib/gitlab/analytics/cycle_analytics/sorting.rb index 828879d466d..c399bac423b 100644 --- a/lib/gitlab/analytics/cycle_analytics/sorting.rb +++ b/lib/gitlab/analytics/cycle_analytics/sorting.rb @@ -4,23 +4,35 @@ module Gitlab module Analytics module CycleAnalytics class Sorting + include StageQueryHelpers + + def initialize(stage:, query:, params: {}) + @stage = stage + @query = query + @params = params + end + # rubocop: disable CodeReuse/ActiveRecord - SORTING_OPTIONS = { - end_event: { - asc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.asc) }, - desc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.desc) } - }.freeze, - duration: { - asc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).asc) }, - desc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).desc) } - }.freeze - }.freeze - # rubocop: enable CodeReuse/ActiveRecord, + def apply(sort, direction) + sorting_options = { + end_event: { + asc: -> { query.reorder(end_event_timestamp_projection.asc) }, + desc: -> { query.reorder(end_event_timestamp_projection.desc) } + }, + duration: { + asc: -> { query.reorder(duration.asc) }, + desc: -> { query.reorder(duration.desc) } + } + } - def self.apply(query, stage, sort, direction) - sort_lambda = SORTING_OPTIONS.dig(sort, direction) || SORTING_OPTIONS.dig(:end_event, :desc) - sort_lambda.call(query, stage) + sort_lambda = sorting_options.dig(sort, direction) || sorting_options.dig(:end_event, :desc) + sort_lambda.call end + # rubocop: enable CodeReuse/ActiveRecord + + private + + attr_reader :stage, :query, :params end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 02b1024b8b3..b7a11bc0418 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -85,4 +85,4 @@ module Gitlab end end -Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_if_ee('::EE::Gitlab::Analytics::CycleAnalytics::StageEvents') +Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::StageEvents') diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index 4bb225b63f1..8e87245e62b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -17,11 +17,6 @@ module Gitlab MergeRequest end - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', column_list) - end - - override :column_list def column_list [ issue_metrics_table[:first_mentioned_in_commit_at], diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb index a159580b7bd..30b457b667c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -17,8 +17,8 @@ module Gitlab Issue end - def timestamp_projection - issue_table[:created_at] + def column_list + [issue_table[:created_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb index 3e93e60e686..4ca3c19051e 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb @@ -17,13 +17,8 @@ module Gitlab Issue end - def timestamp_projection - mr_metrics_table[:first_deployed_to_production_at] - end - - override :column_list def column_list - [timestamp_projection] + [mr_metrics_table[:first_deployed_to_production_at]] end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb index a3b7fa16daf..aa509e8c4d2 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -17,8 +17,8 @@ module Gitlab Issue end - def timestamp_projection - issue_metrics_table[:first_mentioned_in_commit_at] + def column_list + [issue_metrics_table[:first_mentioned_in_commit_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb index 7c1f4436c93..284d8534b96 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -17,11 +17,6 @@ module Gitlab Issue end - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', column_list) - end - - override :column_list def column_list [ issue_metrics_table[:first_associated_with_milestone_at], diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb index 013e068e479..31249ae2036 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_table[:created_at] + def column_list + [mr_table[:created_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb index 654d0befbc3..4c0e9b61e64 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:first_deployed_to_production_at] + def column_list + [mr_metrics_table[:first_deployed_to_production_at]] end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb index a0b1c12756f..178fe03d7db 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:latest_build_finished_at] + def column_list + [mr_metrics_table[:latest_build_finished_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb index da3b5cdfaa4..95e59cd29a6 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:latest_build_started_at] + def column_list + [mr_metrics_table[:latest_build_started_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb index e67a6f7eea6..00ac2e7d56c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -17,8 +17,8 @@ module Gitlab MergeRequest end - def timestamp_projection - mr_metrics_table[:merged_at] + def column_list + [mr_metrics_table[:merged_at]] end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb index fe477490648..fd30ab5277d 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb @@ -11,7 +11,12 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - override :column_list + # rubocop: disable CodeReuse/ActiveRecord + def apply_negated_query_customization(query) + super.joins(:metrics) + end + # rubocop: enable CodeReuse/ActiveRecord + def column_list [timestamp_projection] end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index bddc326de71..9b4cbc9090c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -17,11 +17,6 @@ module Gitlab Issue end - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', column_list) - end - - override :column_list def column_list [ issue_metrics_table[:first_associated_with_milestone_at], diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index cfc9300a710..530e53f9d10 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -34,14 +34,16 @@ module Gitlab # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query. # Example: get me all the Issue records between start event end end event def timestamp_projection - raise NotImplementedError + columns = column_list + + columns.one? ? columns.first : Arel::Nodes::NamedFunction.new('COALESCE', columns) end # List of columns that are referenced in the `timestamp_projection` expression # Example timestamp projection: COALESCE(issue_metrics.created_at, issue_metrics.updated_at) # Expected column list: issue_metrics.created_at, issue_metrics.updated_at def column_list - [] + raise NotImplementedError end # Optionally a StageEvent may apply additional filtering or join other tables on the base query. @@ -49,6 +51,12 @@ module Gitlab query end + # rubocop: disable CodeReuse/ActiveRecord + def apply_negated_query_customization(query) + query.where(timestamp_projection.eq(nil)) + end + # rubocop: enable CodeReuse/ActiveRecord + def self.label_based? false end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 777a8278e6e..11fe1dde12f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -18,22 +18,30 @@ module Gitlab def duration Arel::Nodes::Subtraction.new( - stage.end_event.timestamp_projection, + end_event_timestamp_projection, stage.start_event.timestamp_projection ) end + def end_event_timestamp_projection + if in_progress? + Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i]) + else + stage.end_event.timestamp_projection + end + end + # rubocop: disable CodeReuse/ActiveRecord def order_by(query, sort, direction, extra_columns_to_select = [:id]) - ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.apply(query, stage, sort, direction) + ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.new(stage: stage, query: query, params: params).apply(sort, direction) # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause if requires_grouping? - column_list = [ - *extra_columns_to_select, - *stage.end_event.column_list, - *stage.start_event.column_list - ] + column_list = [].tap do |array| + array.concat(extra_columns_to_select) + array.concat(stage.end_event.column_list) unless in_progress? + array.concat(stage.start_event.column_list) + end ordered_query = ordered_query.group(column_list) end @@ -45,6 +53,10 @@ module Gitlab def requires_grouping? Array(params[:label_name]).size > 1 end + + def in_progress? + params[:end_event_filter] == :in_progress + end end end end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index 09039f3fc43..df342905d2e 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -10,7 +10,17 @@ module Gitlab attr_reader :location - validates :location, inclusion: { in: %i[http_basic_auth http_token] } + validates :location, inclusion: { + in: %i[ + http_basic_auth + http_token + http_bearer_token + http_deploy_token_header + http_job_token_header + http_private_token_header + token_param + ] + } def initialize(location) @location = location @@ -23,6 +33,16 @@ module Gitlab extract_from_http_basic_auth request when :http_token extract_from_http_token request + when :http_bearer_token + extract_from_http_bearer_token request + when :http_deploy_token_header + extract_from_http_deploy_token_header request + when :http_job_token_header + extract_from_http_job_token_header request + when :http_private_token_header + extract_from_http_private_token_header request + when :token_param + extract_from_token_param request end end @@ -41,6 +61,41 @@ module Gitlab UsernameAndPassword.new(nil, password) end + + def extract_from_http_bearer_token(request) + password = request.headers['Authorization'] + return unless password.present? + + UsernameAndPassword.new(nil, password.split(' ').last) + end + + def extract_from_http_deploy_token_header(request) + password = request.headers['Deploy-Token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end + + def extract_from_http_job_token_header(request) + password = request.headers['Job-Token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end + + def extract_from_http_private_token_header(request) + password = request.headers['Private-Token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end + + def extract_from_token_param(request) + password = request.query_parameters['token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index 9234837cdf7..dd9039e37f6 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -15,9 +15,14 @@ module Gitlab personal_access_token job_token deploy_token + personal_access_token_from_jwt + deploy_token_from_jwt + job_token_from_jwt ] } + UsernameAndPassword = ::Gitlab::APIAuthentication::TokenLocator::UsernameAndPassword + def initialize(token_type) @token_type = token_type validate! @@ -56,6 +61,15 @@ module Gitlab when :deploy_token_with_username resolve_deploy_token_with_username raw + + when :personal_access_token_from_jwt + resolve_personal_access_token_from_jwt raw + + when :deploy_token_from_jwt + resolve_deploy_token_from_jwt raw + + when :job_token_from_jwt + resolve_job_token_from_jwt raw end end @@ -116,6 +130,33 @@ module Gitlab end end + def resolve_personal_access_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(Integer) + + pat = ::PersonalAccessToken.find(jwt_token['token']) + break unless pat + + pat + end + end + + def resolve_deploy_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(String) + + resolve_deploy_token(UsernameAndPassword.new(nil, jwt_token['token'])) + end + end + + def resolve_job_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(String) + + resolve_job_token(UsernameAndPassword.new(nil, jwt_token['token'])) + end + end + def with_personal_access_token(raw, &block) pat = ::PersonalAccessToken.find_by_token(raw.password) return unless pat @@ -136,6 +177,13 @@ module Gitlab yield(job) end + + def with_jwt_token(raw, &block) + jwt_token = ::Gitlab::JWTToken.decode(raw.password) + raise ::Gitlab::Auth::UnauthorizedError unless jwt_token + + yield(jwt_token) + end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index ceda82cb6f6..601f2175cfc 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -135,4 +135,4 @@ module Gitlab end end -Gitlab::ApplicationContext.prepend_if_ee('EE::Gitlab::ApplicationContext') +Gitlab::ApplicationContext.prepend_mod_with('Gitlab::ApplicationContext') diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index f74edf2b767..f91a56a0cd2 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -34,6 +34,7 @@ module Gitlab group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, + web_hook_calls: { interval: 1.minute }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, auto_rollback_deployment: { threshold: 1, interval: 3.minutes } diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb deleted file mode 100644 index 4f047ab3ea8..00000000000 --- a/lib/gitlab/artifacts/migration_helper.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Artifacts - class MigrationHelper - def migrate_to_remote_storage(&block) - artifacts = ::Ci::JobArtifact.with_files_stored_locally - migrate(artifacts, ObjectStorage::Store::REMOTE, &block) - end - - def migrate_to_local_storage(&block) - artifacts = ::Ci::JobArtifact.with_files_stored_remotely - migrate(artifacts, ObjectStorage::Store::LOCAL, &block) - end - - private - - def batch_size - ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i - end - - def migrate(artifacts, store, &block) - artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord - artifact.file.migrate!(store) - - yield artifact if block - rescue => e - raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}") - end - end - end - end -end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 1f5cce249d8..c6997288b65 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -29,7 +29,7 @@ module Gitlab CI_JOB_USER = 'gitlab-ci-token' class << self - prepend_if_ee('EE::Gitlab::Auth') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('Gitlab::Auth') # rubocop: disable Cop/InjectEnterpriseEditionModule def omniauth_enabled? Gitlab.config.omniauth.enabled @@ -156,9 +156,9 @@ module Gitlab underscored_service = matched_login['service'].underscore - if Service.available_services_names.include?(underscored_service) + if Integration.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included - # in the Service.available_services_names allowlist. + # in the Integration.available_services_names allowlist. service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend if service && service.activated? && service.valid_token?(password) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 6f6ac79c16b..416e36c7ccb 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -160,7 +160,7 @@ module Gitlab case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes) when AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + raise InsufficientScopeError, scopes when AccessTokenValidationService::EXPIRED raise ExpiredError when AccessTokenValidationService::REVOKED @@ -321,4 +321,4 @@ module Gitlab end end -Gitlab::Auth::AuthFinders.prepend_if_ee('::EE::Gitlab::Auth::AuthFinders') +Gitlab::Auth::AuthFinders.prepend_mod_with('Gitlab::Auth::AuthFinders') diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb index c0dc2b0875f..bf35a9abe41 100644 --- a/lib/gitlab/auth/database/authentication.rb +++ b/lib/gitlab/auth/database/authentication.rb @@ -9,6 +9,7 @@ module Gitlab class Authentication < Gitlab::Auth::OAuth::Authentication def login(login, password) return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? + return false if user.password_based_login_forbidden? return user if user&.valid_password?(password) end diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 66d20ee2b59..62a817d7c4d 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -117,4 +117,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Access.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access') +Gitlab::Auth::Ldap::Access.prepend_mod_with('Gitlab::Auth::Ldap::Access') diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 7f85d3b1cd3..3853709698b 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -141,4 +141,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Adapter.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter') +Gitlab::Auth::Ldap::Adapter.prepend_mod_with('Gitlab::Auth::Ldap::Adapter') diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 97e4f921228..441f0d14b39 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -59,7 +59,7 @@ module Gitlab end def self.invalid_provider(provider) - raise InvalidProvider.new("Unknown provider (#{provider}). Available providers: #{providers}") + raise InvalidProvider, "Unknown provider (#{provider}). Available providers: #{providers}" end def self.encrypted_secrets @@ -288,7 +288,7 @@ module Gitlab def secrets @secrets ||= self.class.encrypted_secrets[@provider.delete_prefix('ldap').to_sym] - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "LDAP encrypted secrets are invalid: #{e.inspect}" nil @@ -320,4 +320,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Config.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config') +Gitlab::Auth::Ldap::Config.prepend_mod_with('Gitlab::Auth::Ldap::Config') diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index 102820d6bd5..79e1937478c 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -121,4 +121,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::Person.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person') +Gitlab::Auth::Ldap::Person.prepend_mod_with('Gitlab::Auth::Ldap::Person') diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 814c17b7e44..d134350775d 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -49,4 +49,4 @@ module Gitlab end end -Gitlab::Auth::Ldap::User.prepend_if_ee('::EE::Gitlab::Auth::Ldap::User') +Gitlab::Auth::Ldap::User.prepend_mod_with('Gitlab::Auth::Ldap::User') diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 46ff6b2ccab..2ec75669d24 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -81,7 +81,7 @@ module Gitlab # Get the first part of the email address (before @) # In addition in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s + email.match(/^[^@]*/)[0].mb_chars.unicode_normalize(:nfkd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) @@ -92,4 +92,4 @@ module Gitlab end end -Gitlab::Auth::OAuth::AuthHash.prepend_if_ee('::EE::Gitlab::Auth::OAuth::AuthHash') +Gitlab::Auth::OAuth::AuthHash.prepend_mod_with('Gitlab::Auth::OAuth::AuthHash') diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index fe1bf730e76..523452d1074 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -115,6 +115,8 @@ module Gitlab log.info "Correct LDAP account has been found. identity to user: #{gl_user.username}." gl_user.identities.build(provider: ldap_person.provider, extern_uid: ldap_person.dn) end + + identity end def find_or_build_ldap_user @@ -292,4 +294,4 @@ module Gitlab end end -Gitlab::Auth::OAuth::User.prepend_if_ee('::EE::Gitlab::Auth::OAuth::User') +Gitlab::Auth::OAuth::User.prepend_mod_with('Gitlab::Auth::OAuth::User') diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 757a0e671c3..da874524826 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -25,4 +25,4 @@ module Gitlab end end -Gitlab::Auth::Result.prepend_if_ee('::EE::Gitlab::Auth::Result') +Gitlab::Auth::Result.prepend_mod_with('Gitlab::Auth::Result') diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 67a53fa3205..3f13a264b0a 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -30,4 +30,4 @@ module Gitlab end end -Gitlab::Auth::Saml::Config.prepend_if_ee('::EE::Gitlab::Auth::Saml::Config') +Gitlab::Auth::Saml::Config.prepend_mod_with('Gitlab::Auth::Saml::Config') diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index 37bc3f9bed0..205d5fe0015 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -62,4 +62,4 @@ module Gitlab end end -Gitlab::Auth::Saml::User.prepend_if_ee('::EE::Gitlab::Auth::Saml::User') +Gitlab::Auth::Saml::User.prepend_mod_with('Gitlab::Auth::Saml::User') diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index 50cd15b7a10..e7eba65bea8 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -161,7 +161,7 @@ module Gitlab end def strip(key) - key.split(/[ ]+/)[0, 2].join(' ') + key.split(/ +/)[0, 2].join(' ') end end end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb new file mode 100644 index 00000000000..79e7a2f2279 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set namespaces.traversal_ids in sub-batches, of all namespaces with + # a parent and not already set. + # rubocop:disable Style/Documentation + class BackfillNamespaceTraversalIdsChildren + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :base_query, -> { where.not(parent_id: nil) } + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + UPDATE namespaces + SET traversal_ids = calculated_ids.traversal_ids + FROM #{calculated_traversal_ids(ranged_query)} calculated_ids + WHERE namespaces.id = calculated_ids.id + AND namespaces.traversal_ids = '{}' + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + # Calculate the ancestor path for a given set of namespaces. + def calculated_traversal_ids(batch) + <<~SQL + ( + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + ) + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceTraversalIdsChildren', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb new file mode 100644 index 00000000000..1c0a83285a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set namespaces.traversal_ids in sub-batches, of all namespaces + # without a parent and not already set. + # rubocop:disable Style/Documentation + class BackfillNamespaceTraversalIdsRoots + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + + scope :base_query, -> { where(parent_id: nil) } + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + ranged_query = Namespace.base_query + .where(id: start_id..end_id) + .where("traversal_ids = '{}'") + + ranged_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + Namespace.unscoped + .base_query + .where(id: first..last) + .where("traversal_ids = '{}'") + .update_all('traversal_ids = ARRAY[id]') + + sleep PAUSE_SECONDS + end + + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceTraversalIdsRoots', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 8befade8c3a..6f37f1846d2 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -36,7 +36,7 @@ module Gitlab create_repository_and_files(snippet) logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id) - rescue => e + rescue StandardError => e set_file_path_error(e) set_signature_error(e) @@ -68,7 +68,7 @@ module Gitlab # Removing the db record def destroy_snippet_repository(snippet) snippet.snippet_repository&.delete - rescue => e + rescue StandardError => e logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id) end @@ -78,7 +78,7 @@ module Gitlab snippet.repository.remove snippet.repository.expire_exists_cache - rescue => e + rescue StandardError => e logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id) end diff --git a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb index 83d60d2db19..41f7f7f2f24 100644 --- a/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb +++ b/lib/gitlab/background_migration/backfill_version_data_from_gitaly.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_if_ee('EE::Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') +Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_mod_with('Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly') diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index 76598f6e2a6..7b334b9c1d0 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -9,7 +9,7 @@ module Gitlab .where(id: start_id..stop_id) .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" end end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index b89ea7dc250..529b8cdf8d4 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -2,8 +2,8 @@ module Gitlab module BackgroundMigration - # Background migration that updates the value of a - # column using the value of another column in the same table. + # Background migration that updates the value of one or more + # columns using the value of other columns in the same table. # # - The {start_id, end_id} arguments are at the start so that it can be used # with `queue_batched_background_migration` @@ -16,8 +16,6 @@ module Gitlab class CopyColumnUsingBackgroundMigrationJob include Gitlab::Database::DynamicModelHelpers - PAUSE_SECONDS = 0.1 - # start_id - The start ID of the range of rows to update. # end_id - The end ID of the range of rows to update. # batch_table - The name of the table that contains the columns. @@ -25,20 +23,26 @@ module Gitlab # sub_batch_size - We don't want updates to take more than ~100ms # This allows us to run multiple smaller batches during # the minimum 2.minute interval that we can schedule jobs - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, copy_from, copy_to) - quoted_copy_from = connection.quote_column_name(copy_from) - quoted_copy_to = connection.quote_column_name(copy_to) + # pause_ms - The number of milliseconds to sleep between each subbatch execution. + # copy_from - List of columns containing the data to copy. + # copy_to - List of columns to copy the data to. Order must match the order in `copy_from`. + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, copy_from, copy_to) + copy_from = Array.wrap(copy_from) + copy_to = Array.wrap(copy_to) + + raise ArgumentError, 'number of source and destination columns must match' unless copy_from.count == copy_to.count + + assignment_clauses = column_assignment_clauses(copy_from, copy_to) parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| batch_metrics.time_operation(:update_all) do - sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + sub_batch.update_all(assignment_clauses) end - sleep(PAUSE_SECONDS) + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) end end @@ -55,6 +59,17 @@ module Gitlab def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) define_batchable_model(source_table).where(source_key_column => start_id..stop_id) end + + def column_assignment_clauses(copy_from, copy_to) + assignments = copy_from.zip(copy_to).map do |from_column, to_column| + from_column = connection.quote_column_name(from_column) + to_column = connection.quote_column_name(to_column) + + "#{to_column} = #{from_column}" + end + + assignments.join(', ') + end end end end diff --git a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb new file mode 100644 index 00000000000..293530f6536 --- /dev/null +++ b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::DropInvalidVulnerabilities + # rubocop: disable Gitlab/NamespacedClass + class Vulnerability < ActiveRecord::Base + self.table_name = "vulnerabilities" + has_many :findings, class_name: 'VulnerabilitiesFinding', inverse_of: :vulnerability + end + + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' + end + # rubocop: enable Gitlab/NamespacedClass + + # rubocop: disable CodeReuse/ActiveRecord + def perform(start_id, end_id) + Vulnerability + .where(id: start_id..end_id) + .left_joins(:findings) + .where(vulnerability_occurrences: { vulnerability_id: nil }) + .delete_all + + mark_job_as_succeeded(start_id, end_id) + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'DropInvalidVulnerabilities', + arguments + ) + end +end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index c0099d44b5a..7b5c32e3d6d 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -24,7 +24,7 @@ module Gitlab certificate_valid_not_before: domain.x509&.not_before&.iso8601, certificate_valid_not_after: domain.x509&.not_after&.iso8601 ) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" end end diff --git a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb index d6ec56ae19e..c50bf430d92 100644 --- a/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb +++ b/lib/gitlab/background_migration/fix_orphan_promoted_issues.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixOrphanPromotedIssues') +Gitlab::BackgroundMigration::FixOrphanPromotedIssues.prepend_mod_with('Gitlab::BackgroundMigration::FixOrphanPromotedIssues') diff --git a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb index 46921a070c3..47a68c61fcc 100644 --- a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb +++ b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') +Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_mod_with('Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') diff --git a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb index 85bcf8558f2..160e6d2fe8b 100644 --- a/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb +++ b/lib/gitlab/background_migration/generate_gitlab_subscriptions.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_if_ee('EE::Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') +Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_mod_with('Gitlab::BackgroundMigration::GenerateGitlabSubscriptions') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb index 27b984b4531..ba66721f65c 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules.rb @@ -12,4 +12,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRules') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb index 053b7363286..4899c50b9cf 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_check_progress.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress') diff --git a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb index 130f97b09d7..2855566d7e8 100644 --- a/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb +++ b/lib/gitlab/background_migration/migrate_approver_to_approval_rules_in_batch.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') +Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_mod_with('Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch') diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb index de2d9909961..d85f980d3f1 100644 --- a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb +++ b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') +Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_mod_with('Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb new file mode 100644 index 00000000000..68bbd3cfebb --- /dev/null +++ b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the context of project taggings from `tags` to `topics` + class MigrateProjectTaggingsContextFromTagsToTopics + # Temporary AR table for taggings + class Tagging < ActiveRecord::Base + include EachBatch + + self.table_name = 'taggings' + end + + def perform(start_id, stop_id) + Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| + relation.update_all(context: 'topics') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_security_scans.rb b/lib/gitlab/background_migration/migrate_security_scans.rb index 189a150cb87..0ae984f2dbc 100644 --- a/lib/gitlab/background_migration/migrate_security_scans.rb +++ b/lib/gitlab/background_migration/migrate_security_scans.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateSecurityScans') +Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_mod_with('Gitlab::BackgroundMigration::MigrateSecurityScans') diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index 9ecf53317d0..c01545e5dca 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -8,18 +8,15 @@ module Gitlab class MoveContainerRegistryEnabledToProjectFeature MAX_BATCH_SIZE = 300 - module Migratable - # Migration model namespace isolated from application code. - class ProjectFeature < ActiveRecord::Base - ENABLED = 20 - DISABLED = 0 - end - end + ENABLED = 20 + DISABLED = 0 def perform(from_id, to_id) (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch| process_batch(batch.first, batch.last) end + + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id]) end private @@ -37,9 +34,9 @@ module Gitlab <<~SQL UPDATE project_features SET container_registry_access_level = (CASE p.container_registry_enabled - WHEN true THEN #{ProjectFeature::ENABLED} - WHEN false THEN #{ProjectFeature::DISABLED} - ELSE #{ProjectFeature::DISABLED} + WHEN true THEN #{ENABLED} + WHEN false THEN #{DISABLED} + ELSE #{DISABLED} END) FROM projects p WHERE project_id = p.id AND diff --git a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb index dc982e703d1..174994c7862 100644 --- a/lib/gitlab/background_migration/move_epic_issues_after_epics.rb +++ b/lib/gitlab/background_migration/move_epic_issues_after_epics.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_if_ee('EE::Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') +Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_mod_with('Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb index c3c0db2495c..890a43800c9 100644 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_merge_requests.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests') diff --git a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb index 2243c7531c0..ac7ed18ba14 100644 --- a/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb +++ b/lib/gitlab/background_migration/populate_any_approval_rule_for_projects.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') +Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_mod_with('Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects') diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb index e352ae71de6..e873ad412f2 100644 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -13,4 +13,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics') +Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb index e8f436b183e..ed7ffce8018 100644 --- a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb +++ b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb @@ -33,7 +33,7 @@ module Gitlab def update_namespace_statistics(namespace) Namespaces::StatisticsRefresherService.new.execute(namespace) - rescue => e + rescue StandardError => e error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}") end diff --git a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb index 7659b63271f..37af320f044 100644 --- a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb +++ b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb @@ -11,12 +11,12 @@ module Gitlab namespace_snippets.group_by(&:project).each do |project, snippets| upsert_snippet_statistics(snippets) update_project_statistics(project) - rescue + rescue StandardError error_message("Error updating statistics for project #{project.id}") end update_namespace_statistics(namespace_snippets.first.project.root_namespace) - rescue => e + rescue StandardError => e error_message("Error updating statistics for namespace #{namespace_id}: #{e.message}") end end diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb index eb72ef1de33..e95955c450d 100644 --- a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb +++ b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb @@ -9,4 +9,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') +Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_mod_with('Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb index 4aff9d1e2c1..175966b940d 100644 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -15,4 +15,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') +Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_mod_with('Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb index fc79f7125e3..8241fea66db 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') +Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb index 2e81b1615d8..9a9f23e29ea 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -11,4 +11,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') +Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb index 8b16db8be35..0efbe72775c 100644 --- a/lib/gitlab/background_migration/prune_orphaned_geo_events.rb +++ b/lib/gitlab/background_migration/prune_orphaned_geo_events.rb @@ -14,4 +14,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') +Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_mod_with('Gitlab::BackgroundMigration::PruneOrphanedGeoEvents') diff --git a/lib/gitlab/background_migration/recalculate_project_authorizations.rb b/lib/gitlab/background_migration/recalculate_project_authorizations.rb index 3d2ce9fc10c..6a250a96c94 100644 --- a/lib/gitlab/background_migration/recalculate_project_authorizations.rb +++ b/lib/gitlab/background_migration/recalculate_project_authorizations.rb @@ -5,37 +5,7 @@ module Gitlab # rubocop:disable Style/Documentation class RecalculateProjectAuthorizations def perform(user_ids) - user_ids.each do |user_id| - user = User.find_by(id: user_id) - - next unless user - - service = Users::RefreshAuthorizedProjectsService.new( - user, - incorrect_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Removing ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end, - missing_auth_found_callback: - ->(project_id, access_level) do - logger.info(message: 'Creating ProjectAuthorizations', - user_id: user.id, - project_id: project_id, - access_level: access_level) - end - ) - - service.execute - end - end - - private - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build + # no-op end end end diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb index cc9b0329556..17ef6dec4c0 100644 --- a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb +++ b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') +Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb index cd305adc7cd..e5772fc7375 100644 --- a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb +++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') +Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_mod_with('Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb index 74c48b237cc..cb6a600a525 100644 --- a/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb +++ b/lib/gitlab/background_migration/remove_inaccessible_epic_todos.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') +Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos.prepend_mod_with('Gitlab::BackgroundMigration::RemoveInaccessibleEpicTodos') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb index 3920e8dc2de..540ffc6f548 100644 --- a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb index f137e41c728..cecb385afa0 100644 --- a/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_occurrence_severity_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') +Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceSeverityLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb index f6ea61f4502..4be61bfb689 100644 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel') diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb index 95540cd5f49..1ea483f929f 100644 --- a/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb +++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_severity_level.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') +Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel.prepend_mod_with('Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilitySeverityLevel') diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb index 6262320128c..49a632952fb 100644 --- a/lib/gitlab/background_migration/sync_blocking_issues_count.rb +++ b/lib/gitlab/background_migration/sync_blocking_issues_count.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount') +Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_mod_with('Gitlab::BackgroundMigration::SyncBlockingIssuesCount') diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb index 651df36fcfd..054b918dade 100644 --- a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb +++ b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') +Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_mod_with('Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb new file mode 100644 index 00000000000..24c9967b88e --- /dev/null +++ b/lib/gitlab/background_migration/update_timelogs_project_id.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to populate project_id for timelogs + class UpdateTimelogsProjectId + BATCH_SIZE = 1000 + + def perform(start_id, stop_id) + (start_id..stop_id).step(BATCH_SIZE).each do |offset| + update_issue_timelogs(offset, offset + BATCH_SIZE) + update_merge_request_timelogs(offset, offset + BATCH_SIZE) + end + end + + def update_issue_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET project_id = issues.project_id + FROM issues + WHERE issues.id = timelogs.issue_id + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} + AND timelogs.project_id IS NULL; + SQL + end + + def update_merge_request_timelogs(batch_start, batch_stop) + execute(<<~SQL) + UPDATE timelogs + SET project_id = merge_requests.target_project_id + FROM merge_requests + WHERE merge_requests.id = timelogs.merge_request_id + AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} + AND timelogs.project_id IS NULL; + SQL + end + + def execute(sql) + @connection ||= ::ActiveRecord::Base.connection + @connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb index bfe9f673b53..1cc03f061fb 100644 --- a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb +++ b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') +Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') diff --git a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb index a2940cba6fa..60adb6b7e3e 100644 --- a/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb +++ b/lib/gitlab/background_migration/update_vulnerabilities_to_dismissed.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed') +Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilitiesToDismissed') diff --git a/lib/gitlab/background_migration/update_vulnerability_confidence.rb b/lib/gitlab/background_migration/update_vulnerability_confidence.rb index 6ffaa836f3c..40d29978dd4 100644 --- a/lib/gitlab/background_migration/update_vulnerability_confidence.rb +++ b/lib/gitlab/background_migration/update_vulnerability_confidence.rb @@ -10,4 +10,4 @@ module Gitlab end end -Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') +Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence') diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index a2b50c41f4a..d76d06606ee 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -38,4 +38,4 @@ module Gitlab end end -Namespace.prepend_if_ee('::EE::Namespace') +Namespace.prepend_mod_with('Namespace') diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index ab7a08ffef9..44106897df8 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -13,7 +13,7 @@ module Gitlab repos_to_import = Dir.glob(import_path + '**/*.git') unless user = User.admins.order_id_asc.first - raise NoAdminError.new('No admin user found to import repositories') + raise NoAdminError, 'No admin user found to import repositories' end repos_to_import.each do |repo_path| @@ -92,7 +92,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e log " * Failed to move repo: #{e.message}".color(:red) false diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb index 57d632afd74..c5b183d113d 100644 --- a/lib/gitlab/blob_helper.rb +++ b/lib/gitlab/blob_helper.rb @@ -38,7 +38,7 @@ module Gitlab # If Charlock says its binary else - detect_encoding[:type] == :binary + find_encoding[:type] == :binary end end @@ -137,23 +137,25 @@ module Gitlab end def ruby_encoding - if hash = detect_encoding + if hash = find_encoding hash[:ruby_encoding] end end def encoding - if hash = detect_encoding + if hash = find_encoding hash[:encoding] end end - def detect_encoding - @detect_encoding ||= CharlockHolmes::EncodingDetector.new.detect(data) if data # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - def empty? data.nil? || data == "" end + + private + + def find_encoding + @find_encoding ||= Gitlab::EncodingHelper.detect_encoding(data) if data # rubocop:disable Gitlab/ModuleWithInstanceVariables + end end end diff --git a/lib/gitlab/cache.rb b/lib/gitlab/cache.rb new file mode 100644 index 00000000000..90a0c38ff7b --- /dev/null +++ b/lib/gitlab/cache.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + class << self + # Utility method for performing a fetch but only + # once per request, storing the returned value in + # the request store, if active. + def fetch_once(key, **kwargs) + Gitlab::SafeRequestStore.fetch(key) do + Rails.cache.fetch(key, **kwargs) do + yield + end + end + end + end + end +end diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb index 31661650eff..52c355478c5 100644 --- a/lib/gitlab/changelog/committer.rb +++ b/lib/gitlab/changelog/committer.rb @@ -55,7 +55,7 @@ module Gitlab result = service.execute - raise Error.new(result[:message]) if result[:status] != :success + raise Error, result[:message] if result[:status] != :success end end diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb index a4c8da283cd..fac6fc19148 100644 --- a/lib/gitlab/changelog/parser.rb +++ b/lib/gitlab/changelog/parser.rb @@ -169,7 +169,7 @@ module Gitlab # We raise a custom error so it's easier to catch different changelog # related errors. In addition, this ensures the caller of this method # doesn't depend on a Parslet specific error class. - raise Error.new("Failed to parse the template: #{ex.message}") + raise Error, "Failed to parse the template: #{ex.message}" end end end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb index 6267fbc20e2..53a625e9d43 100644 --- a/lib/gitlab/chat/responder.rb +++ b/lib/gitlab/chat/responder.rb @@ -11,9 +11,9 @@ module Gitlab # # build - A `Ci::Build` that executed a chat command. def self.responder_for(build) - service = build.pipeline.chat_data&.chat_name&.service + integration = build.pipeline.chat_data&.chat_name&.integration - if (responder = service.try(:chat_responder)) + if (responder = integration.try(:chat_responder)) responder.new(build) end end diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb index 0045d8a4113..68873610408 100644 --- a/lib/gitlab/checks/base_checker.rb +++ b/lib/gitlab/checks/base_checker.rb @@ -57,4 +57,4 @@ module Gitlab end end -Gitlab::Checks::BaseChecker.prepend_if_ee('EE::Gitlab::Checks::BaseChecker') +Gitlab::Checks::BaseChecker.prepend_mod_with('Gitlab::Checks::BaseChecker') diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 67c777f67a7..a2c3de3e775 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -54,4 +54,4 @@ module Gitlab end end -Gitlab::Checks::ChangeAccess.prepend_if_ee('EE::Gitlab::Checks::ChangeAccess') +Gitlab::Checks::ChangeAccess.prepend_mod_with('Gitlab::Checks::ChangeAccess') diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index b146fea66b9..a05181ab58e 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -76,4 +76,4 @@ module Gitlab end end -Gitlab::Checks::DiffCheck.prepend_if_ee('EE::Gitlab::Checks::DiffCheck') +Gitlab::Checks::DiffCheck.prepend_mod_with('Gitlab::Checks::DiffCheck') diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index db7af0088d0..2635ad04770 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -21,4 +21,4 @@ module Gitlab end end -Gitlab::Checks::MatchingMergeRequest.prepend_if_ee('EE::Gitlab::Checks::MatchingMergeRequest') +Gitlab::Checks::MatchingMergeRequest.prepend_mod_with('Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 1fac00337a3..97988d8aa13 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -30,6 +30,8 @@ module Gitlab Converter.new.convert(ansi, state) end + Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride + class Converter def on_0(_) reset @@ -278,9 +280,7 @@ module Gitlab close_open_tags - # TODO: replace OpenStruct with a better type - # https://gitlab.com/gitlab-org/gitlab/issues/34305 - OpenStruct.new( + Ansi2html::Result.new( html: @out.force_encoding(Encoding.default_external), state: state, append: append, diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 4fcb5168847..375e6b4a96f 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -7,39 +7,22 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize def initialize(cache, pipeline) - if multiple_cache_per_job? - cache = Array.wrap(cache) - @cache = cache.map do |cache| - Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) - end - else - @cache = Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache) + cache = Array.wrap(cache) + @cache = cache.map do |cache| + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) end end def cache_attributes strong_memoize(:cache_attributes) do - if multiple_cache_per_job? - if @cache.empty? - {} - else - { options: { cache: @cache.map(&:attributes) } } - end + if @cache.empty? + {} else - @cache.build_attributes + { options: { cache: @cache.map(&:attributes) } } end end end - - private - - def multiple_cache_per_job? - strong_memoize(:multiple_cache_per_job) do - ::Gitlab::Ci::Features.multiple_cache_per_job? - end - end end end end diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb index facb5f619bd..9720bb1123a 100644 --- a/lib/gitlab/ci/build/releaser.rb +++ b/lib/gitlab/ci/build/releaser.rb @@ -18,8 +18,9 @@ module Gitlab command = BASE_COMMAND.dup single_flags.each { |k, v| command.concat(" --#{k.to_s.dasherize} \"#{v}\"") } array_commands.each { |k, v| v.each { |elem| command.concat(" --#{k.to_s.singularize.dasherize} \"#{elem}\"") } } + asset_links.each { |link| command.concat(" --assets-link #{stringified_json(link)}") } - [command] + [command.freeze] end private @@ -31,6 +32,14 @@ module Gitlab def array_commands config.slice(*ARRAY_FLAGS) end + + def asset_links + config.dig(:assets, :links) || [] + end + + def stringified_json(object) + "#{object.to_json.to_json}" + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 23b0c93a3ee..9c6428d701c 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,13 +17,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :ref + attr_reader :root, :context, :ref, :source - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil) @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) @ref = ref + @source = source @config = expand_config(config) @@ -128,4 +129,4 @@ module Gitlab end end -Gitlab::Ci::Config.prepend_if_ee('EE::Gitlab::Ci::ConfigEE') +Gitlab::Ci::Config.prepend_mod_with('Gitlab::Ci::ConfigEE') diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index f9688c500d2..ab79add688b 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -4,88 +4,52 @@ module Gitlab module Ci class Config module Entry - ## - # Entry that represents a cache configuration - # - class Cache < ::Gitlab::Config::Entry::Simplifiable - strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) } - strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) } - - class Caches < ::Gitlab::Config::Entry::ComposableArray - include ::Gitlab::Config::Entry::Validatable - - MULTIPLE_CACHE_LIMIT = 4 - - validations do - validate do - unless config.is_a?(Hash) || config.is_a?(Array) - errors.add(:config, 'can only be a Hash or an Array') - end - - if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT - errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") - end - end - end - - def initialize(*args) - super - - @key = nil - end - - def composable_class - Entry::Cache::Cache + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze + DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } end end - class Cache < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[key untracked paths when policy].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze - DEFAULT_POLICY = 'pull-push' - ALLOWED_WHEN = %w[on_success on_failure always].freeze - DEFAULT_WHEN = 'on_success' + entry :key, Entry::Key, + description: 'Cache key used to define a cache affinity.' - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true - - with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } - end - end + entry :untracked, ::Gitlab::Config::Entry::Boolean, + description: 'Cache all untracked files.' - entry :key, Entry::Key, - description: 'Cache key used to define a cache affinity.' + entry :paths, Entry::Paths, + description: 'Specify which paths should be cached across builds.' - entry :untracked, ::Gitlab::Config::Entry::Boolean, - description: 'Cache all untracked files.' + attributes :policy, :when - entry :paths, Entry::Paths, - description: 'Specify which paths should be cached across builds.' + def value + result = super - attributes :policy, :when + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN - def value - result = super - - result[:key] = key_value - result[:policy] = policy || DEFAULT_POLICY - # Use self.when to avoid conflict with reserved word - result[:when] = self.when || DEFAULT_WHEN - - result - end + result end class UnknownStrategy < ::Gitlab::Config::Entry::Node diff --git a/lib/gitlab/ci/config/entry/caches.rb b/lib/gitlab/ci/config/entry/caches.rb new file mode 100644 index 00000000000..75240599c9c --- /dev/null +++ b/lib/gitlab/ci/config/entry/caches.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents caches configuration + # + class Caches < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + MULTIPLE_CACHE_LIMIT = 4 + + validations do + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + + if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT + errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") + end + end + end + + def initialize(*args) + super + + @key = nil + end + + def composable_class + Entry::Cache + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index ab493ff7d78..eaaf9f69102 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -37,7 +37,7 @@ module Gitlab description: 'Script that will be executed after each job.', inherit: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', inherit: true diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a20b802be58..c8e8f0bc1fc 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -64,7 +64,7 @@ module Gitlab description: 'Commands that will be executed when finishing job.', inherit: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Cache definition for this job.', inherit: true @@ -200,4 +200,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Job.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Job') +::Gitlab::Ci::Config::Entry::Job.prepend_mod_with('Gitlab::Ci::Config::Entry::Job') diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index b3cf0f9e0fd..29dc48c7b42 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -118,4 +118,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Need.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Need') +::Gitlab::Ci::Config::Entry::Need.prepend_mod_with('Gitlab::Ci::Config::Entry::Need') diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index dd01cfeedff..11b202ddde9 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -56,4 +56,4 @@ module Gitlab end end -::Gitlab::Ci::Config::Entry::Needs.prepend_if_ee('::EE::Gitlab::Ci::Config::Entry::Needs') +::Gitlab::Ci::Config::Entry::Needs.prepend_mod_with('Gitlab::Ci::Config::Entry::Needs') diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 54ef84b965a..e6290ef2479 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -61,7 +61,7 @@ module Gitlab description: 'Deprecated: stages for this pipeline.', reserved: true - entry :cache, Entry::Cache, + entry :cache, Entry::Caches, description: 'Configure caching between build jobs.', reserved: true diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 12e182b38fc..c8e4d9ed763 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -18,9 +18,8 @@ module Gitlab Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true) end - # Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/224199 - def self.store_pipeline_messages?(project) - ::Feature.enabled?(:ci_store_pipeline_messages, project, default_enabled: true) + def self.merge_base_pipeline_for_metrics_comparison?(project) + Feature.enabled?(:merge_base_pipeline_for_metrics_comparison, project, default_enabled: :yaml) end def self.raise_job_rules_without_workflow_rules_warning? @@ -47,22 +46,17 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.validate_build_dependencies?(project) - ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && - ::Feature.disabled?(:ci_validate_build_dependencies_override, project) - end - def self.display_quality_on_mr_diff?(project) - ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) - end - - def self.multiple_cache_per_job? - ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: :yaml) end def self.gldropdown_tags_enabled? ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end + + def self.background_pipeline_retry_endpoint?(project) + ::Feature.enabled?(:background_pipeline_retry_endpoint, project) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index a6ae249fa58..0b94debb24e 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -123,4 +123,4 @@ module Gitlab end end -Gitlab::Ci::Jwt.prepend_if_ee('::EE::Gitlab::Ci::Jwt') +Gitlab::Ci::Jwt.prepend_mod_with('Gitlab::Ci::Jwt') diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 2baa8faf849..3469537a2e2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -15,8 +15,8 @@ module Gitlab } end - def self.fabricate!(file_type, *args) - parsers.fetch(file_type.to_sym).new(*args) + def self.fabricate!(file_type, *args, **kwargs) + parsers.fetch(file_type.to_sym).new(*args, **kwargs) rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end @@ -28,4 +28,4 @@ module Gitlab end end -Gitlab::Ci::Parsers.prepend_if_ee('::EE::Gitlab::Ci::Parsers') +Gitlab::Ci::Parsers.prepend_mod_with('Gitlab::Ci::Parsers') diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index eb3adf713d4..d6b3af674a6 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -121,7 +121,7 @@ module Gitlab # Using `Integer()` here to raise exception on invalid values [Integer(line["number"]), Integer(line["hits"])] end - rescue + rescue StandardError raise InvalidLineInformationError, "Line information had invalid values" end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index abfbe18e23f..f9afa58f915 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -19,7 +19,7 @@ module Gitlab end rescue JSON::ParserError terraform_reports.add_plan(job_id, invalid_tfplan(:invalid_json_format, job_details)) - rescue + rescue StandardError details = job_details || {} plan_name = job_id || 'failed_tf_plan' terraform_reports.add_plan(plan_name, invalid_tfplan(:unknown_error, details)) diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 50cd703da4a..ca7fbde6713 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -31,7 +31,7 @@ module Gitlab def ensure_test_cases_limited!(total_parsed, limit) return unless limit > 0 && total_parsed > limit - raise JunitParserError.new("number of test cases exceeded the limit of #{limit}") + raise JunitParserError, "number of test cases exceeded the limit of #{limit}" end def all_cases(root, parent = nil, &blk) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index a7680f6e593..3c150ca26bb 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -52,4 +52,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Config::Content') +Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 8f1c49563f2..49ec1250a5f 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -16,6 +16,7 @@ module Gitlab project: project, ref: @pipeline.ref, sha: @pipeline.sha, + source: @pipeline.source, user: current_user, parent_pipeline: parent_pipeline } @@ -31,7 +32,7 @@ module Gitlab @pipeline.merged_yaml = result.merged_yaml - rescue => ex + rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, sha: @pipeline.sha diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 9988b6f18ed..09158bf8bfd 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -19,6 +19,8 @@ module Gitlab # polluted with other unrelated errors (e.g. state machine) # https://gitlab.com/gitlab-org/gitlab/-/issues/220823 pipeline.errors.add(:base, message) + + pipeline.errors.full_messages end def warning(message) diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb index 3c64278e305..ef9235477db 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/activity.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::Activity') +Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Activity') diff --git a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb index 2e8b437252f..3706dd0b9f6 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/job_activity.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') +Gitlab::Ci::Pipeline::Chain::Limit::JobActivity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::JobActivity') diff --git a/lib/gitlab/ci/pipeline/chain/limit/size.rb b/lib/gitlab/ci/pipeline/chain/limit/size.rb index 739648840e9..761bdb1c484 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/size.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/size.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Limit::Size.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Limit::Size') +Gitlab::Ci::Pipeline::Chain::Limit::Size.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Size') diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index df92e229f12..e4e4f4f484a 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -11,7 +11,14 @@ module Gitlab def perform! if skipped? - @pipeline.skip if @command.save_incompleted + if @command.save_incompleted + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the state transition transaction of the skip call + # which it will lock the InternalId row for the whole transaction + @pipeline.ensure_project_iid! + + @pipeline.skip + end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 55c125e03d5..1c1f7abb6f6 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -57,4 +57,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Validate::Abilities.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::Abilities') +Gitlab::Ci::Pipeline::Chain::Validate::Abilities.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::Abilities') diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 6149d2f04d7..539b44513f0 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -54,7 +54,7 @@ module Gitlab else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" end - rescue => ex + rescue StandardError => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) true @@ -147,4 +147,4 @@ module Gitlab end end -Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External') +Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::External') diff --git a/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb b/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb new file mode 100644 index 00000000000..e3588aa3027 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/security_orchestration_policy.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class SecurityOrchestrationPolicy < Chain::Base + include Chain::Helpers + + def perform! + # no-op + end + + def break? + false + end + end + end + end + end + end +end + +Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 6cb6fd3920d..84b88374a7f 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -13,6 +13,13 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_security_orchestration_policy_processing_duration_histogram + name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds + comment = 'Pipeline security orchestration policy processing duration' + + ::Gitlab::Metrics.histogram(name, comment) + end + def self.pipeline_size_histogram name = :gitlab_ci_pipeline_size_builds comment = 'Pipeline size' @@ -56,6 +63,21 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end + + def ci_minutes_exceeded_builds_counter + name = :ci_minutes_exceeded_builds_counter + comment = 'Count of builds dropped due to CI minutes exceeded' + + Gitlab::Metrics.counter(name, comment) + end + + def self.gitlab_ci_difference_live_vs_actual_minutes + name = :gitlab_ci_difference_live_vs_actual_minutes + comment = 'Comparison between CI minutes consumption from live tracking vs actual consumption' + labels = {} + buckets = [-120.0, -60.0, -30.0, -10.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0] + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end end end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7ecb9a1db16..46e4373ec85 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -10,7 +10,7 @@ module Gitlab QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze - QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 15, 20, 30, 60].freeze METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb index e60a075e3f5..0595b6f966a 100644 --- a/lib/gitlab/ci/reports/codequality_mr_diff.rb +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -6,8 +6,8 @@ module Gitlab class CodequalityMrDiff attr_reader :files - def initialize(raw_report) - @raw_report = raw_report + def initialize(new_errors) + @new_errors = new_errors @files = {} build_report! end @@ -15,7 +15,7 @@ module Gitlab private def build_report! - codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + codequality_files = @new_errors.each_with_object({}) do |degradation, codequality_files| unless codequality_files[degradation.dig(:location, :path)].present? codequality_files[degradation.dig(:location, :path)] = [] end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index 37d0da38065..c110dbf98be 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -13,7 +13,7 @@ module Gitlab def load! recent_failures_count.each do |key_hash, count| - failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master) + failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_main) end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 787dee3b267..cbd72f54ff4 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -20,6 +20,7 @@ module Gitlab scheduler_failure: 'scheduler failure', data_integrity_failure: 'data integrity failure', forward_deployment_failure: 'forward deployment failure', + pipeline_loop_detected: 'job would create infinitely looping pipelines', invalid_bridge_trigger: 'downstream pipeline trigger definition is invalid', downstream_bridge_project_not_found: 'downstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', @@ -28,7 +29,8 @@ module Gitlab secrets_provider_not_found: 'secrets provider can not be found', reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', project_deleted: 'pipeline project was deleted', - user_blocked: 'pipeline user was blocked' + user_blocked: 'pipeline user was blocked', + ci_quota_exceeded: 'no more CI minutes available' }.freeze private_constant :REASONS @@ -68,4 +70,4 @@ module Gitlab end end -Gitlab::Ci::Status::Build::Failed.prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') +Gitlab::Ci::Status::Build::Failed.prepend_mod_with('Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 4779c8d3d53..e7ed2081f6a 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -11,6 +11,8 @@ module Gitlab attr_reader :subject, :user + delegate :cache_key, to: :subject + def initialize(subject, user) @subject = subject @user = user diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml deleted file mode 100644 index 7182b96594d..00000000000 --- a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# You can use artifacts to pass data to jobs in later stages. -# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job might build an important file, and pass it to later jobs." - - echo "This is the content of the important file" > important-file.txt - artifacts: - paths: - - important-file.txt - -test-job-with-artifacts: - stage: test - script: - - echo "This job uses the artifact from the job in the earlier stage." - - cat important-file.txt - - echo "It creates another file, and adds it to the artifacts." - - echo "This is a second important file" > important-file2.txt - artifacts: - paths: - - important-file2.txt - -test-job-with-no-artifacts: - stage: test - dependencies: [] # Use to skip downloading any artifacts - script: - - echo "This job does not get the artifacts from other jobs." - - cat important-file.txt || exit 0 - -deploy-job-with-all-artifacts: - stage: deploy - script: - - echo "By default, jobs download all available artifacts." - - cat important-file.txt - - cat important-file2.txt - -deploy-job-with-1-artifact: - stage: deploy - dependencies: - - build-job # Download artifacts from only this job - script: - - echo "You can configure a job to download artifacts from only certain jobs." - - cat important-file.txt - - cat important-file2.txt || exit 0 diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml deleted file mode 100644 index 382bac09ed7..00000000000 --- a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# You can define common tasks and run them before or after the main scripts in jobs. -# For more information, see: -# - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script -# - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script -# - -stages: - - test - -default: - before_script: - - echo "This script runs before the main script in every job, unless the job overrides it." - - echo "It may set up common dependencies, for example." - after_script: - - echo "This script runs after the main script in every job, unless the job overrides it." - - echo "It may do some common final clean up tasks" - -job-standard: - stage: test - script: - - echo "This job uses both of the globally defined before and after scripts." - -job-override-before: - stage: test - before_script: - - echo "Use a different before_script in this job." - script: - - echo "This job uses its own before_script, and the global after_script." - -job-override-after: - stage: test - after_script: - - echo "Use a different after_script in this job." - script: - - echo "This job uses its own after_script, and the global before_script." diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml deleted file mode 100644 index 5f27def74c9..00000000000 --- a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml +++ /dev/null @@ -1,53 +0,0 @@ -# -# A manual job is a type of job that is not executed automatically and must be explicitly started by a user. -# To make a job manual, add when: manual to its configuration. -# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job is not a manual job" - -manual-build: - stage: build - script: - - echo "This manual job passes after you trigger it." - when: manual - -manual-build-allowed-to-fail: - stage: build - script: - - echo "This manual job fails after you trigger it." - - echo "It is allowed to fail, so the pipeline does not fail. - when: manual - allow_failure: true # Default behavior - -test-job: - stage: test - script: - - echo "This is a normal test job" - - echo "It runs when the when the build stage completes." - - echo "It does not need to wait for the manual jobs in the build stage to run." - -manual-test-not-allowed-to-fail: - stage: test - script: - - echo "This manual job fails after you trigger it." - - echo "It is NOT allowed to fail, so the pipeline is marked as failed - - echo "when this job completes." - - exit 1 - when: manual - allow_failure: false # Optional behavior - -deploy-job: - stage: deploy - script: - - echo "This is a normal deploy job" - - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed, - - echo "this job does not run". diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml deleted file mode 100644 index aced628aacb..00000000000 --- a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -# -# A pipeline is composed of independent jobs that run scripts, grouped into stages. -# Stages run in sequential order, but jobs within stages run in parallel. -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages -# - -stages: - - build - - test - - deploy - -build-job: - stage: build - script: - - echo "This job runs in the build stage, which runs first." - -test-job1: - stage: test - script: - - echo "This job runs in the test stage." - - echo "It only starts when the job in the build stage completes successfully." - -test-job2: - stage: test - script: - - echo "This job also runs in the test stage." - - echo "This job can run at the same time as test-job2." - -deploy-job: - stage: deploy - script: - - echo "This job runs in the deploy stage." - - echo "It only runs when both jobs in the test stage complete successfully" diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml deleted file mode 100644 index 2b8cf7bab44..00000000000 --- a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Variables can be used to for more dynamic behavior in jobs and scripts. -# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html -# - -stages: - - test - -variables: - VAR1: "Variable 1 defined globally" - -use-a-variable: - stage: test - script: - - echo "You can use variables in jobs." - - echo "The content of 'VAR1' is = $VAR1" - -override-a-variable: - stage: test - variables: - VAR1: "Variable 1 was overriden in in the job." - script: - - echo "You can override global variables in jobs." - - echo "The content of 'VAR1' is = $VAR1" - -define-a-new-variable: - stage: test - variables: - VAR2: "Variable 2 is new and defined in the job only." - script: - - echo "You can mix global variables with variables defined in jobs." - - echo "The content of 'VAR1' is = $VAR1" - - echo "The content of 'VAR2' is = $VAR2" - -incorrect-variable-usage: - stage: test - script: - - echo "You can't use variables only defined in other jobs." - - echo "The content of 'VAR2' is = $VAR2" - -predefined-variables: - stage: test - script: - - echo "Some variables are predefined by GitLab CI/CD, for example:" - - echo "The commit author's username is $GITLAB_USER_LOGIN" - - echo "The commit branch is $CI_COMMIT_BRANCH" - - echo "The project path is $CI_PROJECT_PATH" diff --git a/lib/gitlab/ci/templates/Getting-started.yml b/lib/gitlab/ci/templates/Getting-started.yml new file mode 100644 index 00000000000..4dc88418671 --- /dev/null +++ b/lib/gitlab/ci/templates/Getting-started.yml @@ -0,0 +1,39 @@ +# This is a sample GitLab CI/CD configuration file that should run without any modifications. +# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, +# it uses echo commands to simulate the pipeline execution. +# +# A pipeline is composed of independent jobs that run scripts, grouped into stages. +# Stages run in sequential order, but jobs within stages run in parallel. +# +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages + +stages: # List of stages for jobs, and their order of execution + - build + - test + - deploy + +build-job: # This job runs in the build stage, which runs first. + stage: build + script: + - echo "Compiling the code..." + - echo "Compile complete. + +unit-test-job: # This job runs in the test stage. + stage: test # It only starts when the job in the build stage completes successfully. + script: + - echo "Running unit tests... This will take about 60 seconds." + - sleep 60 + - echo "Code coverage is 90%" + +lint-test-job: # This job also runs in the test stage. + stage: test # It can run at the same time as unit-test-job (in parallel). + script: + - echo "Linting code... This will take about 10 seconds." + - sleep 10 + - echo "No lint issues found." + +deploy-job: # This job runs in the deploy stage. + stage: deploy # It only runs when *both* jobs in the test stage complete successfully. + script: + - echo "Deploying application..." + - echo "Application successfully deployed." diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml index c7fb1321055..7f33d048c1e 100644 --- a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci.yml @@ -29,12 +29,8 @@ default: before_script: - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content -stages: - - init_and_plan - - cloudrail - init_and_plan: - stage: init_and_plan + stage: build image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 rules: - if: $SAST_DISABLED @@ -52,7 +48,7 @@ init_and_plan: - ./**/.terraform cloudrail_scan: - stage: cloudrail + stage: test image: indeni/cloudrail-cli:1.2.44 rules: - if: $SAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..5216a46745c --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -0,0 +1,77 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +browser_performance: + stage: performance + image: docker:19.03.12 + allow_failure: true + variables: + DOCKER_TLS_CERTDIR: "" + SITESPEED_IMAGE: sitespeedio/sitespeed.io + SITESPEED_VERSION: 14.1.0 + SITESPEED_OPTIONS: '' + services: + - docker:19.03.12-dind + script: + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js + - mkdir sitespeed-results + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + else + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + fi + - mv sitespeed-results/data/performance.json browser-performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + browser_performance: browser-performance.json + rules: + - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' + when: never + - if: '$PERFORMANCE_DISABLED' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..abcb347b146 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,11 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - name: "docker:20.10.6-dind" + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then @@ -16,6 +17,8 @@ build: fi - /build/build.sh rules: + - if: '$BUILD_DISABLED' + when: never - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -26,4 +29,6 @@ build_artifact: - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" - exit 1 rules: + - if: '$BUILD_DISABLED' + when: never - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 3faf07546de..45bddb1bc6a 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -46,27 +46,23 @@ review: name: review/$CI_COMMIT_REF_NAME url: http://$CI_PROJECT_NAME-$CI_ENVIRONMENT_SLUG.$OPENSHIFT_DOMAIN on_stop: stop-review - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH stop-review: <<: *deploy stage: cleanup script: - oc delete all -l "app=$APP" - when: manual variables: APP: review-$CI_COMMIT_REF_NAME GIT_STRATEGY: none environment: name: review/$CI_COMMIT_REF_NAME action: stop - only: - - branches - except: - - master + rules: + - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + when: manual staging: <<: *deploy @@ -77,8 +73,8 @@ staging: environment: name: staging url: http://$CI_PROJECT_NAME-staging.$OPENSHIFT_DOMAIN - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH production: <<: *deploy @@ -86,9 +82,9 @@ production: variables: APP: production APP_HOST: $CI_PROJECT_NAME.$OPENSHIFT_DOMAIN - when: manual environment: name: production url: http://$CI_PROJECT_NAME.$OPENSHIFT_DOMAIN - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index bf42cd52605..90fad1550ff 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -11,6 +11,7 @@ stages: - fuzz variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" FUZZAPI_PROFILE: Quick FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml @@ -24,7 +25,7 @@ variables: # available (non 500 response to HTTP(s)) FUZZAPI_SERVICE_START_TIMEOUT: "300" # - FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} # apifuzzer_fuzz_unlicensed: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 215029dc952..8fa33026011 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -5,266 +5,30 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest - FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml - FUZZAPI_TIMEOUT: 30 - FUZZAPI_REPORT: gl-api-fuzzing-report.json - FUZZAPI_REPORT_ASSET_PATH: assets - # - FUZZAPI_D_NETWORK: testing-net - # - # Wait up to 5 minutes for API Fuzzer and target url to become - # available (non 500 response to HTTP(s)) - FUZZAPI_SERVICE_START_TIMEOUT: "300" - # - FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine - # - -apifuzzer_fuzz_unlicensed: - stage: fuzz - allow_failure: true - rules: - - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' - - when: never - script: - - | - echo "Error: Your GitLab project is not licensed for API Fuzzing." - - exit 1 + FUZZAPI_VERSION: "1" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} apifuzzer_fuzz: stage: fuzz - image: - name: $FUZZAPI_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 - FUZZAPI_NEW_REPORT: 1 - FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log - TZ: America/Los_Angeles - allow_failure: true - rules: - - if: $FUZZAPI_D_TARGET_IMAGE - when: never - - if: $FUZZAPI_D_WORKER_IMAGE - when: never - - if: $API_FUZZING_DISABLED - when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - script: - # - # Validate options - - | - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT - - $FUZZAPI_LOG_SCANNER - reports: - api_fuzzing: $FUZZAPI_REPORT - -apifuzzer_fuzz_dnd: - stage: fuzz - image: docker:19.03.12 - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + image: $FUZZAPI_IMAGE allow_failure: true rules: - - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null - when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ - services: - - docker:19.03.12-dind + - if: $CI_COMMIT_BRANCH script: - # - # - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # - - docker network create --driver bridge $FUZZAPI_D_NETWORK - # - # Run user provided pre-script - - sh -c "$FUZZAPI_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $FUZZAPI_REPORT_ASSET_PATH - # - # Start peach testing engine container - - | - docker run -d \ - --name apifuzzer \ - --network $FUZZAPI_D_NETWORK \ - -e Proxy:Port=8000 \ - -e TZ=America/Los_Angeles \ - -e GITLAB_FEATURES \ - -p 80:80 \ - -p 8000:8000 \ - -p 514:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - dotnet /peach/Peach.Web.dll - # - # Start target container - - | - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ - docker run -d \ - --name target \ - --network $FUZZAPI_D_NETWORK \ - $FUZZAPI_D_TARGET_ENV \ - $FUZZAPI_D_TARGET_PORTS \ - $FUZZAPI_D_TARGET_VOLUME \ - --restart=no \ - $FUZZAPI_D_TARGET_IMAGE \ - ; fi - # - # Start worker container if provided - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ - $FUZZAPI_D_WORKER_ENV \ - $FUZZAPI_D_WORKER_PORTS \ - $FUZZAPI_D_WORKER_VOLUME \ - --restart=no \ - $FUZZAPI_D_WORKER_IMAGE \ - ; fi - # - # Start API Fuzzing provided worker if no other worker present - - | - if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ - echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ - echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ - exit 1; \ - fi; \ - docker run \ - --name worker \ - --network $FUZZAPI_D_NETWORK \ - -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_REPORT_ASSET_PATH \ - -e FUZZAPI_NEW_REPORT=1 \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_POSTMAN_COLLECTION \ - -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ - -e CI_PROJECT_URL \ - -e CI_JOB_ID \ - -v $CI_PROJECT_DIR:/app \ - -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ - -p 81:80 \ - -p 8001:8000 \ - -p 515:514 \ - --restart=no \ - $FUZZAPI_IMAGE \ - worker-entry \ - ; fi - # - # Propagate exit code from api fuzzing scanner (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi - # - # Run user provided post-script - - sh -c "$FUZZAPI_POST_SCRIPT" - # - after_script: - # - # Shutdown all containers - - echo "Stopping all containers" - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - docker stop worker - - docker stop apifuzzer - # - # Save docker logs - - docker logs apifuzzer &> gl-api_fuzzing-logs.log - - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - docker logs worker &> gl-api_fuzzing-worker-logs.log - # + - /peach/analyzer-fuzz-api artifacts: when: always paths: - - ./gl-api_fuzzing*.log - - ./gl-api_fuzzing*.zip - - $FUZZAPI_REPORT_ASSET_PATH - - $FUZZAPI_REPORT + - gl-assets + - gl-api-fuzzing-report.json + - gl-*.log reports: - api_fuzzing: $FUZZAPI_REPORT + api_fuzzing: gl-api-fuzzing-report.json # end diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml new file mode 100644 index 00000000000..b40c4e982f7 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -0,0 +1,48 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST-API.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API: +# +# stages: +# - build +# - test +# - deploy +# - dast + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html + +# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) +# List of variables available to configure the DAST API scanning tool: +# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_VERSION: "1" + DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + +dast_api: + stage: dast + image: $DAST_API_IMAGE + allow_failure: true + rules: + - if: $DAST_API_DISABLED + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH + script: + - /peach/analyzer-dast-api + artifacts: + when: always + paths: + - gl-assets + - gl-dast-api-report.json + - gl-*.log + reports: + dast: gl-dast-api-report.json diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 533f8bb25f8..b6282da18a4 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -22,19 +22,6 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - # - DAST_API_PROFILE: Full - DAST_API_VERSION: latest - DAST_API_CONFIG: .gitlab-dast-api.yml - DAST_API_TIMEOUT: 30 - DAST_API_REPORT: gl-dast-api-report.json - DAST_API_REPORT_ASSET_PATH: assets - # - # Wait up to 5 minutes for API Security and target url to become - # available (non 500 response to HTTP(s)) - DAST_API_SERVICE_START_TIMEOUT: "300" - # - DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -51,11 +38,6 @@ dast: reports: dast: gl-dast-report.json rules: - - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || - $DAST_API_OPENAPI || - $DAST_API_POSTMAN_COLLECTION || - $DAST_API_HAR ) - when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -71,72 +53,4 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && - $DAST_API_BETA == null && $DAST_API_SPECIFICATION - -dast_api: - stage: dast - image: - name: $DAST_API_IMAGE - entrypoint: ["/bin/bash", "-l", "-c"] - variables: - API_SECURITY_MODE: DAST - DAST_API_NEW_REPORT: 1 - DAST_API_PROJECT: $CI_PROJECT_PATH - DAST_API_API: http://127.0.0.1:5000 - DAST_API_LOG_SCANNER: gl-dast-api-scanner.log - TZ: America/Los_Angeles - allow_failure: true - rules: - - if: $DAST_API_BETA == null - when: never - - if: $DAST_DISABLED - when: never - - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED && - $DAST_API_SPECIFICATION == null && - $DAST_API_OPENAPI == null && - $DAST_API_POSTMAN_COLLECTION == null && - $DAST_API_HAR == null - when: never - - if: $DAST_API_SPECIFICATION == null && - $DAST_API_OPENAPI == null && - $DAST_API_POSTMAN_COLLECTION == null && - $DAST_API_HAR == null - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdast\b/ - script: - # - # Run user provided pre-script - - sh -c "$DAST_API_PRE_SCRIPT" - # - # Make sure asset path exists - - mkdir -p $DAST_API_REPORT_ASSET_PATH - # - # Start API Security background process - - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & - - APISEC_PID=$! - # - # Start scanning - - worker-entry - # - # Run user provided post-script - - sh -c "$DAST_API_POST_SCRIPT" - # - # Shutdown API Security - - kill $APISEC_PID - - wait $APISEC_PID - # - artifacts: - when: always - paths: - - $DAST_API_REPORT_ASSET_PATH - - $DAST_API_REPORT - - $DAST_API_LOG_SCANNER - - gl-*.log - reports: - dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3039d64514b..53d68c24d26 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -8,8 +8,8 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" + DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 @@ -45,6 +45,8 @@ gemnasium-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ @@ -71,6 +73,8 @@ gemnasium-maven-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ @@ -92,6 +96,8 @@ gemnasium-python-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ @@ -120,6 +126,8 @@ bundler-audit-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ @@ -138,6 +146,8 @@ retire-js-dependency_scanning: rules: - if: $DEPENDENCY_SCANNING_DISABLED when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /retire.js/ + when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && $DS_DEFAULT_ANALYZERS =~ /retire.js/ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 3ebccfbba4a..a8d45e80356 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -155,13 +155,8 @@ gosec-sast: exists: - '**/*.go' -mobsf-android-sast: +.mobsf-sast: extends: .sast-analyzer - services: - # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile - # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 - alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: @@ -169,7 +164,9 @@ mobsf-android-sast: # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - MOBSF_API_KEY: key + +mobsf-android-sast: + extends: .mobsf-sast rules: - if: $SAST_DISABLED when: never @@ -179,23 +176,11 @@ mobsf-android-sast: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: + - '**/*.apk' - '**/AndroidManifest.xml' mobsf-ios-sast: - extends: .sast-analyzer - services: - # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile - # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 - alias: mobsf - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to - # override the analyzer image with a custom value. This may be subject to change or - # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" - MOBSF_API_KEY: key + extends: .mobsf-sast rules: - if: $SAST_DISABLED when: never @@ -205,6 +190,7 @@ mobsf-ios-sast: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: + - '**/*.ipa' - '**/*.xcodeproj/*' nodejs-scan-sast: @@ -292,15 +278,14 @@ semgrep-sast: # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never - if: $CI_COMMIT_BRANCH && - $SAST_DEFAULT_ANALYZERS =~ /semgrep/ && - $SAST_EXPERIMENTAL_FEATURES == 'true' + $SAST_DEFAULT_ANALYZERS =~ /semgrep/ exists: - '**/*.py' - '**/*.js' diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 232c320562b..ac975fbbeab 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -13,11 +13,11 @@ variables: SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, klar, clair-vulnerabilities-db, license-finder, - dast + dast, api-fuzzing SECURE_BINARIES_DOWNLOAD_IMAGES: "true" SECURE_BINARIES_PUSH_IMAGES: "true" @@ -134,6 +134,13 @@ secrets: variables: SECURE_BINARIES_ANALYZER_VERSION: "3" +semgrep: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsemgrep\b/ + sobelow: extends: .download_images only: @@ -241,3 +248,12 @@ dast: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ + +api-fuzzing: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bapi-fuzzing\b/ diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 7e2828d010f..6b9db1c2e0f 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -56,6 +56,6 @@ apply: - terraform apply -input=false $PLAN dependencies: - plan - when: manual - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f0621165f8a --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -0,0 +1,52 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +stages: + - build + - test + - deploy + - performance + +browser_performance: + stage: performance + image: docker:git + variables: + URL: '' + SITESPEED_IMAGE: sitespeedio/sitespeed.io + SITESPEED_VERSION: 14.1.0 + SITESPEED_OPTIONS: '' + services: + - docker:stable-dind + script: + - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js + - mkdir sitespeed-results + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - mv sitespeed-results/data/performance.json browser-performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + browser_performance: browser-performance.json diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index c25c4339c35..c4757edf74e 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -317,4 +317,4 @@ module Gitlab end end -::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace') +::Gitlab::Ci::Trace.prepend_mod_with('Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 618438c8887..fdc598c025a 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -93,7 +93,7 @@ module Gitlab end nil - rescue + rescue StandardError # if bad regex or something goes wrong we dont want to interrupt transition # so we just silently ignore error for now end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index dc4951f76bb..a8c1002f2b9 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -141,7 +141,7 @@ module Gitlab end def error!(message) - raise ValidationError.new(message) + raise ValidationError, message end end end diff --git a/lib/gitlab/class_attributes.rb b/lib/gitlab/class_attributes.rb index 6560c97b2e6..6eea7590cbd 100644 --- a/lib/gitlab/class_attributes.rb +++ b/lib/gitlab/class_attributes.rb @@ -14,6 +14,18 @@ module Gitlab class_attributes[name] || superclass_attributes(name) end + def set_class_attribute(name, value) + class_attributes[name] = value + + after_hooks.each(&:call) + + value + end + + def after_set_class_attribute(&block) + after_hooks << block + end + private def class_attributes @@ -25,6 +37,10 @@ module Gitlab superclass.get_class_attribute(name) end + + def after_hooks + @after_hooks ||= [] + end end end end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 48a1ab23fc2..05dfdcd4486 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -139,4 +139,4 @@ module Gitlab end end -Gitlab::Cleanup::OrphanJobArtifactFiles.prepend_if_ee('EE::Gitlab::Cleanup::OrphanJobArtifactFiles') +Gitlab::Cleanup::OrphanJobArtifactFiles.prepend_mod_with('Gitlab::Cleanup::OrphanJobArtifactFiles') diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb index 4b1d16eb974..e222f2834ee 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files_batch.rb @@ -79,4 +79,4 @@ module Gitlab end end -Gitlab::Cleanup::OrphanJobArtifactFilesBatch.prepend_if_ee('EE::Gitlab::Cleanup::OrphanJobArtifactFilesBatch') +Gitlab::Cleanup::OrphanJobArtifactFilesBatch.prepend_mod_with('Gitlab::Cleanup::OrphanJobArtifactFilesBatch') diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 77231665e7e..ed4b363416c 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -44,7 +44,7 @@ module Gitlab return unless upload && upload.local? && upload.model upload.absolute_path - rescue => e + rescue StandardError => e logger.error e.message # absolute_path depends on a lot of code. If it doesn't work, then it @@ -72,7 +72,7 @@ module Gitlab FileUtils.mv(path, new_path) "Did #{action}" - rescue => e + rescue StandardError => e "Error during #{action}: #{e.inspect}" end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 3c71ca9fcf0..b3dc59466ec 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -154,7 +154,7 @@ module Gitlab hooks.each do |hook| hook.call - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, type: 'LifecycleEvents', hook: hook) warn("ERROR: The hook #{name} failed with exception (#{e.class}) \"#{e.message}\".") diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index c3d90aa78fb..d0560807f45 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -8,6 +8,7 @@ module Gitlab class ConanToken HMAC_KEY = 'gitlab-conan-packages' + CONAN_TOKEN_EXPIRE_TIME = 1.day.freeze attr_reader :access_token_id, :user_id @@ -57,7 +58,7 @@ module Gitlab JSONWebToken::HMACToken.new(self.class.secret).tap do |token| token['access_token'] = access_token_id token['user_id'] = user_id - token.expire_time = token.issued_at + 1.hour + token.expire_time = token.issued_at + CONAN_TOKEN_EXPIRE_TIME end end end diff --git a/lib/gitlab/consul/internal.rb b/lib/gitlab/consul/internal.rb index 3afc24ddab9..1994369dee9 100644 --- a/lib/gitlab/consul/internal.rb +++ b/lib/gitlab/consul/internal.rb @@ -57,7 +57,7 @@ module Gitlab def parse_response_body(body) Gitlab::Json.parse(body) - rescue + rescue StandardError raise Consul::Internal::UnexpectedResponseError end @@ -69,7 +69,7 @@ module Gitlab raise Consul::Internal::SSLError rescue Errno::ECONNREFUSED raise Consul::Internal::ECONNREFUSED - rescue + rescue StandardError raise Consul::Internal::UnexpectedResponseError end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index ff844645b11..6f6147f0f32 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -8,11 +8,33 @@ module Gitlab media_src object_src report_uri script_src style_src worker_src).freeze def self.default_settings_hash - { - 'enabled' => false, + settings_hash = { + 'enabled' => true, 'report_only' => false, - 'directives' => DIRECTIVES.each_with_object({}) { |directive, hash| hash[directive] = nil } + 'directives' => { + 'default_src' => "'self'", + 'base_uri' => "'self'", + 'child_src' => "'none'", + 'connect_src' => "'self'", + 'font_src' => "'self'", + 'form_action' => "'self' https: http:", + 'frame_ancestors' => "'self'", + 'frame_src' => "'self' https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com", + 'img_src' => "'self' data: blob: http: https:", + 'manifest_src' => "'self'", + 'media_src' => "'self'", + 'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.recaptcha.net https://apis.google.com", + 'style_src' => "'self' 'unsafe-inline'", + 'worker_src' => "'self'", + 'object_src' => "'none'", + 'report_uri' => nil + } } + + allow_webpack_dev_server(settings_hash) if Rails.env.development? + allow_cdn(settings_hash) if ENV['GITLAB_CDN_HOST'].present? + + settings_hash end def initialize(csp_directives) @@ -38,6 +60,26 @@ module Gitlab arguments.strip.split(' ').map(&:strip) end + + def self.allow_webpack_dev_server(settings_hash) + secure = Settings.webpack.dev_server['https'] + host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}" + http_url = "#{secure ? 'https' : 'http'}://#{host_and_port}" + ws_url = "#{secure ? 'wss' : 'ws'}://#{host_and_port}" + + append_to_directive(settings_hash, 'connect_src', "#{http_url} #{ws_url}") + end + + def self.allow_cdn(settings_hash) + cdn_host = ENV['GITLAB_CDN_HOST'] + + append_to_directive(settings_hash, 'script_src', cdn_host) + append_to_directive(settings_hash, 'style_src', cdn_host) + end + + def self.append_to_directive(settings_hash, directive, text) + settings_hash['directives'][directive] = "#{settings_hash['directives'][directive]} #{text}".strip + end end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 55f381fcb64..7f55734f796 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -43,7 +43,7 @@ module Gitlab begin ::ApplicationSetting.cached - rescue + rescue StandardError # In case Redis isn't running # or the Redis UNIX socket file is not available # or the DB is not running (we use migrations in the cache key) diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index a825d48fb77..67ad75652b0 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -11,11 +11,11 @@ module Gitlab end def title - raise NotImplementedError.new("Expected #{self.name} to implement title") + raise NotImplementedError, "Expected #{self.name} to implement title" end def value - raise NotImplementedError.new("Expected #{self.name} to implement value") + raise NotImplementedError, "Expected #{self.name} to implement value" end end end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index c247ef0d2a7..e5bf6ef616f 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -16,7 +16,7 @@ module Gitlab def deployments_count DeploymentsFinder - .new(project: @project, finished_after: @from, finished_before: @to, status: :success) + .new(project: @project, finished_after: @from, finished_before: @to, status: :success, order_by: :finished_at) .execute .count end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 0e4fc8efa95..4c31f986be5 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -30,6 +30,7 @@ module Gitlab build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, + build_queued_duration: build.queued_duration, build_allow_failure: build.allow_failure, build_failure_reason: build.failure_reason, pipeline_id: commit.id, diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 87ebe832862..f50ca5119b7 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -5,7 +5,7 @@ module Gitlab module Deployment extend self - def build(deployment) + def build(deployment, status_changed_at) # Deployments will not have a deployable when created using the API. deployable_url = if deployment.deployable @@ -15,6 +15,7 @@ module Gitlab { object_kind: 'deployment', status: deployment.status, + status_changed_at: status_changed_at, deployable_id: deployment.deployable_id, deployable_url: deployable_url, environment: deployment.environment.name, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index a56029c0d1d..766eaf54afe 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -31,6 +31,7 @@ module Gitlab created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration, + queued_duration: pipeline.queued_duration, variables: pipeline.variables.map(&:hook_attrs) } end @@ -59,6 +60,8 @@ module Gitlab created_at: build.created_at, started_at: build.started_at, finished_at: build.finished_at, + duration: build.duration, + queued_duration: build.queued_duration, when: build.when, manual: build.action?, allow_failure: build.allow_failure, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 3dc8976d8c5..59249c8bc1f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,6 +2,16 @@ module Gitlab module Database + # This constant is used when renaming tables concurrently. + # If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename. + # Example: + # TABLES_TO_BE_RENAMED = { + # 'old_name' => 'new_name' + # }.freeze + TABLES_TO_BE_RENAMED = { + 'analytics_instance_statistics_measurements' => 'analytics_usage_trends_measurements' + }.freeze + # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements MINIMUM_POSTGRES_VERSION = 11 @@ -35,8 +45,27 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze + DEFAULT_POOL_HEADROOM = 10 + + # We configure the database connection pool size automatically based on the + # configured concurrency. We also add some headroom, to make sure we don't run + # out of connections when more threads besides the 'user-facing' ones are + # running. + # + # Read more about this in doc/development/database/client_side_connection_pool.md + def self.default_pool_size + headroom = (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i + + Gitlab::Runtime.max_threads + headroom + end + def self.config - ActiveRecord::Base.configurations[Rails.env] + default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.config || {} + + default_config_hash.with_indifferent_access.tap do |hash| + # Match config/initializers/database_config.rb + hash[:pool] ||= default_pool_size + end end def self.username @@ -123,6 +152,16 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end + def self.nulls_order(field, direction = :asc, nulls_order = :nulls_last) + raise ArgumentError unless [:nulls_last, :nulls_first].include?(nulls_order) + raise ArgumentError unless [:asc, :desc].include?(direction) + + case nulls_order + when :nulls_last then nulls_last_order(field, direction) + when :nulls_first then nulls_first_order(field, direction) + end + end + def self.nulls_last_order(field, direction = 'ASC') Arel.sql("#{field} #{direction} NULLS LAST") end @@ -204,23 +243,13 @@ module Gitlab # pool_size - The size of the DB pool. # host - An optional host name to use instead of the default one. def self.create_connection_pool(pool_size, host = nil, port = nil) - env = Rails.env - original_config = ActiveRecord::Base.configurations.to_h - - env_config = original_config[env].merge('pool' => pool_size) - env_config['host'] = host if host - env_config['port'] = port if port - - config = ActiveRecord::DatabaseConfigurations.new( - original_config.merge(env => env_config) - ) + original_config = Gitlab::Database.config - spec = - ActiveRecord:: - ConnectionAdapters:: - ConnectionSpecification::Resolver.new(config).spec(env.to_sym) + env_config = original_config.merge(pool: pool_size) + env_config[:host] = host if host + env_config[:port] = port if port - ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(env_config) end def self.connection @@ -246,7 +275,7 @@ module Gitlab connection true - rescue + rescue StandardError false end @@ -347,4 +376,4 @@ module Gitlab end end -Gitlab::Database.prepend_if_ee('EE::Gitlab::Database') +Gitlab::Database.prepend_mod_with('Gitlab::Database') diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index 7c45f416638..e7e3c1766a9 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -3,19 +3,15 @@ module Gitlab module Database # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries. - class AsWithMaterialized < Arel::Nodes::Binary + class AsWithMaterialized < Arel::Nodes::As extend Gitlab::Utils::StrongMemoize - MATERIALIZED = Arel.sql(' MATERIALIZED') - EMPTY_STRING = Arel.sql('') - attr_reader :expr + MATERIALIZED = 'MATERIALIZED ' def initialize(left, right, materialized: true) - @expr = if materialized && self.class.materialized_supported? - MATERIALIZED - else - EMPTY_STRING - end + if materialized && self.class.materialized_supported? + right.prepend(MATERIALIZED) + end super(left, right) end diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb new file mode 100644 index 00000000000..0668490dda8 --- /dev/null +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + # This is an optimizer for throughput of batched migration jobs + # + # The underyling mechanic is based on the concept of time efficiency: + # time efficiency = job duration / interval + # Ideally, this is close but lower than 1 - so we're using time efficiently. + # + # We aim to land in the 90%-98% range, which gives the database a little breathing room + # in between. + # + # The optimizer is based on calculating the exponential moving average of time efficiencies + # for the last N jobs. If we're outside the range, we add 10% to or decrease by 20% of the batch size. + class BatchOptimizer + # Target time efficiency for a job + # Time efficiency is defined as: job duration / interval + TARGET_EFFICIENCY = (0.9..0.95).freeze + + # Lower and upper bound for the batch size + ALLOWED_BATCH_SIZE = (1_000..2_000_000).freeze + + # Limit for the multiplier of the batch size + MAX_MULTIPLIER = 1.2 + + # When smoothing time efficiency, use this many jobs + NUMBER_OF_JOBS = 20 + + # Smoothing factor for exponential moving average + EMA_ALPHA = 0.4 + + attr_reader :migration, :number_of_jobs, :ema_alpha + + def initialize(migration, number_of_jobs: NUMBER_OF_JOBS, ema_alpha: EMA_ALPHA) + @migration = migration + @number_of_jobs = number_of_jobs + @ema_alpha = ema_alpha + end + + def optimize! + return unless Feature.enabled?(:optimize_batched_migrations, type: :ops, default_enabled: :yaml) + + if multiplier = batch_size_multiplier + migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(ALLOWED_BATCH_SIZE) + migration.save! + end + end + + private + + def batch_size_multiplier + efficiency = migration.smoothed_time_efficiency(number_of_jobs: number_of_jobs, alpha: ema_alpha) + + return if efficiency.nil? || efficiency == 0 + + # We hit the range - no change + return if TARGET_EFFICIENCY.include?(efficiency) + + # Assumption: time efficiency is linear in the batch size + [TARGET_EFFICIENCY.max / efficiency, MAX_MULTIPLIER].min + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 3b624df2bfd..869b97b8ac0 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -4,10 +4,23 @@ module Gitlab module Database module BackgroundMigration class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + include FromUnion + self.table_name = :batched_background_migration_jobs + MAX_ATTEMPTS = 3 + STUCK_JOBS_TIMEOUT = 1.hour.freeze + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + scope :active, -> { where(status: [:pending, :running]) } + scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) } + scope :retriable, -> { + failed_jobs = where(status: :failed).where('attempts < ?', MAX_ATTEMPTS) + + from_union([failed_jobs, self.stuck]) + } + enum status: { pending: 0, running: 1, @@ -15,8 +28,22 @@ module Gitlab succeeded: 3 } + scope :successful_in_execution_order, -> { where.not(finished_at: nil).succeeded.order(:finished_at) } + delegate :aborted?, :job_class, :table_name, :column_name, :job_arguments, to: :batched_migration, prefix: :migration + + attribute :pause_ms, :integer, default: 100 + + def time_efficiency + return unless succeeded? + return unless finished_at && started_at + + duration = finished_at - started_at + + # TODO: Switch to individual job interval (prereq: https://gitlab.com/gitlab-org/gitlab/-/issues/328801) + duration.to_f / batched_migration.interval + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 4aa33ed7946..e85162f355e 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -20,9 +20,12 @@ module Gitlab paused: 0, active: 1, aborted: 2, - finished: 3 + finished: 3, + failed: 4 } + attribute :pause_ms, :integer, default: 100 + def self.active_migration active.queue_order.first end @@ -35,7 +38,13 @@ module Gitlab end def create_batched_job!(min, max) - batched_jobs.create!(min_value: min, max_value: max, batch_size: batch_size, sub_batch_size: sub_batch_size) + batched_jobs.create!( + min_value: min, + max_value: max, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + pause_ms: pause_ms + ) end def next_min_value @@ -58,12 +67,40 @@ module Gitlab write_attribute(:batch_class_name, class_name.demodulize) end + def migrated_tuple_count + batched_jobs.succeeded.sum(:batch_size) + end + def prometheus_labels @prometheus_labels ||= { migration_id: id, migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name] } end + + def smoothed_time_efficiency(number_of_jobs: 10, alpha: 0.2) + jobs = batched_jobs.successful_in_execution_order.reverse_order.limit(number_of_jobs) + + return if jobs.size < number_of_jobs + + efficiencies = jobs.map(&:time_efficiency).reject(&:nil?).each_with_index + + dividend = efficiencies.reduce(0) do |total, (job_eff, i)| + total + job_eff * (1 - alpha)**i + end + + divisor = efficiencies.reduce(0) do |total, (job_eff, i)| + total + (1 - alpha)**i + end + + return if divisor == 0 + + (dividend / divisor).round(2) + end + + def optimize! + BatchOptimizer.new(self).optimize! + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index cf8b61f5feb..67fe6c536e6 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -19,8 +19,10 @@ module Gitlab # # Note that this method is primarily intended to called by a scheduled worker. def run_migration_job(active_migration) - if next_batched_job = create_next_batched_job!(active_migration) + if next_batched_job = find_or_create_next_batched_job(active_migration) migration_wrapper.perform(next_batched_job) + + active_migration.optimize! else finish_active_migration(active_migration) end @@ -46,12 +48,12 @@ module Gitlab attr_reader :migration_wrapper - def create_next_batched_job!(active_migration) - next_batch_range = find_next_batch_range(active_migration) - - return if next_batch_range.nil? - - active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + def find_or_create_next_batched_job(active_migration) + if next_batch_range = find_next_batch_range(active_migration) + active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + else + active_migration.batched_jobs.retriable.first + end end def find_next_batch_range(active_migration) @@ -80,7 +82,13 @@ module Gitlab end def finish_active_migration(active_migration) - active_migration.finished! + return if active_migration.batched_jobs.active.exists? + + if active_migration.batched_jobs.failed.exists? + active_migration.failed! + else + active_migration.finished! + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index c276f8ce75b..e37df102872 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -19,10 +19,10 @@ module Gitlab execute_batch(batch_tracking_record) batch_tracking_record.status = :succeeded - rescue => e + rescue Exception # rubocop:disable Lint/RescueException batch_tracking_record.status = :failed - raise e + raise ensure finish_tracking_execution(batch_tracking_record) track_prometheus_metrics(batch_tracking_record) @@ -31,7 +31,7 @@ module Gitlab private def start_tracking_execution(tracking_record) - tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current) + tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current, finished_at: nil, metrics: {}) end def execute_batch(tracking_record) @@ -43,6 +43,7 @@ module Gitlab tracking_record.migration_table_name, tracking_record.migration_column_name, tracking_record.sub_batch_size, + tracking_record.pause_ms, *tracking_record.migration_job_arguments) if job_instance.respond_to?(:batch_metrics) @@ -61,11 +62,12 @@ module Gitlab metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size) metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size) + metric_for(:gauge_interval).set(base_labels, tracking_record.batched_migration.interval) + metric_for(:gauge_job_duration).set(base_labels, (tracking_record.finished_at - tracking_record.started_at).to_i) metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size) - - # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1) - efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f - metric_for(:histogram_time_efficiency).observe(base_labels, efficiency) + metric_for(:gauge_migrated_tuples).set(base_labels, tracking_record.batched_migration.migrated_tuple_count) + metric_for(:gauge_total_tuple_count).set(base_labels, tracking_record.batched_migration.total_tuple_count) + metric_for(:gauge_last_update_time).set(base_labels, Time.current.to_i) if metrics = tracking_record.metrics metrics['timings']&.each do |key, timings| @@ -94,21 +96,35 @@ module Gitlab :batched_migration_job_sub_batch_size, 'Sub-batch size for a batched migration job' ), + gauge_interval: Gitlab::Metrics.gauge( + :batched_migration_job_interval_seconds, + 'Interval for a batched migration job' + ), + gauge_job_duration: Gitlab::Metrics.gauge( + :batched_migration_job_duration_seconds, + 'Duration for a batched migration job' + ), counter_updated_tuples: Gitlab::Metrics.counter( :batched_migration_job_updated_tuples_total, 'Number of tuples updated by batched migration job' ), + gauge_migrated_tuples: Gitlab::Metrics.gauge( + :batched_migration_migrated_tuples_total, + 'Total number of tuples migrated by a batched migration' + ), histogram_timings: Gitlab::Metrics.histogram( - :batched_migration_job_duration_seconds, - 'Timings for a batched migration job', + :batched_migration_job_query_duration_seconds, + 'Query timings for a batched migration job', {}, [0.1, 0.25, 0.5, 1, 5].freeze ), - histogram_time_efficiency: Gitlab::Metrics.histogram( - :batched_migration_job_time_efficiency, - 'Ratio of job duration to interval', - {}, - [0.5, 0.9, 1, 1.5, 2].freeze + gauge_total_tuple_count: Gitlab::Metrics.gauge( + :batched_migration_total_tuple_count, + 'Total tuple count the migration needs to touch' + ), + gauge_last_update_time: Gitlab::Metrics.gauge( + :batched_migration_last_update_time_seconds, + 'Unix epoch time in seconds' ) } end diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index 1b9d7cbc9a1..1121793917b 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -9,7 +9,7 @@ module Gitlab scope :for_migration_class, -> (class_name) { where(class_name: normalize_class_name(class_name)) } scope :for_migration_execution, -> (class_name, arguments) do - for_migration_class(class_name).where('arguments = ?', arguments.to_json) + for_migration_class(class_name).where('arguments = ?', arguments.to_json) # rubocop:disable Rails/WhereEquals end scope :for_partitioning_migration, -> (class_name, table_name) do diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb index b7d06a26ddb..e99ea7a3232 100644 --- a/lib/gitlab/database/consistency.rb +++ b/lib/gitlab/database/consistency.rb @@ -28,4 +28,4 @@ module Gitlab end end -::Gitlab::Database::Consistency.singleton_class.prepend_if_ee('EE::Gitlab::Database::Consistency') +::Gitlab::Database::Consistency.singleton_class.prepend_mod_with('Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb index 884f4d47ff8..26be07f91c4 100644 --- a/lib/gitlab/database/loose_index_scan_distinct_count.rb +++ b/lib/gitlab/database/loose_index_scan_distinct_count.rb @@ -11,7 +11,7 @@ module Gitlab # This query will read each element in the index matching the project_id filter. # If for a project_id has 100_000 issues, all 100_000 elements will be read. # - # A loose index scan will read only one entry from the index for each project_id to reduce the number of disk reads. + # A loose index scan will only read one entry from the index for each project_id to reduce the number of disk reads. # # Usage: # @@ -94,7 +94,7 @@ module Gitlab elsif column.is_a?(Arel::Attributes::Attribute) column else - raise ColumnConfigurationError.new("Cannot transform the column: #{column.inspect}, please provide the column name as string") + raise ColumnConfigurationError, "Cannot transform the column: #{column.inspect}, please provide the column name as string" end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index d06a73da8ac..3a94e109d2a 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module MigrationHelpers include Migrations::BackgroundMigrationHelpers include DynamicModelHelpers + include Migrations::RenameTableHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -51,7 +52,7 @@ module Gitlab allow_null: options[:null] ) else - add_column(table_name, column_name, :datetime_with_timezone, options) + add_column(table_name, column_name, :datetime_with_timezone, **options) end end end @@ -143,13 +144,13 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) - if index_exists?(table_name, column_name, options) + if index_exists?(table_name, column_name, **options) Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" return end disable_statement_timeout do - add_index(table_name, column_name, options) + add_index(table_name, column_name, **options) end end @@ -169,13 +170,13 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) - unless index_exists?(table_name, column_name, options) + unless index_exists?(table_name, column_name, **options) Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" return end disable_statement_timeout do - remove_index(table_name, options.merge({ column: column_name })) + remove_index(table_name, **options.merge({ column: column_name })) end end @@ -205,7 +206,7 @@ module Gitlab end disable_statement_timeout do - remove_index(table_name, options.merge({ name: index_name })) + remove_index(table_name, **options.merge({ name: index_name })) end end @@ -565,7 +566,7 @@ module Gitlab check_trigger_permissions!(table) - remove_rename_triggers_for_postgresql(table, trigger_name) + remove_rename_triggers(table, trigger_name) remove_column(table, new) end @@ -576,8 +577,19 @@ module Gitlab # table - The name of the table to install the trigger in. # old_column - The name of the old column. # new_column - The name of the new column. - def install_rename_triggers(table, old_column, new_column) - install_rename_triggers_for_postgresql(table, old_column, new_column) + # trigger_name - The name of the trigger to use (optional). + def install_rename_triggers(table, old, new, trigger_name: nil) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) + end + + # Removes the triggers used for renaming a column concurrently. + def remove_rename_triggers(table, trigger) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) + end + + # Returns the (base) name to use for triggers when renaming columns. + def rename_trigger_name(table, old, new) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) end # Changes the type of a column concurrently. @@ -663,7 +675,7 @@ module Gitlab install_rename_triggers(table, column, temp_column) end - rescue + rescue StandardError # create_column_from can not run inside a transaction, which means # that there is a risk that if any of the operations that follow it # fail, we'll be left with an inconsistent schema @@ -690,7 +702,7 @@ module Gitlab check_trigger_permissions!(table) - remove_rename_triggers_for_postgresql(table, trigger_name) + remove_rename_triggers(table, trigger_name) remove_column(table, old) end @@ -905,7 +917,11 @@ module Gitlab end end - # Initializes the conversion of an integer column to bigint + def convert_to_bigint_column(column) + "#{column}_convert_to_bigint" + end + + # Initializes the conversion of a set of integer columns to bigint # # It can be used for converting both a Primary Key and any Foreign Keys # that may reference it or any other integer column that we may want to @@ -923,14 +939,14 @@ module Gitlab # Note: this helper is intended to be used in a regular (pre-deployment) migration. # # This helper is part 1 of a multi-step migration process: - # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 1. initialize_conversion_of_integer_to_bigint to create the new columns and database trigger # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column - # column - The name of the column that we want to convert to bigint. + # columns - The name, or array of names, of the column(s) that we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) - def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id) + def initialize_conversion_of_integer_to_bigint(table, columns, primary_key: :id) unless table_exists?(table) raise "Table #{table} does not exist" end @@ -939,34 +955,54 @@ module Gitlab raise "Column #{primary_key} does not exist on #{table}" end - unless column_exists?(table, column) - raise "Column #{column} does not exist on #{table}" + columns = Array.wrap(columns) + columns.each do |column| + next if column_exists?(table, column) + + raise ArgumentError, "Column #{column} does not exist on #{table}" end check_trigger_permissions!(table) - old_column = column_for(table, column) - tmp_column = "#{column}_convert_to_bigint" + conversions = columns.to_h { |column| [column, convert_to_bigint_column(column)] } with_lock_retries do - if (column.to_s == primary_key.to_s) || !old_column.null - # If the column to be converted is either a PK or is defined as NOT NULL, - # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow - # That way, we skip the expensive validation step required to add - # a NOT NULL constraint at the end of the process - add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) - else - add_column(table, tmp_column, :bigint, default: old_column.default) + conversions.each do |(source_column, temporary_name)| + column = column_for(table, source_column) + + if (column.name.to_s == primary_key.to_s) || !column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, temporary_name, :bigint, default: column.default || 0, null: false) + else + add_column(table, temporary_name, :bigint, default: column.default) + end end - install_rename_triggers(table, column, tmp_column) + install_rename_triggers(table, conversions.keys, conversions.values) end end - # Backfills the new column used in the conversion of an integer column to bigint using background migrations. + # Reverts `initialize_conversion_of_integer_to_bigint` + # + # table - The name of the database table containing the columns + # columns - The name, or array of names, of the column(s) that we're converting to bigint. + def revert_initialize_conversion_of_integer_to_bigint(table, columns) + columns = Array.wrap(columns) + temporary_columns = columns.map { |column| convert_to_bigint_column(column) } + + trigger_name = rename_trigger_name(table, columns, temporary_columns) + remove_rename_triggers(table, trigger_name) + + temporary_columns.each { |column| remove_column(table, column) } + end + + # Backfills the new columns used in an integer-to-bigint conversion using background migrations. # # - This helper should be called from a post-deployment migration. - # - In order for this helper to work properly, the new column must be first initialized with + # - In order for this helper to work properly, the new columns must be first initialized with # the `initialize_conversion_of_integer_to_bigint` helper. # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration, # which allows a more thorough check that all jobs succeeded in the @@ -976,12 +1012,12 @@ module Gitlab # deployed (including background job changes) before we begin processing the background migration. # # This helper is part 2 of a multi-step migration process: - # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 1. initialize_conversion_of_integer_to_bigint to create the new columns and database trigger # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column - # column - The name of the column that we want to convert to bigint. + # columns - The name, or an array of names, of the column(s) we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) # batch_size - The number of rows to schedule in a single background migration # sub_batch_size - The smaller batches that will be used by each scheduled job @@ -1001,7 +1037,7 @@ module Gitlab # between the scheduled jobs def backfill_conversion_of_integer_to_bigint( table, - column, + columns, primary_key: :id, batch_size: 20_000, sub_batch_size: 1000, @@ -1016,46 +1052,43 @@ module Gitlab raise "Column #{primary_key} does not exist on #{table}" end - unless column_exists?(table, column) - raise "Column #{column} does not exist on #{table}" - end + conversions = Array.wrap(columns).to_h do |column| + raise ArgumentError, "Column #{column} does not exist on #{table}" unless column_exists?(table, column) - tmp_column = "#{column}_convert_to_bigint" + temporary_name = convert_to_bigint_column(column) + raise ArgumentError, "Column #{temporary_name} does not exist on #{table}" unless column_exists?(table, temporary_name) - unless column_exists?(table, tmp_column) - raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`' + [column, temporary_name] end - batched_migration = queue_batched_background_migration( + queue_batched_background_migration( 'CopyColumnUsingBackgroundMigrationJob', table, primary_key, - column, - tmp_column, + conversions.keys, + conversions.values, job_interval: interval, batch_size: batch_size, sub_batch_size: sub_batch_size) - - if perform_background_migration_inline? - # To ensure the schema is up to date immediately we perform the - # migration inline in dev / test environments. - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration) - end end - # Performs a concurrent column rename when using PostgreSQL. - def install_rename_triggers_for_postgresql(table, old, new, trigger_name: nil) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) - end + # Reverts `backfill_conversion_of_integer_to_bigint` + # + # table - The name of the database table containing the column + # columns - The name, or an array of names, of the column(s) we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + def revert_backfill_conversion_of_integer_to_bigint(table, columns, primary_key: :id) + columns = Array.wrap(columns) - # Removes the triggers used for renaming a PostgreSQL column concurrently. - def remove_rename_triggers_for_postgresql(table, trigger) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) - end + conditions = ActiveRecord::Base.sanitize_sql([ + 'job_class_name = :job_class_name AND table_name = :table_name AND column_name = :column_name AND job_arguments = :job_arguments', + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: table, + column_name: primary_key, + job_arguments: [columns, columns.map { |column| convert_to_bigint_column(column) }].to_json + ]) - # Returns the (base) name to use for triggers when renaming columns. - def rename_trigger_name(table, old, new) - Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) + execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end # Returns an Array containing the indexes for the given column @@ -1162,8 +1195,8 @@ module Gitlab end end - def remove_foreign_key_without_error(*args) - remove_foreign_key(*args) + def remove_foreign_key_without_error(*args, **kwargs) + remove_foreign_key(*args, **kwargs) rescue ArgumentError end diff --git a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb new file mode 100644 index 00000000000..eecf96acb30 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module CascadingNamespaceSettings + include Gitlab::Database::MigrationHelpers + + # Creates the four required columns that constitutes a single cascading + # namespace settings attribute. This helper is only appropriate if the + # setting is not already present as a non-cascading attribute. + # + # Creates the `setting_name` column along with the `lock_setting_name` + # column in both `namespace_settings` and `application_settings`. + # + # This helper is not reversible and must be defined in conjunction with + # `remove_cascading_namespace_setting` in separate up and down directions. + # + # setting_name - The name of the cascading attribute - same as defined + # in `NamespaceSetting` with the `cascading_attr` method. + # type - The column type for the setting itself (:boolean, :integer, etc.) + # options - Standard Rails column options hash. Accepts keys such as + # `null` and `default`. + # + # `null` and `default` options will only be applied to the `application_settings` + # column. In most cases, a non-null default value should be specified. + def add_cascading_namespace_setting(setting_name, type, **options) + lock_column_name = "lock_#{setting_name}".to_sym + + check_cascading_namespace_setting_consistency(setting_name, lock_column_name) + + namespace_options = options.merge(null: true, default: nil) + + with_lock_retries do + add_column(:namespace_settings, setting_name, type, namespace_options) + add_column(:namespace_settings, lock_column_name, :boolean, default: false, null: false) + end + + add_column(:application_settings, setting_name, type, options) + add_column(:application_settings, lock_column_name, :boolean, default: false, null: false) + end + + def remove_cascading_namespace_setting(setting_name) + lock_column_name = "lock_#{setting_name}".to_sym + + with_lock_retries do + remove_column(:namespace_settings, setting_name) if column_exists?(:namespace_settings, setting_name) + remove_column(:namespace_settings, lock_column_name) if column_exists?(:namespace_settings, lock_column_name) + end + + remove_column(:application_settings, setting_name) if column_exists?(:application_settings, setting_name) + remove_column(:application_settings, lock_column_name) if column_exists?(:application_settings, lock_column_name) + end + + private + + def check_cascading_namespace_setting_consistency(setting_name, lock_name) + existing_columns = [] + + %w(namespace_settings application_settings).each do |table| + existing_columns << "#{table}.#{setting_name}" if column_exists?(table.to_sym, setting_name) + existing_columns << "#{table}.#{lock_name}" if column_exists?(table.to_sym, lock_name) + end + + return if existing_columns.empty? + + raise <<~ERROR + One or more cascading namespace columns already exist. `add_cascading_namespace_setting` helper + can only be used for new settings, when none of the required columns already exist. + Existing columns: #{existing_columns.join(', ')} + ERROR + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 959028ce00b..e9ef80d5198 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -4,6 +4,9 @@ module Gitlab module Database module Migrations class Instrumentation + RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze + STATS_FILENAME = 'migration-stats.json' + attr_reader :observations def initialize(observers = ::Gitlab::Database::Migrations::Observers.all_observers) @@ -21,7 +24,7 @@ module Gitlab observation.walltime = Benchmark.realtime do yield - rescue => e + rescue StandardError => e exception = e observation.success = false end @@ -47,7 +50,7 @@ module Gitlab def on_each_observer(&block) observers.each do |observer| yield observer - rescue => e + rescue StandardError => e Gitlab::AppLogger.error("Migration observer #{observer.class} failed with: #{e}") end end diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 592993aeac5..b65a303ef30 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -7,7 +7,8 @@ module Gitlab def self.all_observers [ TotalDatabaseSizeChange.new, - QueryStatistics.new + QueryStatistics.new, + QueryLog.new ] end end diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb new file mode 100644 index 00000000000..45df07fe391 --- /dev/null +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class QueryLog < MigrationObserver + def before + @logger_was = ActiveRecord::Base.logger + @log_file_path = File.join(Instrumentation::RESULT_DIR, 'current.log') + @logger = Logger.new(@log_file_path) + ActiveRecord::Base.logger = @logger + end + + def after + ActiveRecord::Base.logger = @logger_was + @logger.close + end + + def record(observation) + File.rename(@log_file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}.log")) + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb index 547e0b9b957..d4b2b8d50e2 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -38,7 +38,7 @@ module Gitlab create(model, partitions_to_create) end - rescue => e + rescue StandardError => e Gitlab::AppLogger.error("Failed to create partition(s) for #{model.table_name}: #{e.class}: #{e.message}") end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index 0bc1343acca..c0cc97de276 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -40,7 +40,7 @@ module Gitlab end with_lock_retries do - add_index(table_name, column_names, options) + add_index(table_name, column_names, **options) end end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index e8b49c7f62c..aa46b98be5d 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -69,10 +69,8 @@ module Gitlab hll_buckets = Buckets.new while batch_start <= finish - begin - hll_buckets.merge_hash!(hll_buckets_for_batch(batch_start, batch_start + batch_size)) - batch_start += batch_size - end + hll_buckets.merge_hash!(hll_buckets_for_batch(batch_start, batch_start + batch_size)) + batch_start += batch_size sleep(SLEEP_TIME_IN_SECONDS) end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb index a6fe7d61a4f..7e2dd55d21b 100644 --- a/lib/gitlab/database/reindexing/concurrent_reindex.rb +++ b/lib/gitlab/database/reindexing/concurrent_reindex.rb @@ -11,7 +11,14 @@ module Gitlab PG_IDENTIFIER_LENGTH = 63 TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' REPLACED_INDEX_PREFIX = 'old_reindex_' - STATEMENT_TIMEOUT = 6.hours + STATEMENT_TIMEOUT = 9.hours + + # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, + # which only conflicts with DDL and vacuum. We therefore execute this with a rather + # high lock timeout and a long pause in between retries. This is an alternative to + # setting a high statement timeout, which would lead to a long running query with effects + # on e.g. vacuum. + REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 attr_reader :index, :logger @@ -70,7 +77,7 @@ module Gitlab ensure begin remove_index(index.schema, replacement_index_name) - rescue => e + rescue StandardError => e logger.error(e) end end @@ -95,7 +102,13 @@ module Gitlab def remove_index(schema, name) logger.info("Removing index #{schema}.#{name}") - set_statement_timeout do + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: logger + ) + + retries.run(raise_on_exhaustion: false) do connection.execute(<<~SQL) DROP INDEX CONCURRENTLY IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} @@ -121,7 +134,6 @@ module Gitlab def with_lock_retries(&block) arguments = { klass: self.class, logger: logger } - Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 7a7d17ca196..d68f47b5b6c 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -42,7 +42,7 @@ module Gitlab def perform_for(index, action) ConcurrentReindex.new(index).perform - rescue + rescue StandardError action.state = :failed raise diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb index b1e5ecb9ade..f4ea59deb50 100644 --- a/lib/gitlab/database/reindexing/grafana_notifier.rb +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -53,7 +53,7 @@ module Gitlab log_error("Response code #{response.code}") unless success success - rescue => err + rescue StandardError => err log_error(err) false diff --git a/lib/gitlab/database/rename_table_helpers.rb b/lib/gitlab/database/rename_table_helpers.rb new file mode 100644 index 00000000000..7f5af038c6d --- /dev/null +++ b/lib/gitlab/database/rename_table_helpers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module RenameTableHelpers + def rename_table_safely(old_table_name, new_table_name) + with_lock_retries do + rename_table(old_table_name, new_table_name) + execute("CREATE VIEW #{old_table_name} AS SELECT * FROM #{new_table_name}") + end + end + + def undo_rename_table_safely(old_table_name, new_table_name) + with_lock_retries do + execute("DROP VIEW IF EXISTS #{old_table_name}") + rename_table(new_table_name, old_table_name) + end + end + + def finalize_table_rename(old_table_name, new_table_name) + with_lock_retries do + execute("DROP VIEW IF EXISTS #{old_table_name}") + end + end + + def undo_finalize_table_rename(old_table_name, new_table_name) + with_lock_retries do + execute("CREATE VIEW #{old_table_name} AS SELECT * FROM #{new_table_name}") + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb new file mode 100644 index 00000000000..28123edd708 --- /dev/null +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaCacheWithRenamedTable + # Override methods in ActiveRecord::ConnectionAdapters::SchemaCache + + def clear! + super + + clear_renamed_tables_cache! + end + + def clear_data_source_cache!(name) + super(name) + + clear_renamed_tables_cache! + end + + def primary_keys(table_name) + super(underlying_table(table_name)) + end + + def columns(table_name) + super(underlying_table(table_name)) + end + + def columns_hash(table_name) + super(underlying_table(table_name)) + end + + def indexes(table_name) + super(underlying_table(table_name)) + end + + private + + def underlying_table(table_name) + renamed_tables_cache.fetch(table_name, table_name) + end + + def renamed_tables_cache + @renamed_tables ||= begin + Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + ActiveRecord::Base.connection.view_exists?(old_name) + end + end + end + + def clear_renamed_tables_cache! + @renamed_tables = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index 3fb52d786ad..bbf8f133f0f 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -92,7 +92,7 @@ module Gitlab end begin - run_block_with_transaction + run_block_with_lock_timeout rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open? @@ -121,7 +121,7 @@ module Gitlab block.call end - def run_block_with_transaction + def run_block_with_lock_timeout ActiveRecord::Base.transaction(requires_new: true) do execute("SET LOCAL lock_timeout TO '#{current_lock_timeout_in_ms}ms'") diff --git a/lib/gitlab/database/with_lock_retries_outside_transaction.rb b/lib/gitlab/database/with_lock_retries_outside_transaction.rb new file mode 100644 index 00000000000..175cc493e36 --- /dev/null +++ b/lib/gitlab/database/with_lock_retries_outside_transaction.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This retry method behaves similar to WithLockRetries + # except it does not wrap itself into a transaction scope. + # + # In our context, this is only useful if directly connected to + # PostgreSQL. When going through pgbouncer, this method **won't work** + # as it relies on using `SET` outside transactions (and hence can be + # multiplexed across different connections). + class WithLockRetriesOutsideTransaction < WithLockRetries + private + + def run_block_with_lock_timeout + execute("SET lock_timeout TO '#{current_lock_timeout_in_ms}ms'") + + log(message: 'Lock timeout is set', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) + + run_block + + log(message: 'Migration finished', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms) + end + + def run_block_without_lock_timeout + log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) + log(message: "Executing without lock timeout", current_iteration: current_iteration) + + disable_lock_timeout + + run_block + + log(message: 'Migration finished', current_iteration: current_iteration) + end + + def disable_lock_timeout + execute("SET lock_timeout TO '0'") + end + end + end +end diff --git a/lib/gitlab/default_branch.rb b/lib/gitlab/default_branch.rb new file mode 100644 index 00000000000..6bd9a5675c4 --- /dev/null +++ b/lib/gitlab/default_branch.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Class is used while we're migrating from master to main +module Gitlab + module DefaultBranch + def self.value(object: nil) + Feature.enabled?(:main_branch_over_master, object, default_enabled: :yaml) ? 'main' : 'master' + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 627abfbfe7e..9ed03c05f0b 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -117,8 +117,6 @@ module Gitlab end def sort_diffs(diffs) - return diffs unless Feature.enabled?(:sort_diffs, project, default_enabled: :yaml) - Gitlab::Diff::FileCollectionSorter.new(diffs).sort end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 8385bbbb3de..6a41ed0f29e 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,6 +3,8 @@ module Gitlab module Diff class Highlight + PREFIX_REGEXP = /\A(.)/.freeze + attr_reader :diff_file, :diff_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff @@ -85,6 +87,7 @@ module Gitlab def highlight_line(diff_line) return unless diff_file && diff_file.diff_refs + return diff_line_highlighting(diff_line, plain: true) if blobs_too_large? if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml) diff_line_highlighting(diff_line) @@ -93,16 +96,17 @@ module Gitlab end end - def diff_line_highlighting(diff_line) + def diff_line_highlighting(diff_line, plain: false) rich_line = syntax_highlighter(diff_line).highlight( diff_line.text(prefix: false), + plain: plain, context: { line_number: diff_line.line } - )&.html_safe + ) # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' + line_prefix = diff_line.text =~ PREFIX_REGEXP ? Regexp.last_match(1) : ' ' rich_line.prepend(line_prefix).concat("\n") end end @@ -131,7 +135,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' + line_prefix = diff_line.text =~ PREFIX_REGEXP ? Regexp.last_match(1) : ' ' "#{line_prefix}#{rich_line}".html_safe end end @@ -156,6 +160,13 @@ module Gitlab blob.load_all_data! blob.present.highlight.lines end + + def blobs_too_large? + return false unless Feature.enabled?(:limited_diff_highlighting, project, default_enabled: :yaml) + return true if Gitlab::Highlight.too_large?(diff_file.old_blob&.size) + + Gitlab::Highlight.too_large?(diff_file.new_blob&.size) + end end end end diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index 31c5dded3ff..1a1e9fafb1e 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -77,7 +77,7 @@ module Gitlab true rescue OpenSSL::Cipher::CipherError, TypeError false - rescue => e + rescue StandardError => e logger.debug "> Something went wrong for #{data.class.name}[#{data.id}].#{attr}: #{e}".color(:red) false diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 22fc8addcd9..e927a5641e5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -56,10 +56,12 @@ module Gitlab def create_issue Issues::CreateService.new( - project, - author, - title: mail.subject, - description: message_including_reply + project: project, + current_user: author, + params: { + title: mail.subject, + description: message_including_reply + } ).execute end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e8071bcafd0..df12aea1988 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -61,7 +61,7 @@ module Gitlab private def build_merge_request - MergeRequests::BuildService.new(project, author, merge_request_params).execute + MergeRequests::BuildService.new(project: project, current_user: author, params: merge_request_params).execute end def create_merge_request @@ -78,7 +78,7 @@ module Gitlab if merge_request.errors.any? merge_request else - MergeRequests::CreateService.new(project, author).create(merge_request) + MergeRequests::CreateService.new(project: project, current_user: author).create(merge_request) end end diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index 9e476dd4e2b..63334169c8e 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -100,4 +100,4 @@ module Gitlab end end -Gitlab::Email::Handler::ReplyProcessing.prepend_if_ee('::EE::Gitlab::Email::Handler::ReplyProcessing') +Gitlab::Email::Handler::ReplyProcessing.prepend_mod_with('Gitlab::Email::Handler::ReplyProcessing') diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 80e8b726099..cab3538a447 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -38,7 +38,7 @@ module Gitlab if from_address add_email_participant - send_thank_you_email! + send_thank_you_email end end @@ -77,12 +77,14 @@ module Gitlab def create_issue! @issue = Issues::CreateService.new( - project, - User.support_bot, - title: mail.subject, - description: message_including_template, - confidential: true, - external_author: from_address + project: project, + current_user: User.support_bot, + params: { + title: mail.subject, + description: message_including_template, + confidential: true, + external_author: from_address + } ).execute raise InvalidIssueError unless @issue.persisted? @@ -92,8 +94,8 @@ module Gitlab end end - def send_thank_you_email! - Notify.service_desk_thank_you_email(@issue.id).deliver_later! + def send_thank_you_email + Notify.service_desk_thank_you_email(@issue.id).deliver_later end def message_including_template diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb new file mode 100644 index 00000000000..d538238f26f --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + UnknownTrackError = Class.new(StandardError) + + TRACKS = [:create, :verify, :team, :trial].freeze + + def self.for(track) + raise UnknownTrackError unless TRACKS.include?(track) + + "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb new file mode 100644 index 00000000000..6341a7c7596 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Base + include Gitlab::Email::Message::InProductMarketing::Helper + include Gitlab::Routing + + attr_accessor :format + + def initialize(group:, series:, format: :html) + raise ArgumentError, "Only #{total_series} series available for this track." unless series.between?(0, total_series - 1) + + @group = group + @series = series + @format = format + end + + def subject_line + raise NotImplementedError + end + + def tagline + raise NotImplementedError + end + + def title + raise NotImplementedError + end + + def subtitle + raise NotImplementedError + end + + def body_line1 + raise NotImplementedError + end + + def body_line2 + raise NotImplementedError + end + + def cta_text + raise NotImplementedError + end + + def cta_link + case format + when :html + link_to cta_text, group_email_campaigns_url(group, track: track, series: series), target: '_blank', rel: 'noopener noreferrer' + else + [cta_text, group_email_campaigns_url(group, track: track, series: series)].join(' >> ') + end + end + + def unsubscribe + parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(track, series) + + case format + when :html + parts.join(' ') + else + parts.join("\n" + ' ' * 16) + end + end + + def progress + if Gitlab.com? + s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize } + else + s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize, unsubscribe_link: unsubscribe_link } + end + end + + def address + s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options + end + + def footer_links + links = [ + [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], + [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], + [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], + [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] + ] + case format + when :html + links.map do |text, link| + link_to(text, link) + end + else + '| ' + links.map do |text, link| + [text, link].join(' ') + end.join("\n| ") + end + end + + def logo_path + ["mailers/in_product_marketing", "#{track}-#{series}.png"].join('/') + end + + protected + + attr_reader :group, :series + + def total_series + 3 + end + + private + + def track + self.class.name.demodulize.downcase.to_sym + end + + def unsubscribe_com + [ + s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), + s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } + ] + end + + def unsubscribe_self_managed(track, series) + [ + s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, + s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: marketing_preference_link(track, series) } + ] + end + + def unsubscribe_link + unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url + + link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) + end + + def marketing_preference_link(track, series) + params = { + utm_source: 'SM', + utm_medium: 'email', + utm_campaign: 'onboarding', + utm_term: "#{track}_#{series}" + } + + preference_link = "https://about.gitlab.com/company/preference-center/?#{params.to_query}" + + link(s_('InProductMarketing|update your preferences'), preference_link) + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb new file mode 100644 index 00000000000..5d3cac0a121 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Create < Base + def subject_line + [ + s_('InProductMarketing|Create a project in GitLab in 5 minutes'), + s_('InProductMarketing|Import your project and code from GitHub, Bitbucket and others'), + s_('InProductMarketing|Understand repository mirroring') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Get started today'), + s_('InProductMarketing|Get our import guides'), + s_('InProductMarketing|Need an alternative to importing?') + ][series] + end + + def title + [ + s_('InProductMarketing|Take your first steps with GitLab'), + s_('InProductMarketing|Start by importing your projects'), + s_('InProductMarketing|How (and why) mirroring makes sense') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|Dig in and create a project and a repo'), + s_("InProductMarketing|Here's what you need to know"), + s_('InProductMarketing|Try it out') + ][series] + end + + def body_line1 + [ + s_("InProductMarketing|To understand and get the most out of GitLab, start at the beginning and %{project_link}. In GitLab, repositories are part of a project, so after you've created your project you can go ahead and %{repo_link}.") % { project_link: project_link, repo_link: repo_link }, + s_("InProductMarketing|Making the switch? It's easier than you think to import your projects into GitLab. Move %{github_link}, or import something %{bitbucket_link}.") % { github_link: github_link, bitbucket_link: bitbucket_link }, + s_("InProductMarketing|Sometimes you're not ready to make a full transition to a new tool. If you're not ready to fully commit, %{mirroring_link} gives you a safe way to try out GitLab in parallel with your current tool.") % { mirroring_link: mirroring_link } + ][series] + end + + def body_line2 + [ + s_("InProductMarketing|That's all it takes to get going with GitLab, but if you're new to working with Git, check out our %{basics_link} for helpful tips and tricks for getting started.") % { basics_link: basics_link }, + s_("InProductMarketing|Have a different instance you'd like to import? Here's our %{import_link}.") % { import_link: import_link }, + s_("InProductMarketing|It's also possible to simply %{external_repo_link} in order to take advantage of GitLab's CI/CD.") % { external_repo_link: external_repo_link } + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Create your first project!'), + s_('InProductMarketing|Master the art of importing!'), + s_('InProductMarketing|Understand your project options') + ][series] + end + + private + + def project_link + link(s_('InProductMarketing|create a project'), help_page_url('gitlab-basics/create-project')) + end + + def repo_link + link(s_('InProductMarketing|set up a repo'), help_page_url('user/project/repository/index', anchor: 'create-a-repository')) + end + + def github_link + link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('integration/github')) + end + + def bitbucket_link + link(s_('InProductMarketing|from Bitbucket'), help_page_url('user/project/import/bitbucket_server')) + end + + def mirroring_link + link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/repository_mirroring')) + end + + def basics_link + link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/README')) + end + + def import_link + link(s_('InProductMarketing|comprehensive guide'), help_page_url('user/project/import/index')) + end + + def external_repo_link + link(s_('InProductMarketing|connect an external repository'), new_project_url(anchor: 'cicd_for_external_repo')) + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb new file mode 100644 index 00000000000..4780e08322a --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + module Helper + include ActionView::Context + include ActionView::Helpers::TagHelper + include ActionView::Helpers::UrlHelper + + private + + def list(array) + case format + when :html + tag.ul { array.map { |item| tag.li item} } + else + '- ' + array.join("\n- ") + end + end + + def strong_options + case format + when :html + { strong_start: '<b>'.html_safe, strong_end: '</b>'.html_safe } + else + { strong_start: '', strong_end: '' } + end + end + + def link(text, link) + case format + when :html + link_to text, link + else + "#{text} (#{link})" + end + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb new file mode 100644 index 00000000000..46c2797e534 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/team.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Team < Base + def subject_line + [ + s_('InProductMarketing|Working in GitLab = more efficient'), + s_("InProductMarketing|Multiple owners, confusing workstreams? We've got you covered"), + s_('InProductMarketing|Your teams can be more efficient') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Invite your colleagues to join in less than one minute'), + s_('InProductMarketing|Get your team set up on GitLab'), + nil + ][series] + end + + def title + [ + s_('InProductMarketing|Team work makes the dream work'), + s_('InProductMarketing|*GitLab*, noun: a synonym for efficient teams'), + s_('InProductMarketing|Find out how your teams are really doing') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|Actually, GitLab makes the team work (better)'), + s_('InProductMarketing|Our tool brings all the things together'), + s_("InProductMarketing|It's all in the stats") + ][series] + end + + def body_line1 + [ + [ + s_('InProductMarketing|Did you know teams that use GitLab are far more efficient?'), + list([ + s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), + s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') + ]) + ].join("\n"), + s_("InProductMarketing|We know a thing or two about efficiency and we don't want to keep that to ourselves. Sign up for a free trial of GitLab Ultimate and your teams will be on it from day one."), + [ + s_('InProductMarketing|Stop wondering and use GitLab to answer questions like:'), + list([ + s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), + s_('InProductMarketing|How many days does it take our team to complete various tasks?'), + s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') + ]) + ].join("\n") + ][series] + end + + def body_line2 + [ + s_('InProductMarketing|Invite your colleagues and start shipping code faster.'), + s_("InProductMarketing|Streamline code review, know at a glance who's unavailable, communicate in comments or in email and integrate with Slack so everyone's on the same page."), + s_('InProductMarketing|When your team is on GitLab these answers are a click away.') + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Invite your colleagues today'), + s_('InProductMarketing|Invite your team in less than 60 seconds'), + s_('InProductMarketing|Invite your team now') + ][series] + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb new file mode 100644 index 00000000000..d87dc5c1b81 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/trial.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Trial < Base + def subject_line + [ + s_('InProductMarketing|Go farther with GitLab'), + s_('InProductMarketing|Automated security scans directly within GitLab'), + s_('InProductMarketing|Take your source code management to the next level') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Start a free trial of GitLab Ultimate – no CC required'), + s_('InProductMarketing|Improve app security with a 30-day trial'), + s_('InProductMarketing|Start with a GitLab Ultimate free trial') + ][series] + end + + def title + [ + s_('InProductMarketing|Give us one minute...'), + s_("InProductMarketing|Security that's integrated into your development lifecycle"), + s_('InProductMarketing|Improve code quality and streamline reviews') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|...and you can get a free trial of GitLab Ultimate'), + s_('InProductMarketing|Try GitLab Ultimate for free'), + s_('InProductMarketing|Better code in less time') + ][series] + end + + def body_line1 + [ + [ + s_("InProductMarketing|GitLab's premium tiers are designed to make you, your team and your application more efficient and more secure with features including but not limited to:"), + list([ + s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, + s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options + ]) + ].join("\n"), + s_('InProductMarketing|GitLab provides static application security testing (SAST), dynamic application security testing (DAST), container scanning, and dependency scanning to help you deliver secure applications along with license compliance.'), + s_('InProductMarketing|By enabling code owners and required merge approvals the right person will review the right MR. This is a win-win: cleaner code and a more efficient review process.') + ][series] + end + + def body_line2 + [ + s_('InProductMarketing|Start a GitLab Ultimate trial today in less than one minute, no credit card required.'), + s_('InProductMarketing|Get started today with a 30-day GitLab Ultimate trial, no credit card required.'), + s_('InProductMarketing|Code owners and required merge approvals are part of the paid tiers of GitLab. You can start a free 30-day trial of GitLab Ultimate and enable these features in less than 5 minutes with no credit card required.') + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Start a trial'), + s_('InProductMarketing|Beef up your security'), + s_('InProductMarketing|Start your trial now!') + ][series] + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb new file mode 100644 index 00000000000..d563de6c77e --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class Verify < Base + def subject_line + [ + s_('InProductMarketing|Feel the need for speed?'), + s_('InProductMarketing|3 ways to dive into GitLab CI/CD'), + s_('InProductMarketing|Explore the power of GitLab CI/CD') + ][series] + end + + def tagline + [ + s_('InProductMarketing|Use GitLab CI/CD'), + s_('InProductMarketing|Test, create, deploy'), + s_('InProductMarketing|Are your runners ready?') + ][series] + end + + def title + [ + s_('InProductMarketing|Rapid development, simplified'), + s_('InProductMarketing|Get started with GitLab CI/CD'), + s_('InProductMarketing|Launch GitLab CI/CD in 20 minutes or less') + ][series] + end + + def subtitle + [ + s_('InProductMarketing|How to build and test faster'), + s_('InProductMarketing|Explore the options'), + s_('InProductMarketing|Follow our steps') + ][series] + end + + def body_line1 + [ + s_("InProductMarketing|Tired of wrestling with disparate tool chains, information silos and inefficient processes? GitLab's CI/CD is built on a DevOps platform with source code management, planning, monitoring and more ready to go. Find out %{ci_link}.") % { ci_link: ci_link }, + s_("InProductMarketing|GitLab's CI/CD makes software development easier. Don't believe us? Here are three ways you can take it for a fast (and satisfying) test drive:"), + s_("InProductMarketing|Get going with CI/CD quickly using our %{quick_start_link}. Start with an available runner and then create a CI .yml file – it's really that easy.") % { quick_start_link: quick_start_link } + ][series] + end + + def body_line2 + [ + nil, + list([ + s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, + s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, + s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } + ]), + nil + ][series] + end + + def cta_text + [ + s_('InProductMarketing|Get to know GitLab CI/CD'), + s_('InProductMarketing|Try it yourself'), + s_('InProductMarketing|Explore GitLab CI/CD') + ][series] + end + + private + + def ci_link + link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/README')) + end + + def quick_start_link + link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/README')) + end + + def performance_link + link(s_('InProductMarketing|testing browser performance'), help_page_url('user/project/merge_requests/browser_performance_testing')) + end + + def ci_template_link + link(s_('InProductMarketing|using a CI/CD template'), help_page_url('user/project/pages/getting_started/pages_ci_cd_template')) + end + + def deploy_link + link(s_('InProductMarketing|test and deploy'), help_page_url('ci/examples/test-and-deploy-python-application-to-heroku')) + end + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index f5e47b43a9a..71db8ab6067 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -6,6 +6,8 @@ require_dependency 'gitlab/email/handler' module Gitlab module Email class Receiver + include Gitlab::Utils::StrongMemoize + def initialize(raw) @raw = raw end @@ -13,11 +15,7 @@ module Gitlab def execute raise EmptyEmailError if @raw.blank? - mail = build_mail - - ignore_auto_reply!(mail) - - handler = find_handler(mail) + ignore_auto_reply! raise UnknownIncomingEmail unless handler @@ -26,13 +24,33 @@ module Gitlab end end + def mail_metadata + { + mail_uid: mail.message_id, + from_address: mail.from, + to_address: mail.to, + mail_key: mail_key, + references: Array(mail.references), + delivered_to: delivered_to.map(&:value), + envelope_to: envelope_to.map(&:value), + x_envelope_to: x_envelope_to.map(&:value) + } + end + private - def find_handler(mail) - mail_key = extract_mail_key(mail) + def handler + strong_memoize(:handler) { find_handler } + end + + def find_handler Handler.for(mail, mail_key) end + def mail + strong_memoize(:mail) { build_mail } + end + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, @@ -40,22 +58,24 @@ module Gitlab raise EmailUnparsableError, e end - def extract_mail_key(mail) - key_from_to_header(mail) || key_from_additional_headers(mail) + def mail_key + strong_memoize(:mail_key) do + key_from_to_header || key_from_additional_headers + end end - def key_from_to_header(mail) + def key_from_to_header mail.to.find do |address| key = Gitlab::IncomingEmail.key_from_address(address) break key if key end end - def key_from_additional_headers(mail) - find_key_from_references(mail) || - find_key_from_delivered_to_header(mail) || - find_key_from_envelope_to_header(mail) || - find_key_from_x_envelope_to_header(mail) + def key_from_additional_headers + find_key_from_references || + find_key_from_delivered_to_header || + find_key_from_envelope_to_header || + find_key_from_x_envelope_to_header end def ensure_references_array(references) @@ -71,41 +91,53 @@ module Gitlab end end - def find_key_from_references(mail) + def find_key_from_references ensure_references_array(mail.references).find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end end - def find_key_from_delivered_to_header(mail) - Array(mail[:delivered_to]).find do |header| + def delivered_to + Array(mail[:delivered_to]) + end + + def envelope_to + Array(mail[:envelope_to]) + end + + def x_envelope_to + Array(mail[:x_envelope_to]) + end + + def find_key_from_delivered_to_header + delivered_to.find do |header| key = Gitlab::IncomingEmail.key_from_address(header.value) break key if key end end - def find_key_from_envelope_to_header(mail) - Array(mail[:envelope_to]).find do |header| + def find_key_from_envelope_to_header + envelope_to.find do |header| key = Gitlab::IncomingEmail.key_from_address(header.value) break key if key end end - def find_key_from_x_envelope_to_header(mail) - Array(mail[:x_envelope_to]).find do |header| + def find_key_from_x_envelope_to_header + x_envelope_to.find do |header| key = Gitlab::IncomingEmail.key_from_address(header.value) break key if key end end - def ignore_auto_reply!(mail) - if auto_submitted?(mail) || auto_replied?(mail) + def ignore_auto_reply! + if auto_submitted? || auto_replied? raise AutoGeneratedEmailError end end - def auto_submitted?(mail) + def auto_submitted? # Mail::Header#[] is case-insensitive auto_submitted = mail.header['Auto-Submitted']&.value @@ -114,7 +146,7 @@ module Gitlab auto_submitted && auto_submitted != 'no' end - def auto_replied?(mail) + def auto_replied? autoreply = mail.header['X-Autoreply']&.value autoreply && autoreply == 'yes' diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index dc44e9d7481..7579f3d8680 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -68,7 +68,7 @@ module Gitlab else object.body.to_s end - rescue + rescue StandardError nil end end diff --git a/lib/gitlab/email/service_desk_receiver.rb b/lib/gitlab/email/service_desk_receiver.rb index 1ee5c10097b..133c4ee4b45 100644 --- a/lib/gitlab/email/service_desk_receiver.rb +++ b/lib/gitlab/email/service_desk_receiver.rb @@ -5,14 +5,19 @@ module Gitlab class ServiceDeskReceiver < Receiver private - def find_handler(mail) - key = service_desk_key(mail) - return unless key + def find_handler + return unless service_desk_key - Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: key) + Gitlab::Email::Handler::ServiceDeskHandler.new(mail, nil, service_desk_key: service_desk_key) end - def service_desk_key(mail) + def service_desk_key + strong_memoize(:service_desk_key) do + find_service_desk_key + end + end + + def find_service_desk_key mail.to.find do |address| key = ::Gitlab::ServiceDeskEmail.key_from_address(address) break key if key diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 7b79de00c66..8ee53d0de28 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -20,7 +20,7 @@ module Gitlab return message if message.valid_encoding? # return message if message type is binary - detect = CharlockHolmes::EncodingDetector.detect(message) + detect = detect_encoding(message) return message.force_encoding("BINARY") if detect_binary?(message, detect) if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD @@ -37,16 +37,30 @@ module Gitlab "--broken encoding: #{encoding}" end + def detect_encoding(data, limit: CharlockHolmes::EncodingDetector::DEFAULT_BINARY_SCAN_LEN, cache_key: nil) + return if data.nil? + + if Feature.enabled?(:cached_encoding_detection, type: :development, default_enabled: :yaml) + return CharlockHolmes::EncodingDetector.new(limit).detect(data) unless cache_key.present? + + Rails.cache.fetch([:detect_binary, CharlockHolmes::VERSION, cache_key], expires_in: 1.week) do + CharlockHolmes::EncodingDetector.new(limit).detect(data) + end + else + CharlockHolmes::EncodingDetector.new(limit).detect(data) + end + end + def detect_binary?(data, detect = nil) - detect ||= CharlockHolmes::EncodingDetector.detect(data) + detect ||= detect_encoding(data) detect && detect[:type] == :binary && detect[:confidence] == 100 end - def detect_libgit2_binary?(data) - # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks - # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), - # which is what we use below to keep a consistent behavior. - detect = CharlockHolmes::EncodingDetector.new(8000).detect(data) + # EncodingDetector checks the first 1024 * 1024 bytes for NUL byte, libgit2 checks + # only the first 8000 (https://github.com/libgit2/libgit2/blob/2ed855a9e8f9af211e7274021c2264e600c0f86b/src/filter.h#L15), + # which is what we use below to keep a consistent behavior. + def detect_libgit2_binary?(data, cache_key: nil) + detect = detect_encoding(data, limit: 8000, cache_key: cache_key) detect && detect[:type] == :binary end @@ -54,7 +68,8 @@ module Gitlab message = force_encode_utf8(message) return message if message.valid_encoding? - detect = CharlockHolmes::EncodingDetector.detect(message) + detect = detect_encoding(message) + if detect && detect[:encoding] begin CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index fe49af3ab33..6b64281e631 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -65,7 +65,7 @@ module Gitlab contents = deserialize(read) - raise InvalidConfigError.new unless contents.is_a?(Hash) + raise InvalidConfigError unless contents.is_a?(Hash) @config = contents.deep_symbolize_keys end @@ -115,7 +115,7 @@ module Gitlab end def handle_missing_key! - raise MissingKeyError.new if @key.nil? + raise MissingKeyError if @key.nil? end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 47d361fb95c..38ac5d9af74 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -31,9 +31,6 @@ module Gitlab # Sanitize fields based on those sanitized from Rails. config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) - config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor - config.processors << ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor - config.processors << ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] diff --git a/lib/gitlab/error_tracking/context_payload_generator.rb b/lib/gitlab/error_tracking/context_payload_generator.rb index c99283b3d20..3d0a707608f 100644 --- a/lib/gitlab/error_tracking/context_payload_generator.rb +++ b/lib/gitlab/error_tracking/context_payload_generator.rb @@ -49,7 +49,7 @@ module Gitlab # Static tags that are set on application start def extra_tags_from_env Gitlab::Json.parse(ENV.fetch('GITLAB_SENTRY_EXTRA_TAGS', '{}')).to_hash - rescue => e + rescue StandardError => e Gitlab::AppLogger.debug("GITLAB_SENTRY_EXTRA_TAGS could not be parsed as JSON: #{e.class.name}: #{e.message}") {} diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb index 758f6aa11d7..9559d6807da 100644 --- a/lib/gitlab/error_tracking/processor/context_payload_processor.rb +++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb @@ -3,21 +3,12 @@ module Gitlab module ErrorTracking module Processor - class ContextPayloadProcessor < ::Raven::Processor + module ContextPayloadProcessor # This processor is added to inject application context into Sentry # events generated by Sentry built-in integrations. When the # integrations are re-implemented and use Gitlab::ErrorTracking, this # processor should be removed. - def process(payload) - return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - - context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}) - payload.deep_merge!(context_payload) - end - def self.call(event) - return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}).each do |key, value| event.public_send(key).deep_merge!(value) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index 419098dbd09..e2a9192806f 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -3,22 +3,11 @@ module Gitlab module ErrorTracking module Processor - class GrpcErrorProcessor < ::Raven::Processor + module GrpcErrorProcessor DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') - def process(payload) - return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - - self.class.process_first_exception_value(payload) - self.class.process_custom_fingerprint(payload) - - payload - end - class << self def call(event) - return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - process_first_exception_value(event) process_custom_fingerprint(event) @@ -27,8 +16,9 @@ module Gitlab # Sentry can report multiple exceptions in an event. Sanitize # only the first one since that's what is used for grouping. - def process_first_exception_value(event_or_payload) - exceptions = exceptions(event_or_payload) + def process_first_exception_value(event) + # Better in new version, will be event.exception.values + exceptions = event.instance_variable_get(:@interfaces)[:exception]&.values return unless exceptions.is_a?(Array) @@ -36,18 +26,21 @@ module Gitlab return unless valid_exception?(exception) - exception_type, raw_message = type_and_value(exception) + raw_message = exception.value - return unless exception_type&.start_with?('GRPC::') + return unless exception.type&.start_with?('GRPC::') return unless raw_message.present? message, debug_str = split_debug_error_string(raw_message) - set_new_values!(event_or_payload, exception, message, debug_str) + # Worse in new version, no setter! Have to poke at the + # instance variable + exception.value = message if message + event.extra[:grpc_debug_error_string] = debug_str if debug_str end def process_custom_fingerprint(event) - fingerprint = fingerprint(event) + fingerprint = event.fingerprint return event unless custom_grpc_fingerprint?(fingerprint) @@ -71,61 +64,14 @@ module Gitlab [match[1], match[2]] end - # The below methods can be removed once we remove the - # sentry_processors_before_send feature flag, and we can - # assume we always have an Event object - def exceptions(event_or_payload) - case event_or_payload - when Raven::Event - # Better in new version, will be event_or_payload.exception.values - event_or_payload.instance_variable_get(:@interfaces)[:exception]&.values - when Hash - event_or_payload.dig(:exception, :values) - end - end - def valid_exception?(exception) case exception when Raven::SingleExceptionInterface exception&.value - when Hash - true else false end end - - def type_and_value(exception) - case exception - when Raven::SingleExceptionInterface - [exception.type, exception.value] - when Hash - exception.values_at(:type, :value) - end - end - - def set_new_values!(event_or_payload, exception, message, debug_str) - case event_or_payload - when Raven::Event - # Worse in new version, no setter! Have to poke at the - # instance variable - exception.value = message if message - event_or_payload.extra[:grpc_debug_error_string] = debug_str if debug_str - when Hash - exception[:value] = message if message - extra = event_or_payload[:extra] || {} - extra[:grpc_debug_error_string] = debug_str if debug_str - end - end - - def fingerprint(event_or_payload) - case event_or_payload - when Raven::Event - event_or_payload.fingerprint - when Hash - event_or_payload[:fingerprint] - end - end end end end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index 93310745ece..0d2f673d73c 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -5,7 +5,7 @@ require 'set' module Gitlab module ErrorTracking module Processor - class SidekiqProcessor < ::Raven::Processor + module SidekiqProcessor FILTERED_STRING = '[FILTERED]' class << self @@ -29,7 +29,7 @@ module Gitlab @permitted_arguments_for_worker[klass] ||= begin klass.constantize&.loggable_arguments&.to_set - rescue + rescue StandardError Set.new end end @@ -42,8 +42,6 @@ module Gitlab end def call(event) - return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - sidekiq = event&.extra&.dig(:sidekiq) return event unless sidekiq @@ -64,29 +62,6 @@ module Gitlab event end end - - def process(value, key = nil) - return value if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - - sidekiq = value.dig(:extra, :sidekiq) - - return value unless sidekiq - - sidekiq = sidekiq.deep_dup - sidekiq.delete(:jobstr) - - # 'args' in this hash => from Gitlab::ErrorTracking.track_* - # 'args' in :job => from default error handler - job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] - - if job_holder['args'] - job_holder['args'] = self.class.filter_arguments(job_holder['args'], job_holder['class']).to_a - end - - value[:extra][:sidekiq] = sidekiq - - value - end end end end diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index f1737f0ce5a..2b8639b9411 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -12,6 +12,11 @@ module Gitlab %r(\Apipelines/id/\d+\z), 'pipelines_graph', 'continuous_integration' + ], + [ + %r(\Apipelines/sha/\w{7,40}\z), + 'ci_editor', + 'pipeline_authoring' ] ].map(&method(:build_route)).freeze diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb index 08c20e30a48..fba4b9e433a 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/restful.rb @@ -109,4 +109,4 @@ module Gitlab end end -Gitlab::EtagCaching::Router::Restful.prepend_if_ee('EE::Gitlab::EtagCaching::Router::Restful') +Gitlab::EtagCaching::Router::Restful.prepend_mod_with('Gitlab::EtagCaching::Router::Restful') diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index ef0236f8275..6749bd6ca60 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -113,4 +113,4 @@ module Gitlab end end -Gitlab::ExclusiveLease.prepend_if_ee('EE::Gitlab::ExclusiveLease') +Gitlab::ExclusiveLease.prepend_mod_with('Gitlab::ExclusiveLease') diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 145bb6d7b8f..e4233b8a935 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,10 +34,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - invite_members_version_b: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', - use_backwards_compatible_subject_index: true - }, invite_members_empty_group_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', use_backwards_compatible_subject_index: true @@ -55,10 +51,6 @@ module Gitlab trial_during_signup: { tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, - ci_syntax_templates_b: { - tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', - rollout_strategy: :user - }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, @@ -154,7 +146,7 @@ module Gitlab elsif subject.respond_to?(:to_s) subject.to_s else - raise ArgumentError.new('Subject must respond to `to_global_id` or `to_s`') + raise ArgumentError, 'Subject must respond to `to_global_id` or `to_s`' end end end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 248abfeada5..e53689eb89b 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -19,13 +19,18 @@ module Gitlab end def set_experimentation_subject_id_cookie - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } + if Gitlab.dev_env_or_com? + return if cookies[:experimentation_subject_id].present? + + cookies.permanent.signed[:experimentation_subject_id] = { + value: SecureRandom.uuid, + secure: ::Gitlab.config.gitlab.https, + httponly: true + } + else + # We set the cookie before, although experiments are not conducted on self managed instances. + cookies.delete(:experimentation_subject_id) + end end def push_frontend_experiment(experiment_key, subject: nil) diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb index 582051010d3..43f7f042592 100644 --- a/lib/gitlab/external_authorization/client.rb +++ b/lib/gitlab/external_authorization/client.rb @@ -24,7 +24,7 @@ module Gitlab ) ::Gitlab::ExternalAuthorization::Response.new(response) rescue *Gitlab::HTTP::HTTP_ERRORS => e - raise ::Gitlab::ExternalAuthorization::RequestFailed.new(e) + raise ::Gitlab::ExternalAuthorization::RequestFailed, e end private diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index 71d2b2396f8..211c0967f89 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -33,4 +33,4 @@ module Gitlab end end -Gitlab::FakeApplicationSettings.prepend_if_ee('EE::Gitlab::FakeApplicationSettings') +Gitlab::FakeApplicationSettings.prepend_mod_with('Gitlab::FakeApplicationSettings') diff --git a/lib/gitlab/faraday/error_callback.rb b/lib/gitlab/faraday/error_callback.rb index f99be5b4d04..9b436c3a08e 100644 --- a/lib/gitlab/faraday/error_callback.rb +++ b/lib/gitlab/faraday/error_callback.rb @@ -28,7 +28,7 @@ module Gitlab def call(env) @app.call(env) - rescue => e + rescue StandardError => e @options.callback&.call(env, e) raise diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index ce1370bab0f..721518c6fcc 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -61,4 +61,4 @@ module Gitlab end end -Gitlab::Favicon.prepend_if_ee('EE::Gitlab::Favicon') +Gitlab::Favicon.prepend_mod_with('Gitlab::Favicon') diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb index 55eba2858fb..e398a3f9585 100644 --- a/lib/gitlab/file_hook.rb +++ b/lib/gitlab/file_hook.rb @@ -28,7 +28,7 @@ module Gitlab exit_status = result.status&.exitstatus [exit_status == 0, result.stderr] - rescue => e + rescue StandardError => e [false, e.message] end end diff --git a/lib/gitlab/fogbugz_import/repository.rb b/lib/gitlab/fogbugz_import/repository.rb index b958dcf6cbf..4a5152021b4 100644 --- a/lib/gitlab/fogbugz_import/repository.rb +++ b/lib/gitlab/fogbugz_import/repository.rb @@ -26,7 +26,7 @@ module Gitlab end def path - safe_name.gsub(/[\s]/, '_') + safe_name.gsub(/\s/, '_') end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 5d91eb605e8..1c8e55ecf50 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -110,8 +110,8 @@ module Gitlab end end - def binary?(data) - EncodingHelper.detect_libgit2_binary?(data) + def binary?(data, cache_key: nil) + EncodingHelper.detect_libgit2_binary?(data, cache_key: cache_key) end def size_could_be_lfs?(size) diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 9447cfa0fb6..fbe52db9c0b 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -28,6 +28,10 @@ module Gitlab def state active? ? :active : :stale end + + def cache_key + "branch:" + Digest::SHA1.hexdigest([name, target, dereferenced_target&.sha].join(':')) + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 51baed32935..a863b952390 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -263,7 +263,7 @@ module Gitlab def has_zero_stats? stats.total == 0 - rescue + rescue StandardError true end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 26e82643a4c..751184b23df 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -20,9 +20,9 @@ module Gitlab gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e - raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message rescue GRPC::BadStatus => e - raise Gitlab::Git::CommandError.new(e) + raise Gitlab::Git::CommandError, e end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 3361cee733b..102fe60f2cb 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -89,9 +89,9 @@ module Gitlab def root_ref gitaly_ref_client.default_branch_name rescue GRPC::NotFound => e - raise NoRepository.new(e.message) + raise NoRepository, e.message rescue GRPC::Unknown => e - raise Gitlab::Git::CommandError.new(e.message) + raise Gitlab::Git::CommandError, e.message end def exists? @@ -348,7 +348,7 @@ module Gitlab limit = options[:limit] if limit == 0 || !limit.is_a?(Integer) - raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") + raise ArgumentError, "invalid Repository#log limit: #{limit.inspect}" end wrapped_gitaly_errors do @@ -414,7 +414,7 @@ module Gitlab end end rescue ArgumentError => e - raise Gitlab::Git::Repository::GitError.new(e) + raise Gitlab::Git::Repository::GitError, e end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -700,11 +700,11 @@ module Gitlab end end - def find_remote_root_ref(remote_name) - return unless remote_name.present? + def find_remote_root_ref(remote_name, remote_url, authorization = nil) + return unless remote_name.present? && remote_url.present? wrapped_gitaly_errors do - gitaly_remote_client.find_remote_root_ref(remote_name) + gitaly_remote_client.find_remote_root_ref(remote_name, remote_url, authorization) end end @@ -836,7 +836,7 @@ module Gitlab def fsck msg, status = gitaly_repository_client.fsck - raise GitError.new("Could not fsck repository: #{msg}") unless status == 0 + raise GitError, "Could not fsck repository: #{msg}" unless status == 0 end def create_from_bundle(bundle_path) diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index 8679d977773..ea10b4e7cd8 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -31,7 +31,7 @@ module Gitlab def rugged @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) rescue ::Rugged::RepositoryError, ::Rugged::OSError - raise ::Gitlab::Git::Repository::NoRepository.new('no repository for such path') + raise ::Gitlab::Git::Repository::NoRepository, 'no repository for such path' end def cleanup diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 75d6b949874..5616b61de07 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -73,12 +73,6 @@ module Gitlab end end - def delete_page(page_path, commit_details) - wrapped_gitaly_errors do - gitaly_delete_page(page_path, commit_details) - end - end - def update_page(page_path, title, format, content, commit_details) wrapped_gitaly_errors do gitaly_update_page(page_path, title, format, content, commit_details) @@ -102,22 +96,6 @@ module Gitlab end end - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def page_versions(page_path, options = {}) - versions = wrapped_gitaly_errors do - gitaly_wiki_client.page_versions(page_path, options) - end - - # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 - # per page, but also fetches 20 if `limit` or `per_page` < 20. - # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || DEFAULT_PAGINATION - versions[0..slice_bound] - end - def count_page_versions(page_path) @repository.count_commits(ref: 'HEAD', path: page_path) end @@ -140,10 +118,6 @@ module Gitlab gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) end - def gitaly_delete_page(page_path, commit_details) - gitaly_wiki_client.delete_page(page_path, commit_details) - end - def gitaly_find_page(title:, version: nil, dir: nil) return unless title.present? diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb index 2009683d32c..1d34f3c8eb2 100644 --- a/lib/gitlab/git/wraps_gitaly_errors.rb +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -6,13 +6,13 @@ module Gitlab def wrapped_gitaly_errors(&block) yield block rescue GRPC::NotFound => e - raise Gitlab::Git::Repository::NoRepository.new(e) + raise Gitlab::Git::Repository::NoRepository, e rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) + raise ArgumentError, e rescue GRPC::DeadlineExceeded => e - raise Gitlab::Git::CommandTimedOut.new(e) + raise Gitlab::Git::CommandTimedOut, e rescue GRPC::BadStatus => e - raise Gitlab::Git::CommandError.new(e) + raise Gitlab::Git::CommandError, e end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 31e4755192e..b5e7220889e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -22,7 +22,7 @@ module Gitlab auth_download: 'You are not allowed to download code.', deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', - project_not_found: 'The project you were looking for could not be found.', + project_not_found: "The project you were looking for could not be found or you don't have permission to view it.", command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', @@ -538,4 +538,4 @@ module Gitlab end end -Gitlab::GitAccess.prepend_if_ee('EE::Gitlab::GitAccess') +Gitlab::GitAccess.prepend_mod_with('Gitlab::GitAccess') diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb index 6bea9fe53b3..bf89c01305a 100644 --- a/lib/gitlab/git_access_design.rb +++ b/lib/gitlab/git_access_design.rb @@ -32,4 +32,4 @@ module Gitlab end end -Gitlab::GitAccessDesign.prepend_if_ee('EE::Gitlab::GitAccessDesign') +Gitlab::GitAccessDesign.prepend_mod_with('Gitlab::GitAccessDesign') diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 88a75f72840..9a431dc7088 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -141,4 +141,4 @@ module Gitlab end end -Gitlab::GitAccessSnippet.prepend_if_ee('EE::Gitlab::GitAccessSnippet') +Gitlab::GitAccessSnippet.prepend_mod_with('Gitlab::GitAccessSnippet') diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3011b794b8f..0963eb6b72a 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -48,4 +48,4 @@ module Gitlab end end -Gitlab::GitAccessWiki.prepend_if_ee('EE::Gitlab::GitAccessWiki') +Gitlab::GitAccessWiki.prepend_mod_with('Gitlab::GitAccessWiki') diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 19a473e4785..affd3986381 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -115,7 +115,7 @@ module Gitlab # necessary graph walk to detect only new LFS pointers and instead scan # through all quarantined objects. git_env = ::Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository) - if Feature.enabled?(:lfs_integrity_inspect_quarantined_objects, @project, default_enabled: :yaml) && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + if git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? repository = @gitaly_repo.dup repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb index f860d8ce517..2f6d146b5c4 100644 --- a/lib/gitlab/gitaly_client/blobs_stitcher.rb +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -19,9 +19,9 @@ module Gitlab yield new_blob(current_blob_data) if current_blob_data current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode) - current_blob_data[:data] = msg.data.dup + current_blob_data[:data_parts] = [msg.data] else - current_blob_data[:data] << msg.data + current_blob_data[:data_parts] << msg.data end end @@ -31,6 +31,8 @@ module Gitlab private def new_blob(blob_data) + data = blob_data[:data_parts].join + Gitlab::Git::Blob.new( id: blob_data[:oid], mode: blob_data[:mode].to_s(8), @@ -38,8 +40,8 @@ module Gitlab path: blob_data[:path], size: blob_data[:size], commit_id: blob_data[:revision], - data: blob_data[:data], - binary: Gitlab::Git::Blob.binary?(blob_data[:data]) + data: data, + binary: Gitlab::Git::Blob.binary?(data, cache_key: blob_data[:oid]) ) end end diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb index 4bb184bee2f..3fe3702cfe1 100644 --- a/lib/gitlab/gitaly_client/call.rb +++ b/lib/gitlab/gitaly_client/call.rb @@ -30,7 +30,7 @@ module Gitlab store_timings response end - rescue => err + rescue StandardError => err store_timings raise err end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 5ce1b1f0c87..fd794acb4dd 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -59,7 +59,7 @@ module Gitlab :user_create_branch, request, timeout: GitalyClient.long_timeout) if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error) + raise Gitlab::Git::PreReceiveError, response.pre_receive_error end branch = response.branch @@ -159,7 +159,7 @@ module Gitlab branch_update = second_response.branch_update return if branch_update.nil? - raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? + raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) ensure diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 97b6813c080..ac2db99ee01 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -292,7 +292,7 @@ module Gitlab end def invalid_ref!(message) - raise Gitlab::Git::Repository::InvalidRef.new(message) + raise Gitlab::Git::Repository::InvalidRef, message end end end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 06aaf460751..04dd394a2bd 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -43,11 +43,20 @@ module Gitlab GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result end - def find_remote_root_ref(remote_name) - request = Gitaly::FindRemoteRootRefRequest.new( - repository: @gitaly_repo, - remote: remote_name - ) + # The remote_name parameter is deprecated and will be removed soon. + def find_remote_root_ref(remote_name, remote_url, authorization) + request = if Feature.enabled?(:find_remote_root_refs_inmemory, default_enabled: :yaml) + Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote_url: remote_url, + http_authorization_header: authorization + ) + else + Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote: remote_name + ) + end response = GitalyClient.call(@storage, :remote_service, :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index a93f4071efc..d2dbd456180 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -319,7 +319,7 @@ module Gitlab response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) response.checksum.presence rescue GRPC::DataLoss => e - raise Gitlab::Git::Repository::InvalidRepository.new(e) + raise Gitlab::Git::Repository::InvalidRepository, e end def raw_changes_between(from, to) diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index dd9e3d5d28b..f66dc3010ea 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -34,7 +34,7 @@ module Gitlab return false if rugged_enabled? !temporarily_allowed?(ALLOW_KEY) - rescue + rescue StandardError false # Err on the side of caution, don't break gitlab for people end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index fecc2b7023d..3613cd01122 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -64,16 +64,6 @@ module Gitlab GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) end - def delete_page(page_path, commit_details) - request = Gitaly::WikiDeletePageRequest.new( - repository: @gitaly_repo, - page_path: encode_binary(page_path), - commit_details: gitaly_commit_details(commit_details) - ) - - GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request, timeout: GitalyClient.medium_timeout) - end - def find_page(title:, version: nil, dir: nil) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, @@ -129,30 +119,6 @@ module Gitlab pages end - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def page_versions(page_path, options) - request = Gitaly::WikiGetPageVersionsRequest.new( - repository: @gitaly_repo, - page_path: encode_binary(page_path), - page: options[:page] || 1, - per_page: options[:per_page] || Gitlab::Git::Wiki::DEFAULT_PAGINATION - ) - - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout) - - versions = [] - stream.each do |message| - message.versions.each do |version| - versions << new_wiki_page_version(version) - end - end - - versions - end - private # If a block is given and the yielded value is truthy, iteration will be diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 328f1f742c5..138716b1b53 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -70,7 +70,7 @@ module Gitlab end def pull_request_reviews(repo_name, iid) - with_rate_limit { octokit.pull_request_reviews(repo_name, iid) } + each_object(:pull_request_reviews, repo_name, iid) end # Returns the details of a GitHub repository. diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 53b17f77ccd..d2f5af63621 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -21,8 +21,7 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = - MarkdownText.format(note.note, note.author, author_found) + note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { noteable_type: 'MergeRequest', diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 41f179d275b..ae9996d81ef 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -21,8 +21,7 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = - MarkdownText.format(note.note, note.author, author_found) + note_body = MarkdownText.format(note.note, note.author, author_found) attributes = { noteable_type: note.noteable_type, diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index f09e0bd9806..3c17ea1195e 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -44,8 +44,7 @@ module Gitlab def create_merge_request author_id, author_found = user_finder.author_id_for(pull_request) - description = MarkdownText - .format(pull_request.description, pull_request.author, author_found) + description = MarkdownText.format(pull_request.description, pull_request.author, author_found) attributes = { iid: pull_request.iid, diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index 9f495913897..f476ee13392 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -36,12 +36,12 @@ module Gitlab def add_complementary_review_note!(author_id) return if review.note.empty? && !review.approval? - note = "*Created by %{login}*\n\n%{note}" % { - note: review_note_content, - login: review.author.login - } + note_body = MarkdownText.format( + review_note_content, + review.author + ) - add_note!(author_id, note) + add_note!(author_id, note_body) end def review_note_content diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb index 466288fde4c..94472cd341e 100644 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb @@ -22,14 +22,18 @@ module Gitlab :pull_requests_merged_by end - def id_for_already_imported_cache(pr) - pr.number + def id_for_already_imported_cache(merge_request) + merge_request.id end def each_object_to_import project.merge_requests.with_state(:merged).find_each do |merge_request| + next if already_imported?(merge_request) + pull_request = client.pull_request(project.import_source, merge_request.iid) yield(pull_request) + + mark_as_imported(merge_request) end end end diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index 6d1b588f0e0..827027203ff 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -22,17 +22,22 @@ module Gitlab :pull_request_reviews end - def id_for_already_imported_cache(review) - review.github_id + def id_for_already_imported_cache(merge_request) + merge_request.id end def each_object_to_import project.merge_requests.find_each do |merge_request| - reviews = client.pull_request_reviews(project.import_source, merge_request.iid) - reviews.each do |review| - review.merge_request_id = merge_request.id - yield(review) - end + next if already_imported?(merge_request) + + client + .pull_request_reviews(project.import_source, merge_request.iid) + .each do |review| + review.merge_request_id = merge_request.id + yield(review) + end + + mark_as_imported(merge_request) end end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index b25c4f7becf..e5f4dabe42d 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -3,7 +3,7 @@ module Gitlab module GithubImport class MarkdownText - attr_reader :text, :author, :exists + include Gitlab::EncodingHelper def self.format(*args) new(*args).to_s @@ -19,10 +19,19 @@ module Gitlab end def to_s - if exists - text - else + # Gitlab::EncodingHelper#clean remove `null` chars from the string + clean(format) + end + + private + + attr_reader :text, :author, :exists + + def format + if author&.login.present? && !exists "*Created by: #{author.login}*\n\n#{text}" + else + text end end end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 1b4750da868..2429fa4de1d 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -40,4 +40,4 @@ module Gitlab end end -Gitlab::GithubImport::ParallelImporter.prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter') +Gitlab::GithubImport::ParallelImporter.prepend_mod_with('Gitlab::GithubImport::ParallelImporter') diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 51859010ec3..92f9e8a646d 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -48,7 +48,7 @@ module Gitlab info(project.id, message: "importer finished") retval - rescue => e + rescue StandardError => e error(project.id, e) raise e diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index f3071b3e2b3..0e04b5ad57f 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -25,6 +25,7 @@ module Gitlab hash = { iid: issue.number, + github_id: issue.number, title: issue.title, description: issue.body, milestone_number: issue.milestone&.number, diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index a4606173f49..41723759645 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -13,7 +13,12 @@ module Gitlab # Builds a lfs_object def self.from_api_response(lfs_object) - new({ oid: lfs_object.oid, link: lfs_object.link, size: lfs_object.size }) + new( + oid: lfs_object.oid, + link: lfs_object.link, + size: lfs_object.size, + github_id: lfs_object.oid + ) end # Builds a new lfs_object using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index be192762e05..e4f54fcc833 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -25,6 +25,7 @@ module Gitlab hash = { iid: pr.number, + github_id: pr.number, title: pr.title, description: pr.body, source_branch: pr.head.ref, diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index e00dcfca33d..d97b90b6291 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -15,7 +15,11 @@ module Gitlab # # user - An instance of `Sawyer::Resource` containing the user details. def self.from_api_response(user) - new(id: user.id, login: user.login) + new( + id: user.id, + github_id: user.id, + login: user.login + ) end # Builds a user using a Hash that was built from a JSON payload. diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 34d1231b9a5..8d584415202 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -63,7 +63,7 @@ module Gitlab # # user - An instance of `Gitlab::GithubImport::Representation::User`. def user_id_for(user) - find(user.id, user.login) + find(user.id, user.login) if user.present? end # Returns the GitLab ID for the given GitHub ID or username. diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 4b1f4fcc2a2..05278b2dd35 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -81,4 +81,4 @@ module Gitlab end end -Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType') +Gitlab::GlRepository::RepoType.prepend_mod_with('Gitlab::GlRepository::RepoType') diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index 31b7a198b92..1b625a3a514 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -69,13 +69,13 @@ module Gitlab # Error messages are based on the responses of proxy.golang.org # Verify that the SHA fragment references a commit - raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit + raise ArgumentError, 'invalid pseudo-version: unknown commit' unless commit # Require the SHA fragment to be 12 characters long - raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 + raise ArgumentError, 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 # Require the timestamp to match that of the commit - raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp + raise ArgumentError, 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp commit end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 08c17058fcb..1fd210c521e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -80,4 +80,4 @@ module Gitlab end end -Gitlab::GonHelper.prepend_if_ee('EE::Gitlab::GonHelper') +Gitlab::GonHelper.prepend_mod_with('Gitlab::GonHelper') diff --git a/lib/gitlab/gpg.rb b/lib/gitlab/gpg.rb index b1494cf8cf2..3d9b06855ff 100644 --- a/lib/gitlab/gpg.rb +++ b/lib/gitlab/gpg.rb @@ -133,7 +133,7 @@ module Gitlab Retriable.retriable(max_elapsed_time: cleanup_time, base_interval: 0.1, tries: 15) do FileUtils.remove_entry(tmp_dir) if File.exist?(tmp_dir) end - rescue => e + rescue StandardError => e raise CleanupError, e end diff --git a/lib/gitlab/grape_logging/loggers/route_logger.rb b/lib/gitlab/grape_logging/loggers/route_logger.rb index f3146b4dfd9..7cbd2340e85 100644 --- a/lib/gitlab/grape_logging/loggers/route_logger.rb +++ b/lib/gitlab/grape_logging/loggers/route_logger.rb @@ -13,7 +13,7 @@ module Gitlab return {} unless route { route: route } - rescue + rescue StandardError # endpoint.route calls env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response # so we're rescuing exceptions and bailing out diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index e0176e2d6e0..8b73eeb4e52 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -41,7 +41,7 @@ module Gitlab parts = [ "#{deprecated_in(format: :markdown)}.", reason_text, - replacement.then { |r| "Use: `#{r}`." if r } + replacement.then { |r| "Use: [`#{r}`](##{r.downcase.tr('.', '')})." if r } ].compact case context diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index f4173e26224..b598b605141 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -5,11 +5,52 @@ return if Rails.env.production? module Gitlab module Graphql module Docs + # We assume a few things about the schema. We use the graphql-ruby gem, which enforces: + # - All mutations have a single input field named 'input' + # - All mutations have a payload type, named after themselves + # - All mutations have an input type, named after themselves + # If these things change, then some of this code will break. Such places + # are guarded with an assertion that our assumptions are not violated. + ViolatedAssumption = Class.new(StandardError) + + SUGGESTED_ACTION = <<~MSG + We expect it to be impossible to violate our assumptions about + how mutation arguments work. + + If that is not the case, then something has probably changed in the + way we generate our schema, perhaps in the library we use: graphql-ruby + + Please ask for help in the #f_graphql or #backend channels. + MSG + + CONNECTION_ARGS = %w[after before first last].to_set + + FIELD_HEADER = <<~MD + #### Fields + + | Name | Type | Description | + | ---- | ---- | ----------- | + MD + + ARG_HEADER = <<~MD + # Arguments + + | Name | Type | Description | + | ---- | ---- | ----------- | + MD + + CONNECTION_NOTE = <<~MD + This field returns a [connection](#connections). It accepts the + four standard [pagination arguments](#connection-pagination-arguments): + `before: String`, `after: String`, `first: Int`, `last: Int`. + MD + # Helper with functions to be used by HAML templates # This includes graphql-docs gem helpers class. # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb module Helper include GraphQLDocs::Helpers + include Gitlab::Utils::StrongMemoize def auto_generated_comment <<-MD.strip_heredoc @@ -30,44 +71,52 @@ module Gitlab # Template methods: # Methods that return chunks of Markdown for insertion into the document - def render_name_and_description(object, owner: nil, level: 3) - content = [] + def render_full_field(field, heading_level: 3, owner: nil) + conn = connection?(field) + args = field[:arguments].reject { |arg| conn && CONNECTION_ARGS.include?(arg[:name]) } + arg_owner = [owner, field[:name]] + + chunks = [ + render_name_and_description(field, level: heading_level, owner: owner), + render_return_type(field), + render_input_type(field), + render_connection_note(field), + render_argument_table(heading_level, args, arg_owner), + render_return_fields(field, owner: owner) + ] + + join(:block, chunks) + end - content << "#{'#' * level} `#{object[:name]}`" + def render_argument_table(level, args, owner) + arg_header = ('#' * level) + ARG_HEADER + render_field_table(arg_header, args, owner) + end - if object[:description].present? - desc = object[:description].strip - desc += '.' unless desc.ends_with?('.') - end + def render_name_and_description(object, owner: nil, level: 3) + content = [] - if object[:is_deprecated] - owner = Array.wrap(owner) - deprecation = schema_deprecation(owner, object[:name]) - content << (deprecation&.original_description || desc) - content << render_deprecation(object, owner, :block) - else - content << desc - end + heading = '#' * level + name = [owner, object[:name]].compact.join('.') - content.compact.join("\n\n") - end + content << "#{heading} `#{name}`" + content << render_description(object, owner, :block) - def render_return_type(query) - "Returns #{render_field_type(query[:type])}.\n" + join(:block, content) end - def sorted_by_name(objects) - return [] unless objects.present? + def render_object_fields(fields, owner:, level_bump: 0) + return if fields.blank? - objects.sort_by { |o| o[:name] } - end + (with_args, no_args) = fields.partition { |f| args?(f) } + type_name = owner[:name] if owner + header_prefix = '#' * level_bump + sections = [ + render_simple_fields(no_args, type_name, header_prefix), + render_fields_with_arguments(with_args, type_name, header_prefix) + ] - def render_field(field, owner) - render_row( - render_name(field, owner), - render_field_type(field[:type]), - render_description(field, owner, :inline) - ) + join(:block, sections) end def render_enum_value(enum, value) @@ -82,104 +131,302 @@ module Gitlab # Methods that return parts of the schema, or related information: - # We are ignoring connections and built in types for now, - # they should be added when queries are generated. - def objects - object_types = graphql_object_types.select do |object_type| - !object_type[:name]["__"] - end + def connection_object_types + objects.select { |t| t[:is_edge] || t[:is_connection] } + end + + def object_types + objects.reject { |t| t[:is_edge] || t[:is_connection] || t[:is_payload] } + end + + def interfaces + graphql_interface_types.map { |t| t.merge(fields: t[:fields] + t[:connections]) } + end - object_types.each do |type| - type[:fields] += type[:connections] + def fields_of(type_name) + graphql_operation_types + .find { |type| type[:name] == type_name } + .values_at(:fields, :connections) + .flatten + .then { |fields| sorted_by_name(fields) } + end + + # Place the arguments of the input types on the mutation itself. + # see: `#input_types` - this method must not call `#input_types` to avoid mutual recursion + def mutations + @mutations ||= sorted_by_name(graphql_mutation_types).map do |t| + inputs = t[:input_fields] + input = inputs.first + name = t[:name] + + assert!(inputs.one?, "Expected exactly 1 input field named #{name}. Found #{inputs.count} instead.") + assert!(input[:name] == 'input', "Expected the input of #{name} to be named 'input'") + + input_type_name = input[:type][:name] + input_type = graphql_input_object_types.find { |t| t[:name] == input_type_name } + assert!(input_type.present?, "Cannot find #{input_type_name} for #{name}.input") + + arguments = input_type[:input_fields] + seen_type!(input_type_name) + t.merge(arguments: arguments) end end - def queries - graphql_operation_types.find { |type| type[:name] == 'Query' }.to_h.values_at(:fields, :connections).flatten + # We assume that the mutations have been processed first, marking their + # inputs as `seen_type?` + def input_types + mutations # ensure that mutations have seen their inputs first + graphql_input_object_types.reject { |t| seen_type?(t[:name]) } end - # We ignore the built-in enum types. + # We ignore the built-in enum types, and sort values by name def enums - graphql_enum_types.select do |enum_type| - !enum_type[:name].in?(%w[__DirectiveLocation __TypeKind]) - end + graphql_enum_types + .reject { |type| type[:values].empty? } + .reject { |enum_type| enum_type[:name].start_with?('__') } + .map { |type| type.merge(values: sorted_by_name(type[:values])) } end private # DO NOT CALL THESE METHODS IN TEMPLATES # Template methods + def render_return_type(query) + return unless query[:type] # for example, mutations + + "Returns #{render_field_type(query[:type])}." + end + + def render_simple_fields(fields, type_name, header_prefix) + render_field_table(header_prefix + FIELD_HEADER, fields, type_name) + end + + def render_fields_with_arguments(fields, type_name, header_prefix) + return if fields.empty? + + level = 5 + header_prefix.length + sections = sorted_by_name(fields).map do |f| + render_full_field(f, heading_level: level, owner: type_name) + end + + <<~MD.chomp + #{header_prefix}#### Fields with arguments + + #{join(:block, sections)} + MD + end + + def render_field_table(header, fields, owner) + return if fields.empty? + + fields = sorted_by_name(fields) + header + join(:table, fields.map { |f| render_field(f, owner) }) + end + + def render_field(field, owner) + render_row( + render_name(field, owner), + render_field_type(field[:type]), + render_description(field, owner, :inline) + ) + end + + def render_return_fields(mutation, owner:) + fields = mutation[:return_fields] + return if fields.blank? + + name = owner.to_s + mutation[:name] + render_object_fields(fields, owner: { name: name }) + end + + def render_connection_note(field) + return unless connection?(field) + + CONNECTION_NOTE.chomp + end + def render_row(*values) "| #{values.map { |val| val.to_s.squish }.join(' | ')} |" end def render_name(object, owner = nil) rendered_name = "`#{object[:name]}`" - rendered_name += ' **{warning-solid}**' if object[:is_deprecated] - rendered_name + rendered_name += ' **{warning-solid}**' if deprecated?(object, owner) + + return rendered_name unless owner + + owner = Array.wrap(owner).join('') + id = (owner + object[:name]).downcase + + %(<a id="#{id}"></a>) + rendered_name end # Returns the object description. If the object has been deprecated, # the deprecation reason will be returned in place of the description. def render_description(object, owner = nil, context = :block) - owner = Array.wrap(owner) - return render_deprecation(object, owner, context) if object[:is_deprecated] - return if object[:description].blank? + if deprecated?(object, owner) + render_deprecation(object, owner, context) + else + render_description_of(object, owner, context) + end + end + + def deprecated?(object, owner) + return true if object[:is_deprecated] # only populated for fields, not arguments! + + key = [*Array.wrap(owner), object[:name]].join('.') + deprecations.key?(key) + end + + def render_description_of(object, owner, context = nil) + desc = if object[:is_edge] + base = object[:name].chomp('Edge') + "The edge type for [`#{base}`](##{base.downcase})." + elsif object[:is_connection] + base = object[:name].chomp('Connection') + "The connection type for [`#{base}`](##{base.downcase})." + else + object[:description]&.strip + end + + return if desc.blank? - desc = object[:description].strip desc += '.' unless desc.ends_with?('.') + see = doc_reference(object, owner) + desc += " #{see}" if see + desc += " (see [Connections](#connections))" if connection?(object) && context != :block desc end + def doc_reference(object, owner) + field = schema_field(owner, object[:name]) if owner + return unless field + + ref = field.try(:doc_reference) + return if ref.blank? + + parts = ref.to_a.map do |(title, url)| + "[#{title.strip}](#{url.strip})" + end + + "See #{parts.join(', ')}." + end + def render_deprecation(object, owner, context) + buff = [] deprecation = schema_deprecation(owner, object[:name]) - return deprecation.markdown(context: context) if deprecation - reason = object[:deprecation_reason] || 'Use of this is deprecated.' - "**Deprecated:** #{reason}" + buff << (deprecation&.original_description || render_description_of(object, owner)) if context == :block + buff << if deprecation + deprecation.markdown(context: context) + else + "**Deprecated:** #{object[:deprecation_reason]}" + end + + join(context, buff) end def render_field_type(type) "[`#{type[:info]}`](##{type[:name].downcase})" end + def join(context, chunks) + chunks.compact! + return if chunks.blank? + + case context + when :block + chunks.join("\n\n") + when :inline + chunks.join(" ").squish.presence + when :table + chunks.join("\n") + end + end + # Queries + def sorted_by_name(objects) + return [] unless objects.present? + + objects.sort_by { |o| o[:name] } + end + + def connection?(field) + type_name = field.dig(:type, :name) + type_name.present? && type_name.ends_with?('Connection') + end + + # We are ignoring connections and built in types for now, + # they should be added when queries are generated. + def objects + strong_memoize(:objects) do + mutations = schema.mutation&.fields&.keys&.to_set || [] + + graphql_object_types + .reject { |object_type| object_type[:name]["__"] || object_type[:name] == 'Subscription' } # We ignore introspection and subscription types. + .map do |type| + name = type[:name] + type.merge( + is_edge: name.ends_with?('Edge'), + is_connection: name.ends_with?('Connection'), + is_payload: name.ends_with?('Payload') && mutations.include?(name.chomp('Payload').camelcase(:lower)), + fields: type[:fields] + type[:connections] + ) + end + end + end + + def args?(field) + args = field[:arguments] + return false if args.blank? + return true unless connection?(field) + + args.any? { |arg| CONNECTION_ARGS.exclude?(arg[:name]) } + end + # returns the deprecation information for a field or argument # See: Gitlab::Graphql::Deprecation def schema_deprecation(type_name, field_name) - schema_member(type_name, field_name)&.deprecation - end - - # Return a part of the schema. - # - # This queries the Schema by owner and name to find: - # - # - fields (e.g. `schema_member('Query', 'currentUser')`) - # - arguments (e.g. `schema_member(['Query', 'project], 'fullPath')`) - def schema_member(type_name, field_name) - type_name = Array.wrap(type_name) - if type_name.size == 2 - arg_name = field_name - type_name, field_name = type_name - else - type_name = type_name.first - arg_name = nil - end + key = [*Array.wrap(type_name), field_name].join('.') + deprecations[key] + end - return if type_name.nil? || field_name.nil? + def render_input_type(query) + input_field = query[:input_fields]&.first + return unless input_field + "Input type: `#{input_field[:type][:name]}`" + end + + def schema_field(type_name, field_name) type = schema.types[type_name] return unless type && type.kind.fields? - field = type.fields[field_name] - return field if arg_name.nil? + type.fields[field_name] + end + + def deprecations + strong_memoize(:deprecations) do + mapping = {} + + schema.types.each do |type_name, type| + next unless type.kind.fields? - args = field.arguments - is_mutation = field.mutation && field.mutation <= ::Mutations::BaseMutation - args = args['input'].type.unwrap.arguments if is_mutation + type.fields.each do |field_name, field| + mapping["#{type_name}.#{field_name}"] = field.try(:deprecation) + field.arguments.each do |arg_name, arg| + mapping["#{type_name}.#{field_name}.#{arg_name}"] = arg.try(:deprecation) + end + end + end + + mapping.compact + end + end - args[arg_name] + def assert!(claim, message) + raise ViolatedAssumption, "#{message}\n#{SUGGESTED_ACTION}" unless claim end end end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb index 497567f9389..ae0898e6198 100644 --- a/lib/gitlab/graphql/docs/renderer.rb +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -24,6 +24,7 @@ module Gitlab @layout = Haml::Engine.new(File.read(template)) @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse @schema = schema + @seen = Set.new end def contents @@ -37,6 +38,16 @@ module Gitlab FileUtils.mkdir_p(@output_dir) File.write(filename, contents) end + + private + + def seen_type?(name) + @seen.include?(name) + end + + def seen_type!(name) + @seen << name + end end end end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index fe73297d0d9..7d42fb3a9f8 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -17,7 +17,9 @@ Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-and-removal-process) can be found in [Removed Items](../removed_items.md). - <!-- vale gitlab.Spelling = NO --> + <!-- vale off --> + <!-- Docs linting disabled after this line. --> + <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests --> \ :plain @@ -26,17 +28,81 @@ The `Query` type contains the API's top-level entry points for all executable queries. \ -- sorted_by_name(queries).each do |query| - = render_name_and_description(query, owner: 'Query') +- fields_of('Query').each do |field| + = render_full_field(field, heading_level: 3, owner: 'Query') + \ + +:plain + ## `Mutation` type + + The `Mutation` type contains all the mutations you can execute. + + All mutations receive their arguments in a single input object named `input`, and all mutations + support at least a return field `errors` containing a list of error messages. + + All input objects may have a `clientMutationId: String` field, identifying the mutation. + + For example: + + ```graphql + mutation($id: NoteableID!, $body: String!) { + createNote(input: { noteableId: $id, body: $body }) { + errors + } + } + ``` +\ + +- mutations.each do |field| + = render_full_field(field, heading_level: 3, owner: 'Mutation') + \ + +:plain + ## Connections + + Some types in our schema are `Connection` types - they represent a paginated + collection of edges between two nodes in the graph. These follow the + [Relay cursor connections specification](https://relay.dev/graphql/connections.htm). + + ### Pagination arguments {#connection-pagination-arguments} + + All connection fields support the following pagination arguments: + + | Name | Type | Description | + |------|------|-------------| + | `after` | [`String`](#string) | Returns the elements in the list that come after the specified cursor. | + | `before` | [`String`](#string) | Returns the elements in the list that come before the specified cursor. | + | `first` | [`Int`](#int) | Returns the first _n_ elements from the list. | + | `last` | [`Int`](#int) | Returns the last _n_ elements from the list. | + + Since these arguments are common to all connection fields, they are not repeated for each connection. + + ### Connection fields + + All connections have at least the following fields: + + | Name | Type | Description | + |------|------|-------------| + | `pageInfo` | [`PageInfo!`](#pageinfo) | Pagination information. | + | `edges` | `[edge!]` | The edges. | + | `nodes` | `[item!]` | The items in the current page. | + + The precise type of `Edge` and `Item` depends on the kind of connection. A + [`ProjectConnection`](#projectconnection) will have nodes that have the type + [`[Project!]`](#project), and edges that have the type [`ProjectEdge`](#projectedge). + + ### Connection types + + Some of the types in the schema exist solely to model connections. Each connection + has a distinct, named type, with a distinct named edge type. These are listed separately + below. +\ + +- connection_object_types.each do |type| + = render_name_and_description(type, level: 4) + \ + = render_object_fields(type[:fields], owner: type, level_bump: 1) \ - = render_return_type(query) - - unless query[:arguments].empty? - ~ "#### Arguments\n" - ~ "| Name | Type | Description |" - ~ "| ---- | ---- | ----------- |" - - sorted_by_name(query[:arguments]).each do |argument| - = render_field(argument, query[:type][:name]) - \ :plain ## Object types @@ -44,22 +110,20 @@ Object types represent the resources that the GitLab GraphQL API can return. They contain _fields_. Each field has its own type, which will either be one of the basic GraphQL [scalar types](https://graphql.org/learn/schema/#scalar-types) - (e.g.: `String` or `Boolean`) or other object types. + (e.g.: `String` or `Boolean`) or other object types. Fields may have arguments. + Fields with arguments are exactly like top-level queries, and are listed beneath + the table of fields for each object type. For more information, see [Object Types and Fields](https://graphql.org/learn/schema/#object-types-and-fields) on `graphql.org`. \ -- objects.each do |type| - - unless type[:fields].empty? - = render_name_and_description(type) - \ - ~ "| Field | Type | Description |" - ~ "| ----- | ---- | ----------- |" - - sorted_by_name(type[:fields]).each do |field| - = render_field(field, type[:name]) - \ +- object_types.each do |type| + = render_name_and_description(type) + \ + = render_object_fields(type[:fields], owner: type) + \ :plain ## Enumeration types @@ -73,14 +137,13 @@ \ - enums.each do |enum| - - unless enum[:values].empty? - = render_name_and_description(enum) - \ - ~ "| Value | Description |" - ~ "| ----- | ----------- |" - - sorted_by_name(enum[:values]).each do |value| - = render_enum_value(enum, value) - \ + = render_name_and_description(enum) + \ + ~ "| Value | Description |" + ~ "| ----- | ----------- |" + - enum[:values].each do |value| + = render_enum_value(enum, value) + \ :plain ## Scalar types @@ -133,7 +196,7 @@ ### Interfaces \ -- graphql_interface_types.each do |type| +- interfaces.each do |type| = render_name_and_description(type, level: 4) \ Implementations: @@ -141,8 +204,21 @@ - type[:implemented_by].each do |type_name| ~ "- [`#{type_name}`](##{type_name.downcase})" \ - ~ "| Field | Type | Description |" - ~ "| ----- | ---- | ----------- |" - - sorted_by_name(type[:fields] + type[:connections]).each do |field| - = render_field(field, type[:name]) + = render_object_fields(type[:fields], owner: type, level_bump: 1) + \ + +:plain + ## Input types + + Types that may be used as arguments (all scalar types may also + be used as arguments). + + Only general use input types are listed here. For mutation input types, + see the associated mutation type above. +\ + +- input_types.each do |type| + = render_name_and_description(type) + \ + = render_argument_table(3, type[:input_fields], type[:name]) \ diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index e525996ec10..61903c566f0 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -114,7 +114,7 @@ module Gitlab def limited_nodes strong_memoize(:limited_nodes) do if first && last - raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + raise Gitlab::Graphql::Errors::ArgumentError, "Can only provide either `first` or `last`, not both" end if last @@ -158,7 +158,7 @@ module Gitlab def ordered_items strong_memoize(:ordered_items) do unless items.primary_key.present? - raise ArgumentError.new('Relation must have a primary key') + raise ArgumentError, 'Relation must have a primary key' end list = OrderInfo.build_order_list(items) diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index 318c6e1734f..f1b74999897 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -10,6 +10,8 @@ module Gitlab extend ActiveSupport::Concern def ordered_items + raise ArgumentError, 'Relation must have a primary key' unless items.primary_key.present? + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) items @@ -40,6 +42,17 @@ module Gitlab sliced = slice_nodes(sliced, after, :after) if after.present? sliced end + + def items + original_items = super + return original_items if Gitlab::Pagination::Keyset::Order.keyset_aware?(original_items) || Feature.disabled?(:new_graphql_keyset_pagination) + + strong_memoize(:generic_keyset_pagination_items) do + rebuilt_items_with_keyset_order, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(original_items) + + success ? rebuilt_items_with_keyset_order : original_items + end + end end end end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index 0494329bfd9..57e85ebe7f6 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -36,24 +36,24 @@ module Gitlab def self.validate_ordering(relation, order_list) if order_list.empty? - raise ArgumentError.new('A minimum of 1 ordering field is required') + raise ArgumentError, 'A minimum of 1 ordering field is required' end if order_list.count > 2 # Keep in mind an order clause for primary key is added if one is not present # lib/gitlab/graphql/pagination/keyset/connection.rb:97 - raise ArgumentError.new('A maximum of 2 ordering fields are allowed') + raise ArgumentError, 'A maximum of 2 ordering fields are allowed' end # make sure the last ordering field is non-nullable attribute_name = order_list.last&.attribute_name if relation.columns_hash[attribute_name].null - raise ArgumentError.new("Column `#{attribute_name}` must not allow NULL") + raise ArgumentError, "Column `#{attribute_name}` must not allow NULL" end if order_list.last.attribute_name != relation.primary_key - raise ArgumentError.new("Last ordering field must be the primary key, `#{relation.primary_key}`") + raise ArgumentError, "Last ordering field must be the primary key, `#{relation.primary_key}`" end end @@ -121,4 +121,4 @@ module Gitlab end end -Gitlab::Graphql::Pagination::Keyset::OrderInfo.prepend_if_ee('EE::Gitlab::Graphql::Pagination::Keyset::OrderInfo') +Gitlab::Graphql::Pagination::Keyset::OrderInfo.prepend_mod_with('Gitlab::Graphql::Pagination::Keyset::OrderInfo') diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb index ee9c902c735..a2f53ae83dd 100644 --- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb +++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb @@ -12,7 +12,7 @@ module Gitlab @before_or_after = before_or_after if order_list.empty? - raise ArgumentError.new('No ordering scopes have been supplied') + raise ArgumentError, 'No ordering scopes have been supplied' end end @@ -49,7 +49,7 @@ module Gitlab end if order_list.count == 1 && attr_values.first.nil? - raise Gitlab::Graphql::Errors::ArgumentError.new('Before/after cursor invalid: `nil` was provided as only sortable value') + raise Gitlab::Graphql::Errors::ArgumentError, 'Before/after cursor invalid: `nil` was provided as only sortable value' end if order_list.count == 1 || attr_values.first.present? diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb index fdaf075eb25..3608cb4c0e8 100644 --- a/lib/gitlab/graphql/present.rb +++ b/lib/gitlab/graphql/present.rb @@ -10,14 +10,14 @@ module Gitlab end def self.presenter_class - @presenter_class + @presenter_class || superclass.try(:presenter_class) end def self.present(object, attrs) - klass = @presenter_class + klass = presenter_class return object if !klass || object.is_a?(klass) - @presenter_class.new(object, **attrs) + klass.new(object, **attrs) end end diff --git a/lib/gitlab/graphql/present/field_extension.rb b/lib/gitlab/graphql/present/field_extension.rb index 2e211b70d35..050a3a276ea 100644 --- a/lib/gitlab/graphql/present/field_extension.rb +++ b/lib/gitlab/graphql/present/field_extension.rb @@ -13,7 +13,8 @@ module Gitlab # inner Schema::Object#object. This depends on whether the field # has a @resolver_proc or not. if object.is_a?(::Types::BaseObject) - object.present(field.owner, attrs) + type = field.owner.kind.abstract? ? object.class : field.owner + object.present(type, attrs) yield(object, arguments) else # This is the legacy code-path, hit if the field has a @resolver_proc diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index 74f55abccbc..5d3a9245427 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -264,7 +264,7 @@ module Gitlab definitions = [] ::Find.find(root.to_s) do |path| - definitions << Definition.new(path, fragments) if query?(path) + definitions << Definition.new(path, fragments) if query_for_gitlab_schema?(path) end definitions @@ -288,10 +288,11 @@ module Gitlab @known_failures.fetch('filenames', []).any? { |known_failure| path.to_s.ends_with?(known_failure) } end - def self.query?(path) + def self.query_for_gitlab_schema?(path) path.ends_with?('.graphql') && !path.ends_with?('.fragment.graphql') && - !path.ends_with?('typedefs.graphql') + !path.ends_with?('typedefs.graphql') && + !/.*\.customer\.(query|mutation)\.graphql$/.match?(path) end end end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index c6f22e0bd4f..b8d2f5b0f29 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -16,7 +16,7 @@ module Gitlab query_string: query.query_string, variables: variables }) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) default_initial_values(query) end @@ -41,7 +41,7 @@ module Gitlab RequestStore.store[:graphql_logs] ||= [] RequestStore.store[:graphql_logs] << memo GraphqlLogger.info(memo.except!(:time_started, :query)) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb index 1c6fb011012..e17ca56d022 100644 --- a/lib/gitlab/graphql/variables.rb +++ b/lib/gitlab/graphql/variables.rb @@ -32,7 +32,7 @@ module Gitlab raise Invalid, "Unexpected parameter: #{ambiguous_param}" end rescue JSON::ParserError => e - raise Invalid.new(e) + raise Invalid, e end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index dd872caee0e..4eea96f8344 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -39,4 +39,4 @@ module Gitlab end end -Gitlab::GroupSearchResults.prepend_if_ee('EE::Gitlab::GroupSearchResults') +Gitlab::GroupSearchResults.prepend_mod_with('Gitlab::GroupSearchResults') diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb index b57560544c8..912e2ee99e9 100644 --- a/lib/gitlab/hashed_storage/migrator.rb +++ b/lib/gitlab/hashed_storage/migrator.rb @@ -66,7 +66,7 @@ module Gitlab Gitlab::AppLogger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." project.migrate_to_hashed_storage! - rescue => err + rescue StandardError => err Gitlab::AppLogger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end @@ -77,7 +77,7 @@ module Gitlab Gitlab::AppLogger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." project.rollback_to_legacy_storage! - rescue => err + rescue StandardError => err Gitlab::AppLogger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") end diff --git a/lib/gitlab/health_checks/probes/collection.rb b/lib/gitlab/health_checks/probes/collection.rb index b34e4273d85..76ad1c84214 100644 --- a/lib/gitlab/health_checks/probes/collection.rb +++ b/lib/gitlab/health_checks/probes/collection.rb @@ -20,7 +20,7 @@ module Gitlab success ? 200 : 503, status(success).merge(payload(readiness)) ) - rescue => e + rescue StandardError => e exception_payload = { message: "#{e.class} : #{e.message}" } Probes::Status.new( diff --git a/lib/gitlab/health_checks/simple_abstract_check.rb b/lib/gitlab/health_checks/simple_abstract_check.rb index ae99768b7b4..432d5d5e5ea 100644 --- a/lib/gitlab/health_checks/simple_abstract_check.rb +++ b/lib/gitlab/health_checks/simple_abstract_check.rb @@ -16,7 +16,7 @@ module Gitlab else HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{check_result}") end - rescue => e + rescue StandardError => e HealthChecks::Result.new(name, false, "unexpected #{human_name} check result: #{e}") end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 765d3dfca56..e4857280969 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -4,13 +4,20 @@ module Gitlab class Highlight TIMEOUT_BACKGROUND = 30.seconds TIMEOUT_FOREGROUND = 1.5.seconds - MAXIMUM_TEXT_HIGHLIGHT_SIZE = 512.kilobytes def self.highlight(blob_name, blob_content, language: nil, plain: false) new(blob_name, blob_content, language: language) .highlight(blob_content, continue: false, plain: plain) end + def self.too_large?(size) + return false unless size.to_i > Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] + + over_highlight_size_limit.increment(source: "text highlighter") if Feature.enabled?(:track_file_size_over_highlight_limit) + + true + end + attr_reader :blob_name def initialize(blob_name, blob_content, language: nil) @@ -23,7 +30,7 @@ module Gitlab def highlight(text, continue: false, plain: false, context: {}) @context = context - plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE + plain ||= self.class.too_large?(text.length) highlighted_text = highlight_text(text, continue: continue, plain: plain) highlighted_text = link_dependencies(text, highlighted_text) if blob_name @@ -65,9 +72,11 @@ module Gitlab tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } rescue Timeout::Error => e + add_highlight_timeout_metric + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) - rescue + rescue StandardError highlight_plain(text) end @@ -78,5 +87,25 @@ module Gitlab def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end + + def add_highlight_timeout_metric + return unless Feature.enabled?(:track_highlight_timeouts) + + highlight_timeout.increment(source: Gitlab::Runtime.sidekiq? ? "background" : "foreground") + end + + def highlight_timeout + @highlight_timeout ||= Gitlab::Metrics.counter( + :highlight_timeout, + 'Counts the times highlights have timed out' + ) + end + + def self.over_highlight_size_limit + @over_highlight_size_limit ||= Gitlab::Metrics.counter( + :over_highlight_size_limit, + 'Count the times files have been over the highlight size limit' + ) + end end end diff --git a/lib/gitlab/hook_data/group_member_builder.rb b/lib/gitlab/hook_data/group_member_builder.rb index 32cfd032ffe..2998550a4b5 100644 --- a/lib/gitlab/hook_data/group_member_builder.rb +++ b/lib/gitlab/hook_data/group_member_builder.rb @@ -62,4 +62,4 @@ module Gitlab end end -Gitlab::HookData::GroupMemberBuilder.prepend_if_ee('EE::Gitlab::HookData::GroupMemberBuilder') +Gitlab::HookData::GroupMemberBuilder.prepend_mod_with('Gitlab::HookData::GroupMemberBuilder') diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index f38012c9804..d5595e80bdf 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -58,4 +58,4 @@ module Gitlab end end -Gitlab::HookData::IssueBuilder.prepend_if_ee('EE::Gitlab::HookData::IssueBuilder') +Gitlab::HookData::IssueBuilder.prepend_mod_with('Gitlab::HookData::IssueBuilder') diff --git a/lib/gitlab/hook_data/key_builder.rb b/lib/gitlab/hook_data/key_builder.rb new file mode 100644 index 00000000000..8eaf4dfd762 --- /dev/null +++ b/lib/gitlab/hook_data/key_builder.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class KeyBuilder < BaseBuilder + alias_method :key, :object + + # Sample data + # { + # event_name: "key_create", + # created_at: "2021-04-19T06:13:24Z", + # updated_at: "2021-04-19T06:13:24Z", + # key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQClDn/5BaESHlSb3NxQtiUc0BXgK6lsqdAUIdS3lwZ2gbACDhtoLYnc+qhZ4b8gWzE+2A8RmkvLe98T7noRoW4DAYs67NSqMs/kXd2ESPNV8qqv0u7tCxPz+c7DaYp2oC/avlxVQ2AeULZLCEwalYZ7irde0EZMeTwNIRu5s88gOw== dummy@gitlab.com", + # id: 1, + # username: "johndoe" + # } + + def build(event) + [ + event_data(event), + timestamps_data, + key_data, + user_data + ].reduce(:merge) + end + + private + + def key_data + { + key: key.key, + id: key.id + } + end + + def user_data + user = key.user + return {} unless user + + { + username: user.username + } + end + end + end +end diff --git a/lib/gitlab/hook_data/project_builder.rb b/lib/gitlab/hook_data/project_builder.rb new file mode 100644 index 00000000000..65c237f743f --- /dev/null +++ b/lib/gitlab/hook_data/project_builder.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class ProjectBuilder < BaseBuilder + alias_method :project, :object + + # Sample data + # { + # event_name: "project_rename", + # created_at: "2021-04-19T07:05:36Z", + # updated_at: "2021-04-19T07:05:36Z", + # name: "my_project", + # path: "my_project", + # path_with_namespace: "namespace2/my_project", + # project_id: 1, + # owner_name: "John", + # owner_email: "user1@example.org", + # project_visibility: "internal", + # old_path_with_namespace: "old-path-with-namespace" + # } + + def build(event) + [ + event_data(event), + timestamps_data, + project_data, + event_specific_project_data(event) + ].reduce(:merge) + end + + private + + def project_data + owner = project.owner + + { + name: project.name, + path: project.path, + path_with_namespace: project.full_path, + project_id: project.id, + owner_name: owner.name, + owner_email: owner.respond_to?(:email) ? owner.email : "", + project_visibility: project.visibility.downcase + } + end + + def event_specific_project_data(event) + return {} unless event == :rename || event == :transfer + + { + old_path_with_namespace: project.old_path_with_namespace + } + end + end + end +end diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb index 537245e948f..54f03b863e5 100644 --- a/lib/gitlab/hook_data/user_builder.rb +++ b/lib/gitlab/hook_data/user_builder.rb @@ -50,4 +50,4 @@ module Gitlab end end -Gitlab::HookData::UserBuilder.prepend_if_ee('EE::Gitlab::HookData::UserBuilder') +Gitlab::HookData::UserBuilder.prepend_mod_with('Gitlab::HookData::UserBuilder') diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3b19ae3d7ff..023dbd1c601 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -4,20 +4,6 @@ module Gitlab module I18n extend self - # Languages with less then 2% of available translations will not - # be available in the UI. - # https://gitlab.com/gitlab-org/gitlab/-/issues/221012 - NOT_AVAILABLE_IN_UI = %w[ - fil_PH - pl_PL - nl_NL - id_ID - cs_CZ - bg - eo - gl_ES - ].freeze - AVAILABLE_LANGUAGES = { 'bg' => 'Bulgarian - български', 'cs_CZ' => 'Czech - čeština', @@ -42,9 +28,49 @@ module Gitlab 'zh_HK' => 'Chinese, Traditional (Hong Kong) - 繁體中文 (香港)', 'zh_TW' => 'Chinese, Traditional (Taiwan) - 繁體中文 (台灣)' }.freeze + private_constant :AVAILABLE_LANGUAGES + + # Languages with less then MINIMUM_TRANSLATION_LEVEL% of available translations will not + # be available in the UI. + # https://gitlab.com/gitlab-org/gitlab/-/issues/221012 + MINIMUM_TRANSLATION_LEVEL = 2 + + # Currently monthly updated manually by ~group::import PM. + # https://gitlab.com/gitlab-org/gitlab/-/issues/18923 + TRANSLATION_LEVELS = { + 'bg' => 1, + 'cs_CZ' => 1, + 'de' => 19, + 'en' => 100, + 'eo' => 1, + 'es' => 41, + 'fil_PH' => 1, + 'fr' => 14, + 'gl_ES' => 1, + 'id_ID' => 0, + 'it' => 2, + 'ja' => 45, + 'ko' => 14, + 'nl_NL' => 1, + 'pl_PL' => 1, + 'pt_BR' => 22, + 'ru' => 32, + 'tr_TR' => 17, + 'uk' => 43, + 'zh_CN' => 72, + 'zh_HK' => 3, + 'zh_TW' => 4 + }.freeze + private_constant :TRANSLATION_LEVELS def selectable_locales - AVAILABLE_LANGUAGES.reject { |key, _value| NOT_AVAILABLE_IN_UI.include? key } + AVAILABLE_LANGUAGES.reject do |code, _name| + percentage_translated_for(code) < MINIMUM_TRANSLATION_LEVEL + end + end + + def percentage_translated_for(code) + TRANSLATION_LEVELS.fetch(code, 0) end def available_locales diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index c4867746b0f..231f2a977c0 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -99,11 +99,19 @@ module Gitlab def group_config_file Rails.root.join('lib/gitlab/import_export/group/import_export.yml') end + + def group_wiki_repo_bundle_filename(group_id) + "#{group_id}.wiki.bundle" + end + + def group_wiki_repo_bundle_path(shared, filename) + File.join(shared.export_path, 'repositories', filename) + end + + def group_wiki_repo_bundle_full_path(shared, group_id) + group_wiki_repo_bundle_path(shared, group_wiki_repo_bundle_filename(group_id)) + end end end -Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport') - -# The methods in `Gitlab::ImportExport::GroupHelper` should be available as both -# instance and class methods. -Gitlab::ImportExport.extend_if_ee('Gitlab::ImportExport::GroupHelper') +Gitlab::ImportExport.prepend_mod diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b30258123d4..b43d0a0c3eb 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -42,7 +42,7 @@ module Gitlab strategy_execute true - rescue => e + rescue StandardError => e project.import_export_shared.error(e) false ensure diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index e2dba831661..1e8009d29c2 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -28,7 +28,7 @@ module Gitlab def handle_response_error(response) unless response.success? - raise StrategyError.new("Error uploading the project. Code #{response.code}: #{response.message}") + raise StrategyError, "Error uploading the project. Code #{response.code}: #{response.message}" end end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb index 37394f46a99..d7b30f46903 100644 --- a/lib/gitlab/import_export/after_export_strategy_builder.rb +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -12,7 +12,7 @@ module Gitlab klass = strategy_klass.constantize rescue nil unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy - raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + raise StrategyNotFoundError, "Strategy #{strategy_klass} not found" end klass.new(**attributes.symbolize_keys) diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 1e98595bb07..4abc3da1190 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class AttributesFinder - attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders def initialize(config:) @tree = config[:tree] || {} @@ -11,6 +11,7 @@ module Gitlab @excluded_attributes = config[:excluded_attributes] || {} @methods = config[:methods] || {} @preloads = config[:preloads] || {} + @export_reorders = config[:export_reorders] || {} end def find_root(model_key) @@ -33,7 +34,8 @@ module Gitlab except: @excluded_attributes[model_key], methods: @methods[model_key], include: resolve_model_tree(model_tree), - preload: resolve_preloads(model_key, model_tree) + preload: resolve_preloads(model_key, model_tree), + export_reorder: @export_reorders[model_key] }.compact end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index be1b97bd7a7..01ff99798d5 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -13,7 +13,7 @@ module Gitlab @project.avatar = File.open(avatar_export_file) @project.save! - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 47ca898c690..7534ab5a9ce 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -16,7 +16,7 @@ module Gitlab shared: @shared, relative_export_path: 'avatar' ).save - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05a4a8f4c93..959ece4b903 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -44,8 +44,9 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + def initialize(relation_sym:, relation_index:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym + @relation_index = relation_index @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @object_builder = object_builder diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f8769e261d..ace9d83dc9a 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,19 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def gzip(dir:, filename:) + filepath = File.join(dir, filename) + cmd = %W(gzip #{filepath}) + + _, status = Gitlab::Popen.popen(cmd) + + if status == 0 + status + else + raise Gitlab::ImportExport::Error.file_compression_error + end + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) FileUtils.chmod(DEFAULT_DIR_MODE, path) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 37f1bdc3009..2baf2c61f7c 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -61,7 +61,7 @@ module Gitlab Process.kill(-1, pgrp) if pgrp false - rescue => e + rescue StandardError => e log_error(e.message) Process.kill(-1, pgrp) if pgrp diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index f11b7a0a298..4af6b03fe94 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -3,12 +3,20 @@ module Gitlab module ImportExport class Error < StandardError - def self.permission_error(user, importable) + def self.permission_error(user, object) self.new( "User with ID: %s does not have required permissions for %s: %s with ID: %s" % - [user.id, importable.class.name, importable.name, importable.id] + [user.id, object.class.name, object.name, object.id] ) end + + def self.unsupported_object_type_error + self.new('Unknown object type') + end + + def self.file_compression_error + self.new('File compression failed') + end end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 51d58aae54f..4b3258f8caa 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -33,7 +33,7 @@ module Gitlab validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) decompress_archive end - rescue => e + rescue StandardError => e @shared.error(e) false ensure @@ -57,7 +57,7 @@ module Gitlab def decompress_archive result = untar_zxf(archive: @archive_file, dir: @shared.export_path) - raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + raise ImporterError, "Unable to decompress #{@archive_file} into #{@shared.export_path}" unless result result end @@ -67,7 +67,17 @@ module Gitlab @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable)) - download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file) + remote_download_or_download_or_copy_upload + end + + def remote_download_or_download_or_copy_upload + import_export_upload = @importable.import_export_upload + + if import_export_upload.remote_import_url.present? + download(import_export_upload.remote_import_url, @archive_file) + else + download_or_copy_upload(import_export_upload.import_file, @archive_file) + end end def remove_symlinks @@ -87,7 +97,7 @@ module Gitlab end def validate_decompressed_archive_size - raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid? + raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid? end def size_validator diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index e30206dc509..aceb4821a06 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -58,6 +58,8 @@ methods: preloads: +export_reorders: + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 5008639077c..19611e1b010 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -60,6 +60,8 @@ methods: preloads: +export_reorders: + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 5499b79cee6..2b95c098b59 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -45,7 +45,7 @@ module Gitlab return false if @shared.errors.any? true - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb index 7ab81c09885..0f74fabeac3 100644 --- a/lib/gitlab/import_export/group/legacy_tree_saver.rb +++ b/lib/gitlab/import_export/group/legacy_tree_saver.rb @@ -19,7 +19,7 @@ module Gitlab tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -35,7 +35,7 @@ module Gitlab end group_tree - rescue => e + rescue StandardError => e @shared.error(e) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index 925ab6680ba..ea7de4cc896 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -26,7 +26,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -74,7 +74,7 @@ module Gitlab group = create_group(group_attributes) restore_group(group, group_attributes) - rescue => e + rescue StandardError => e import_failure_service.log_import_failure( source: 'process_child', relation_key: 'group', diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index d538de33c51..0f588a55f9d 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -25,7 +25,7 @@ module Gitlab json_writer.write_relation_array('groups', '_all', all_groups) true - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 390909efe36..c2510bbe938 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -21,15 +21,15 @@ module Gitlab if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project else - raise Projects::ImportService::Error.new(shared.errors.to_sentence) + raise Projects::ImportService::Error, shared.errors.to_sentence end - rescue => e + rescue StandardError => e # If some exception was raised could mean that the SnippetsRepoRestorer # was not called. This would leave us with snippets without a repository. # This is a state we don't want them to be, so we better delete them. remove_non_migrated_snippets - raise Projects::ImportService::Error.new(e.message) + raise Projects::ImportService::Error, e.message ensure remove_base_tmp_dir remove_import_file diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index 12d6458aedc..f29c0a44188 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -28,9 +28,9 @@ module Gitlab def read_hash ActiveSupport::JSON.decode(IO.read(@path)) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 05b7679e0ff..ec42c5e51c0 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -38,16 +38,6 @@ module Gitlab end end - private - - attr_reader :json_writer, :relations_schema, :exportable - - def serialize_root - attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) - json_writer.write_attributes(@exportable_path, attributes) - end - def serialize_relation(definition) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? @@ -64,17 +54,22 @@ module Gitlab end end + private + + attr_reader :json_writer, :relations_schema, :exportable + + def serialize_root + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + json_writer.write_attributes(@exportable_path, attributes) + end + def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) - records = records.preload(key_preloads) if key_preloads - records.in_batches(of: batch_size) do |batch| # rubocop:disable Cop/InBatches - # order each batch by its primary key to ensure - # consistent and predictable ordering of each exported relation - # as additional `WHERE` clauses can impact the order in which data is being - # returned by database when no `ORDER` is specified - batch = batch.reorder(batch.klass.primary_key) + batch(records, key) do |batch| + batch = batch.preload(key_preloads) if key_preloads batch.each do |record| items << Raw.new(record.to_json(options)) @@ -85,6 +80,29 @@ module Gitlab json_writer.write_relation_array(@exportable_path, key, enumerator) end + def batch(relation, key) + opts = { of: batch_size } + order_by = reorders(relation, key) + + # we need to sort issues by non primary key column(relative_position) + # and `in_batches` does not support that + if order_by + scope = relation.reorder(order_by) + + Gitlab::Pagination::Keyset::Iterator.new(scope: scope, use_union_optimization: true).each_batch(**opts) do |batch| + yield batch + end + else + relation.in_batches(**opts) do |batch| # rubocop:disable Cop/InBatches + # order each batch by its primary key to ensure + # consistent and predictable ordering of each exported relation + # as additional `WHERE` clauses can impact the order in which data is being + # returned by database when no `ORDER` is specified + yield batch.reorder(batch.klass.primary_key) + end + end + end + def serialize_many_each(key, records, options) enumerator = Enumerator.new do |items| records.each do |record| @@ -112,6 +130,42 @@ module Gitlab def batch_size @batch_size ||= self.class.batch_size(@exportable) end + + def reorders(relation, key) + export_reorder = relations_schema[:export_reorder]&.dig(key) + return unless export_reorder + + custom_reorder(relation.klass, export_reorder) + end + + def custom_reorder(klass, order_by) + arel_table = klass.arel_table + column = order_by[:column] || klass.primary_key + direction = order_by[:direction] || :asc + nulls_position = order_by[:nulls_position] || :nulls_last + + arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert + reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction] + reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] + order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position) + reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position) + + ::Gitlab::Pagination::Keyset::Order.build([ + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: column, + column_expression: arel_table[column], + order_expression: order_expression, + reversed_order_expression: reverse_order_expression, + order_direction: direction, + nullable: nulls_position, + distinct: false + ), + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: klass.primary_key, + order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) + ) + ]) + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index ef83cdf24b1..d73ae1410a3 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -20,7 +20,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -73,8 +73,8 @@ module Gitlab begin json = IO.read(lfs_json_path) ActiveSupport::JSON.decode(json) - rescue - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + rescue StandardError + raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 4964b8b16f4..47acd49d529 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -27,7 +27,7 @@ module Gitlab write_lfs_json true - rescue => e + rescue StandardError => e shared.error(e) false diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 6b37683ea68..ff972cf9352 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -52,7 +52,7 @@ module Gitlab @importable.members.destroy_all # rubocop: disable Cop/DestroyAll relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true) - rescue => e + rescue StandardError => e raise e, "Error adding importer user to #{@importable.class} members. #{e.message}" end diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 4643742b607..3910afef108 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -40,7 +40,7 @@ module Gitlab # the commits are missing. def create_source_branch @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) - rescue => err + rescue StandardError => err Gitlab::Import::Logger.warn( message: 'Import warning: Failed to create source branch', source_branch: @merge_request.source_branch, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 42d32593cbd..d000c331b6d 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -132,6 +132,7 @@ excluded_attributes: - :avatar - :import_type - :import_source + - :integrations - :mirror - :runners_token - :runners_token_encrypted @@ -152,6 +153,7 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :topic_list - :mirror_user_id - :mirror_trigger_builds - :only_mirror_protected_branches @@ -261,6 +263,8 @@ excluded_attributes: - :resource_group_id - :waiting_for_resource_at - :processed + - :id_convert_to_bigint + - :stage_id_convert_to_bigint sentry_issue: - :issue_id push_event_payload: @@ -393,6 +397,8 @@ methods: - :state preloads: + issues: + project: :route statuses: # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` # tags: # needed by tag_list @@ -402,6 +408,29 @@ preloads: target_project: # needed by target_branch_sha assignees: # needed by assigne_id that is implemented by DeprecatedAssignee +# Specify a custom export reordering for a given relationship +# For example for issues we use a custom export reordering by relative_position, so that on import, we can reset the +# relative position value, but still keep the issues order to the order in which issues were in the exported project. +# By default the ordering of relations is done by PK. +# column - specify the column by which to reorder, by default it is relation's PK +# direction - specify the ordering direction :asc or :desc, default :asc +# nulls_position - specify where would null values be positioned. Because custom ordering column can contain nulls we +# need to also specify where would the nulls be placed. It can be :nulls_last or :nulls_first, defaults +# to :nulls_last +# Example: +# export_reorders: +# project: +# issues: +# column: :relative_position +# direction: :asc +# nulls_position: :nulls_last +export_reorders: + project: + issues: + column: :relative_position + direction: :asc + nulls_position: :nulls_last + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index ae92228276e..4678396f97e 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -80,6 +80,7 @@ module Gitlab when :notes then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build + when :issues then setup_issue end update_project_references @@ -135,6 +136,22 @@ module Gitlab end end + def setup_issue + @relation_hash['relative_position'] = compute_relative_position + end + + def compute_relative_position + return unless max_relative_position + + max_relative_position + (@relation_index + 1) * Gitlab::RelativePositioning::IDEAL_DISTANCE + end + + def max_relative_position + Rails.cache.fetch("import:#{@importable.model_name.plural}:#{@importable.id}:hierarchy_max_issues_relative_position", expires_in: 24.hours) do + ::RelativePositioning.mover.context(Issue.in_projects(@importable.root_ancestor.all_projects).first)&.max_relative_position || ::Gitlab::RelativePositioning::START_POSITION + end + end + def legacy_trigger? @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? end @@ -158,4 +175,4 @@ module Gitlab end end -Gitlab::ImportExport::Project::RelationFactory.prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') +Gitlab::ImportExport::Project::RelationFactory.prepend_mod_with('Gitlab::ImportExport::Project::RelationFactory') diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index fb9e5be1877..113502b4e3c 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -39,7 +39,7 @@ module Gitlab else false end - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 80dacf2eb20..16012f3c0c0 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab ).execute true - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 8d36d05ca6f..b9a1aee3b8e 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -35,7 +35,7 @@ module Gitlab def tree_by_key(key) attributes_finder.find_root(key) - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 8bc87ecb071..46b82240ef7 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -48,7 +48,7 @@ module Gitlab @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -81,7 +81,7 @@ module Gitlab relation_object.save! log_relation_creation(@importable, relation_key, relation_object) end - rescue => e + rescue StandardError => e import_failure_service.log_import_failure( source: 'process_relation_item!', relation_key: relation_key, @@ -155,7 +155,7 @@ module Gitlab transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) end - relation = @relation_factory.create(**relation_factory_params(relation_key, data_hash)) + relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash)) if relation && !relation.valid? @shared.logger.warn( @@ -221,8 +221,9 @@ module Gitlab importable_class.to_s.downcase.to_sym end - def relation_factory_params(relation_key, data_hash) + def relation_factory_params(relation_key, relation_index, data_hash) { + relation_index: relation_index, relation_sym: relation_key.to_sym, relation_hash: data_hash, importable: @importable, diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 998da3e4afb..1c6629cf942 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -22,7 +22,7 @@ module Gitlab update_importable_repository_info true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -52,4 +52,4 @@ module Gitlab end end -Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer') +Gitlab::ImportExport::RepoRestorer.prepend_mod_with('Gitlab::ImportExport::RepoRestorer') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 0fdd0722b65..fae07039139 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -40,7 +40,7 @@ module Gitlab mkdir_p(File.dirname(bundle_full_path)) repository.bundle_to_disk(bundle_full_path) - rescue => e + rescue StandardError => e shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bb2bbda4bd6..bec709f4a36 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -27,7 +27,7 @@ module Gitlab @shared.error(Gitlab::ImportExport::Error.new(error_message)) false end - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 09ed4eb568d..f295ab38de0 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -90,7 +90,7 @@ module Gitlab when 'Group' @exportable.full_path else - raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}" end end diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb index 2d0aa05fc3c..cb13972f8f2 100644 --- a/lib/gitlab/import_export/snippet_repo_restorer.rb +++ b/lib/gitlab/import_export/snippet_repo_restorer.rb @@ -23,7 +23,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb index 3fafb01c37c..a3ad5edc2cc 100644 --- a/lib/gitlab/import_export/statistics_restorer.rb +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -10,7 +10,7 @@ module Gitlab def restore @project.statistics.refresh! - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 2f15cdd7506..ad19508fb99 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -17,7 +17,7 @@ module Gitlab copy_project_uploads true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -30,7 +30,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 5f422dcbefa..741c6555aad 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -8,7 +8,7 @@ module Gitlab project: @project, shared: @shared ).restore - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index be1066c30b2..9f58609fa17 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -13,7 +13,7 @@ module Gitlab project: @project, shared: @shared ).save - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 48f5b558e52..5ec9db00d0a 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -14,7 +14,7 @@ module Gitlab def check! version = File.open(version_file, &:readline) verify_version!(version) - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -27,7 +27,7 @@ module Gitlab def verify_version!(version) if different_version?(version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") + raise Gitlab::ImportExport::Error, "Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}" else true end @@ -35,13 +35,13 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) - rescue => e + rescue StandardError => e Gitlab::Import::Logger.error( message: 'Import error', error: e.message ) - raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + raise Gitlab::ImportExport::Error, 'Incorrect VERSION format' end end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index dab8bbf539d..e8f68f93af0 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -15,7 +15,7 @@ module Gitlab File.write(version_file, Gitlab::ImportExport.version, mode: 'w') File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 4b1cf4915e4..162b474bfeb 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') +Gitlab::ImportExport::WikiRepoSaver.prepend_mod_with('Gitlab::ImportExport::WikiRepoSaver') diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 95c002edf0a..c9f5005cede 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -25,7 +25,7 @@ module Gitlab ].freeze class << self - prepend_if_ee('EE::Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule + prepend_mod_with('Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule def options import_table.to_h { |importer| [importer.title, importer.name] } diff --git a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb index 768c8bb4cbb..6aeeb1d31aa 100644 --- a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb +++ b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb @@ -33,7 +33,7 @@ module Gitlab def incident_created_at Time.zone.parse(incident_payload['created_at']) - rescue + rescue StandardError Time.current.utc # PagerDuty provides time in UTC end diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 644a5fc4fff..005751fb0db 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -62,7 +62,7 @@ module Gitlab end if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord - raise CrossSlotError.new("Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") + raise CrossSlotError, "Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end end diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb new file mode 100644 index 00000000000..e6ea98e6d66 --- /dev/null +++ b/lib/gitlab/integrations/sti_type.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Integrations + class StiType < ActiveRecord::Type::String + NAMESPACED_INTEGRATIONS = Set.new(%w( + Asana Assembla Bamboo Campfire Confluence Datadog EmailsOnPush + )).freeze + + def cast(value) + new_cast(value) || super + end + + def serialize(value) + new_serialize(value) || super + end + + def deserialize(value) + value + end + + def changed?(original_value, value, _new_value_before_type_cast) + original_value != serialize(value) + end + + def changed_in_place?(original_value_for_database, value) + original_value_for_database != serialize(value) + end + + private + + def new_cast(value) + value = prepare_value(value) + return unless value + + stripped_name = value.delete_suffix('Service') + return unless NAMESPACED_INTEGRATIONS.include?(stripped_name) + + "Integrations::#{stripped_name}" + end + + def new_serialize(value) + value = prepare_value(value) + return unless value&.starts_with?('Integrations::') + + "#{value.delete_prefix('Integrations::')}Service" + end + + # Returns value cast to a `String`, or `nil` if value is `nil`. + def prepare_value(value) + return value if value.nil? || value.is_a?(String) + + value.to_s + end + end + end +end diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index f96c937aec3..d0702fb5c7d 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -98,4 +98,4 @@ module Gitlab end end -Gitlab::IssuableMetadata.prepend_if_ee('EE::Gitlab::IssuableMetadata') +Gitlab::IssuableMetadata.prepend_mod_with('Gitlab::IssuableMetadata') diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index f0b08bb6b6a..3e7659db240 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -12,7 +12,7 @@ module Gitlab def request(*args) result = make_request(*args) - raise JIRA::HTTPError.new(result.response) unless result.response.is_a?(Net::HTTPSuccess) + raise JIRA::HTTPError, result.response unless result.response.is_a?(Net::HTTPSuccess) result end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 26fa01755d1..8a03162f111 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -70,7 +70,7 @@ module Gitlab # These ids are cleaned-up when import finishes. # see Gitlab::JiraImport::Stage::FinishImportWorker mark_as_imported(jira_issue.id) - rescue => ex + rescue StandardError => ex # handle exceptionn here and skip the failed to import issue, instead of # failing to import the entire batch of issues diff --git a/lib/gitlab/jira_import/labels_importer.rb b/lib/gitlab/jira_import/labels_importer.rb index 6e6842e06bf..046dc3fd04f 100644 --- a/lib/gitlab/jira_import/labels_importer.rb +++ b/lib/gitlab/jira_import/labels_importer.rb @@ -47,7 +47,7 @@ module Gitlab Gitlab::JiraImport::HandleLabelsService.new(project, response['values']).execute response['isLast'] - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, project_id: project.id, request: request) end end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index b51c0a33457..561cd4509b1 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -84,7 +84,7 @@ module Gitlab Oj.load(string, opts) rescue Oj::ParseError, Encoding::UndefinedConversionError => ex - raise parser_error.new(ex) + raise parser_error, ex end # Take a Ruby object and convert it to a string. This method varies @@ -169,7 +169,7 @@ module Gitlab # @return [Boolean] def feature_table_exists? Feature::FlipperFeature.table_exists? - rescue + rescue StandardError false end end diff --git a/lib/gitlab/jwt_token.rb b/lib/gitlab/jwt_token.rb new file mode 100644 index 00000000000..11bc5479b6e --- /dev/null +++ b/lib/gitlab/jwt_token.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + class JWTToken < JSONWebToken::HMACToken + HMAC_ALGORITHM = 'SHA256' + HMAC_KEY = 'gitlab-jwt' + HMAC_EXPIRES_IN = 5.minutes.freeze + + class << self + def decode(jwt) + payload = super(jwt, secret).first + + new.tap do |jwt_token| + jwt_token.id = payload.delete('jti') + jwt_token.issued_at = payload.delete('iat') + jwt_token.not_before = payload.delete('nbf') + jwt_token.expire_time = payload.delete('exp') + + payload.each do |key, value| + jwt_token[key] = value + end + end + rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex + # we want to log and return on expired and errored tokens + Gitlab::ErrorTracking.track_exception(ex) + nil + end + + def secret + OpenSSL::HMAC.hexdigest( + HMAC_ALGORITHM, + ::Settings.attr_encrypted_db_key_base, + HMAC_KEY + ) + end + end + + def initialize + super(self.class.secret) + self.expire_time = self.issued_at + HMAC_EXPIRES_IN.to_i + end + + def ==(other) + self.id == other.id && + self.payload == other.payload + end + + def issued_at=(value) + super(convert_time(value)) + end + + def not_before=(value) + super(convert_time(value)) + end + + def expire_time=(value) + super(convert_time(value)) + end + + private + + def convert_time(value) + # JSONWebToken::Token truncates subsecond precision causing comparisons to + # fail unless we truncate it here first + value = value.to_i if value.is_a?(Float) + value = Time.zone.at(value) if value.is_a?(Integer) + value + end + end +end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 7a674cb5c21..7b2c792ebca 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -3,6 +3,7 @@ module Gitlab module Kas INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request' + VERSION_FILE = 'GITLAB_KAS_VERSION' JWT_ISSUER = 'gitlab-kas' include JwtAuthenticatable @@ -29,6 +30,27 @@ module Gitlab Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml) end + + # Return GitLab KAS version + # + # @return [String] version + def version + @_version ||= Rails.root.join(VERSION_FILE).read.chomp + end + + # Return GitLab KAS external_url + # + # @return [String] external_url + def external_url + Gitlab.config.gitlab_kas.external_url + end + + # Return whether GitLab KAS is enabled + # + # @return [Boolean] external_url + def enabled? + !!Gitlab.config['gitlab_kas']&.fetch('enabled', false) + end end end end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index a25f005d81e..6caebf445e5 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -116,7 +116,7 @@ module Gitlab { status: :authentication_failure, connection_error: :authentication_error } rescue Kubeclient::HttpError => e { status: kubeclient_error_status(e.message), connection_error: :http_error } - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id) { status: :unknown_failure, connection_error: :unknown_error } diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index a17e3b1ad5c..fc5834613fd 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -94,7 +94,7 @@ module Gitlab labels.each do |raw| gh_label = LabelFormatter.new(project, raw) gh_label.create! - rescue => e + rescue StandardError => e errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end @@ -107,7 +107,7 @@ module Gitlab milestones.each do |raw| gh_milestone = MilestoneFormatter.new(project, raw) gh_milestone.create! - rescue => e + rescue StandardError => e errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end @@ -128,7 +128,7 @@ module Gitlab end apply_labels(issuable, raw) - rescue => e + rescue StandardError => e errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end @@ -153,7 +153,7 @@ module Gitlab if project.gitea_import? apply_labels(merge_request, raw) end - rescue => e + rescue StandardError => e errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure clean_up_restored_branches(gh_pull_request) @@ -236,7 +236,7 @@ module Gitlab next unless issuable issuable.notes.create!(comment.attributes) - rescue => e + rescue StandardError => e errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } end end @@ -280,7 +280,7 @@ module Gitlab releases.each do |raw| gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? - rescue => e + rescue StandardError => e errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end diff --git a/lib/gitlab/legacy_github_import/label_formatter.rb b/lib/gitlab/legacy_github_import/label_formatter.rb index 0b6e4612843..415b1b8878f 100644 --- a/lib/gitlab/legacy_github_import/label_formatter.rb +++ b/lib/gitlab/legacy_github_import/label_formatter.rb @@ -20,7 +20,7 @@ module Gitlab service = ::Labels::FindOrCreateService.new(nil, project, params) label = service.execute(skip_authorization: true) - raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + raise ActiveRecord::RecordInvalid, label unless label.persisted? label end diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index 825d7399190..a05e8107cad 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -43,7 +43,7 @@ module Gitlab body = Gitlab::Json.parse(rsp.body) transfer = body.fetch('transfer', 'basic') - raise UnsupportedTransferError.new(transfer.inspect) unless transfer == 'basic' + raise UnsupportedTransferError, transfer.inspect unless transfer == 'basic' body end @@ -97,7 +97,10 @@ module Gitlab end def basic_auth - return unless credentials[:auth_method] == "password" + # Some legacy credentials have a nil auth_method, which means password + # https://gitlab.com/gitlab-org/gitlab/-/issues/328674 + return unless credentials.fetch(:auth_method, 'password') == 'password' + return if credentials.empty? { username: credentials[:user], password: credentials[:password] } end diff --git a/lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb b/lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb new file mode 100644 index 00000000000..b25305382b2 --- /dev/null +++ b/lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module LocalAndRemoteStorageMigration + class ArtifactMigrater < Gitlab::LocalAndRemoteStorageMigration::BaseMigrater + private + + def items_with_files_stored_locally + ::Ci::JobArtifact.with_files_stored_locally + end + + def items_with_files_stored_remotely + ::Ci::JobArtifact.with_files_stored_remotely + end + end + end +end diff --git a/lib/gitlab/local_and_remote_storage_migration/base_migrater.rb b/lib/gitlab/local_and_remote_storage_migration/base_migrater.rb new file mode 100644 index 00000000000..f859d293e76 --- /dev/null +++ b/lib/gitlab/local_and_remote_storage_migration/base_migrater.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module LocalAndRemoteStorageMigration + class BaseMigrater + def initialize(logger = nil) + @logger = logger + end + + def migrate_to_remote_storage + logger.info('Starting transfer to remote storage') + + migrate(items_with_files_stored_locally, ObjectStorage::Store::REMOTE) + end + + def migrate_to_local_storage + logger.info('Starting transfer to local storage') + + migrate(items_with_files_stored_remotely, ObjectStorage::Store::LOCAL) + end + + private + + attr_reader :logger + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate(items, store) + items.find_each(batch_size: batch_size) do |item| # rubocop:disable CodeReuse/ActiveRecord + item.file.migrate!(store) + + log_success(item, store) + rescue StandardError => e + log_error(e, item) + end + end + + def log_success(item, store) + logger.info("Transferred #{item.class.name} ID #{item.id} of type #{item.file_type} with size #{item.size} to #{storage_label(store)} storage") + end + + def log_error(err, item) + logger.warn("Failed to transfer #{item.class.name} of type #{item.file_type} and ID #{item.id} with error: #{err.message}") + end + + def storage_label(store) + if store == ObjectStorage::Store::LOCAL + 'local' + else + 'object' + end + end + end + end +end diff --git a/lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb b/lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb new file mode 100644 index 00000000000..70437936332 --- /dev/null +++ b/lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module LocalAndRemoteStorageMigration + class PagesDeploymentMigrater < Gitlab::LocalAndRemoteStorageMigration::BaseMigrater + private + + def items_with_files_stored_locally + ::PagesDeployment.with_files_stored_locally + end + + def items_with_files_stored_remotely + ::PagesDeployment.with_files_stored_remotely + end + end + end +end diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index 3ec5f2339b5..d0702190ac0 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -2,8 +2,12 @@ module Gitlab module MarkdownCache - # Increment this number every time the renderer changes its output - CACHE_COMMONMARK_VERSION = 27 + # Increment this number every time the renderer changes its output. + # Changing this value puts strain on the database, as every row with + # cached markdown needs to be updated. As a result, this line should + # not be changed. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330313 + CACHE_COMMONMARK_VERSION = 28 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index 8f9f6d19ce8..e800fe14cf1 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -45,9 +45,12 @@ module Gitlab end # This method returns a hash with the following keys: - # - mem_objects: a number of allocated heap slots (as reflected by GC) - # - mem_mallocs: a number of malloc calls - # - mem_bytes: a number of bytes allocated with a mallocs tied to heap slots + # - mem_objects: number of allocated heap slots (as reflected by GC) + # - mem_mallocs: number of malloc calls + # - mem_bytes: number of bytes allocated by malloc for objects that did not fit + # into a heap slot + # - mem_total_bytes: number of bytes allocated for both objects consuming an object slot + # and objects that required a malloc (mem_malloc_bytes) def self.measure_thread_memory_allocations(previous) return unless available? return unless previous @@ -56,9 +59,13 @@ module Gitlab return unless current # calculate difference in a memory allocations - previous.to_h do |key, value| + result = previous.to_h do |key, value| [KEY_MAPPING.fetch(key), current[key].to_i - value] end + + result[:mem_total_bytes] = result[:mem_bytes] + result[:mem_objects] * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] + + result end def self.with_memory_allocations diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb index 07ddd315bcc..1a951172f74 100644 --- a/lib/gitlab/metrics/dashboard/errors.rb +++ b/lib/gitlab/metrics/dashboard/errors.rb @@ -33,7 +33,7 @@ module Gitlab end def panels_not_found!(opts) - raise PanelNotFoundError.new(_("No panels matching properties %{opts}") % { opts: opts }) + raise PanelNotFoundError, _("No panels matching properties %{opts}") % { opts: opts } end end end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index ee2d36621b4..c2a8a88108f 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -23,15 +23,15 @@ module Gitlab protected def missing_panel_groups! - raise Errors::LayoutError.new('Top-level key :panel_groups must be an array') + raise Errors::LayoutError, 'Top-level key :panel_groups must be an array' end def missing_panels! - raise Errors::LayoutError.new('Each "panel_group" must define an array :panels') + raise Errors::LayoutError, 'Each "panel_group" must define an array :panels' end def missing_metrics! - raise Errors::LayoutError.new('Each "panel" must define an array :metrics') + raise Errors::LayoutError, 'Each "panel" must define an array :metrics' end def for_metrics diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb index a12082b704c..2c17982d299 100644 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -39,7 +39,7 @@ module Gitlab end def error!(message) - raise Errors::DashboardProcessingError.new(message) + raise Errors::DashboardProcessingError, message end def group_url(metric) @@ -67,14 +67,14 @@ module Gitlab def query_for_metric(metric) query = metric[query_type(metric)] - raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query + raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query query end def verify_params - raise Errors::DashboardProcessingError.new(_('Cluster is required for Stages::ClusterEndpointInserter')) unless params[:cluster] - raise Errors::DashboardProcessingError.new(_('Cluster type must be specificed for Stages::ClusterEndpointInserter')) unless params[:cluster_type] + raise Errors::DashboardProcessingError, _('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster] + raise Errors::DashboardProcessingError, _('Cluster type must be specificed for Stages::ClusterEndpointInserter') unless params[:cluster_type] end end end diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb index dd85bd0beb1..d885d978524 100644 --- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb @@ -6,7 +6,7 @@ module Gitlab module Stages class MetricEndpointInserter < BaseStage def transform! - raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::MetricEndpointInserter')) unless params[:environment] + raise Errors::DashboardProcessingError, _('Environment is required for Stages::MetricEndpointInserter') unless params[:environment] for_metrics do |metric| metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) @@ -43,7 +43,7 @@ module Gitlab def query_for_metric(metric) query = metric[query_type(metric)] - raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query + raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query # We need to remove any newlines since our UrlBlocker does not allow # multiline URLs. diff --git a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb index 20e7fe477e5..b3ce0b79675 100644 --- a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb @@ -8,7 +8,7 @@ module Gitlab VARIABLE_TYPE_METRIC_LABEL_VALUES = 'metric_label_values' def transform! - raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::VariableEndpointInserter')) unless params[:environment] + raise Errors::DashboardProcessingError, _('Environment is required for Stages::VariableEndpointInserter') unless params[:environment] for_variables do |variable_name, variable| if variable.is_a?(Hash) && variable[:type] == VARIABLE_TYPE_METRIC_LABEL_VALUES diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 23d7eb67312..19a835b9fc4 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -83,7 +83,7 @@ module Gitlab end [status, headers, body] - rescue + rescue StandardError RequestsRackMiddleware.rack_uncaught_errors_count.increment raise ensure diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 7f9055fed5d..258aa93be38 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -22,7 +22,7 @@ module Gitlab def safe_sample sample - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") stop end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index c0336a4d0fb..0a0ac6c5386 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -55,4 +55,4 @@ module Gitlab end end -Gitlab::Metrics::Samplers::DatabaseSampler.prepend_if_ee('EE::Gitlab::Metrics::Samplers::DatabaseSampler') +Gitlab::Metrics::Samplers::DatabaseSampler.prepend_mod_with('Gitlab::Metrics::Samplers::DatabaseSampler') diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 0d1cd641ffe..3db3317e833 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -87,4 +87,4 @@ module Gitlab end end -Gitlab::Metrics::Subscribers::ActiveRecord.prepend_if_ee('EE::Gitlab::Metrics::Subscribers::ActiveRecord') +Gitlab::Metrics::Subscribers::ActiveRecord.prepend_mod_with('Gitlab::Metrics::Subscribers::ActiveRecord') diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index 2791a39fb16..1c7767f5ca9 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -19,7 +19,8 @@ module Gitlab :throttle_authenticated_api, :throttle_authenticated_web, :throttle_authenticated_protected_paths_api, - :throttle_authenticated_protected_paths_web + :throttle_authenticated_protected_paths_web, + :throttle_authenticated_packages_api ].freeze PAYLOAD_KEYS = [ diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 1811389a744..ee9e6f449d3 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -57,7 +57,7 @@ module Gitlab begin route = endpoint.route - rescue + rescue StandardError # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response # so we're rescuing exceptions and bailing out diff --git a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb index d16c068c3c0..1686c3324b4 100644 --- a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb +++ b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb @@ -14,9 +14,7 @@ module Gitlab end def call(env) - if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1' - env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY - end + env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY @app.call(env) end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 226ef2041b2..65c08664a2b 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -153,4 +153,4 @@ module Gitlab end end -Gitlab::Middleware::ReadOnly::Controller.prepend_if_ee('EE::Gitlab::Middleware::ReadOnly::Controller') +Gitlab::Middleware::ReadOnly::Controller.prepend_mod_with('Gitlab::Middleware::ReadOnly::Controller') diff --git a/lib/gitlab/middleware/speedscope.rb b/lib/gitlab/middleware/speedscope.rb new file mode 100644 index 00000000000..74f334d9ab3 --- /dev/null +++ b/lib/gitlab/middleware/speedscope.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class Speedscope + def initialize(app) + @app = app + end + + def call(env) + request = ActionDispatch::Request.new(env) + + return @app.call(env) unless rendering_flamegraph?(request) + + body = nil + + ::Gitlab::SafeRequestStore[:capturing_flamegraph] = true + + require 'stackprof' + + begin + flamegraph = ::StackProf.run( + mode: :wall, + raw: true, + aggregate: false, + interval: ::Gitlab::StackProf::DEFAULT_INTERVAL_US + ) do + _, _, body = @app.call(env) + end + ensure + body.close if body.respond_to?(:close) + end + + render_flamegraph(flamegraph, request) + end + + private + + def rendering_flamegraph?(request) + request.params['performance_bar'] == 'flamegraph' && ::Gitlab::PerformanceBar.allowed_for_user?(request.env['warden']&.user) + end + + def render_flamegraph(graph, request) + headers = { 'Content-Type' => 'text/html' } + path = request.env['PATH_INFO'].sub('//', '/') + + speedscope_path = ::Gitlab::Utils.append_path(::Gitlab.config.gitlab.relative_url_root, '/-/speedscope/index.html') + + html = <<~HTML + <!DOCTYPE html> + <html> + <head> + <style> + body { margin: 0; height: 100vh; } + #speedscope-iframe { width: 100%; height: 100%; border: none; } + </style> + </head> + <body> + <script type="text/javascript" nonce="#{request.content_security_policy_nonce}"> + var graph = #{Gitlab::Json.generate(graph)}; + var json = JSON.stringify(graph); + var blob = new Blob([json], { type: 'text/plain' }); + var objUrl = encodeURIComponent(URL.createObjectURL(blob)); + var iframe = document.createElement('IFRAME'); + iframe.setAttribute('id', 'speedscope-iframe'); + document.body.appendChild(iframe); + var iframeUrl = '#{speedscope_path}#profileURL=' + objUrl + '&title=' + 'Flamegraph for #{CGI.escape(path)}'; + iframe.setAttribute('src', iframeUrl); + </script> + </body> + </html> + HTML + + [200, headers, [html]] + end + end + end +end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index 002171854ad..87cc0a0d3d2 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -5,7 +5,7 @@ module Gitlab attr_reader :first_collection, :second_collection, :per_page def initialize(*collections, per_page: nil) - raise ArgumentError.new('Only 2 collections are supported') if collections.size != 2 + raise ArgumentError, 'Only 2 collections are supported' if collections.size != 2 @per_page = (per_page || Kaminari.config.default_per_page).to_i @first_collection, @second_collection = collections diff --git a/lib/gitlab/nav/top_nav_menu_builder.rb b/lib/gitlab/nav/top_nav_menu_builder.rb new file mode 100644 index 00000000000..721ae1889b8 --- /dev/null +++ b/lib/gitlab/nav/top_nav_menu_builder.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavMenuBuilder + def initialize + @primary = [] + @secondary = [] + end + + def add_primary_menu_item(**args) + add_menu_item(dest: @primary, **args) + end + + def add_secondary_menu_item(**args) + add_menu_item(dest: @secondary, **args) + end + + def build + { + primary: @primary, + secondary: @secondary + } + end + + private + + def add_menu_item(dest:, **args) + item = ::Gitlab::Nav::TopNavMenuItem.build(**args) + + dest.push(item) + end + end + end +end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb new file mode 100644 index 00000000000..ee11f1f4560 --- /dev/null +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavMenuItem + # We want to have all keyword arguments for type safety. + # Ordinarily we could introduce a params object, but that's kind of what + # this is already :/. We could also take a hash and manually check every + # entry, but it's much more maintainable to do rely on native Ruby. + # rubocop: disable Metrics/ParameterLists + def self.build(id:, title:, active: false, icon: '', href: '', method: nil, view: '', css_class: '', data: {}) + { + id: id, + title: title, + active: active, + icon: icon, + href: href, + method: method, + view: view.to_s, + css_class: css_class, + data: data + } + end + # rubocop: enable Metrics/ParameterLists + end + end +end diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb new file mode 100644 index 00000000000..60f5b267071 --- /dev/null +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavViewModelBuilder + def initialize + @menu_builder = ::Gitlab::Nav::TopNavMenuBuilder.new + @views = {} + end + + delegate :add_primary_menu_item, :add_secondary_menu_item, to: :@menu_builder + + def add_view(name, props) + @views[name] = props + end + + def build + menu = @menu_builder.build + + menu.merge({ + views: @views, + activeTitle: _('Menu') + }) + end + end + end +end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 9a74266693b..e6e7d97d296 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -7,18 +7,19 @@ module Gitlab class ObjectHierarchy DEPTH_COLUMN = :depth - attr_reader :ancestors_base, :descendants_base, :model, :options + attr_reader :ancestors_base, :descendants_base, :model, :options, :unscoped_model # ancestors_base - An instance of ActiveRecord::Relation for which to # get parent objects. # descendants_base - An instance of ActiveRecord::Relation for which to # get child objects. If omitted, ancestors_base is used. def initialize(ancestors_base, descendants_base = ancestors_base, options: {}) - raise ArgumentError.new("Model of ancestors_base does not match model of descendants_base") if ancestors_base.model != descendants_base.model + raise ArgumentError, "Model of ancestors_base does not match model of descendants_base" if ancestors_base.model != descendants_base.model @ancestors_base = ancestors_base @descendants_base = descendants_base @model = ancestors_base.model + @unscoped_model = @model.unscoped @options = options end @@ -70,23 +71,23 @@ module Gitlab # if hierarchy_order is given, the calculated `depth` should be present in SELECT if expose_depth - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct - read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all).distinct + read_only(unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) else - recursive_query = base_and_ancestors_cte(upto).apply_to(model.all) + recursive_query = base_and_ancestors_cte(upto).apply_to(unscoped_model.all) if skip_ordering? recursive_query = recursive_query.distinct else recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) + recursive_query = unscoped_model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) end read_only(recursive_query) end else - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(unscoped_model.all) recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order read_only(recursive_query) end @@ -103,23 +104,23 @@ module Gitlab if use_distinct? # Always calculate `depth`, remove it later if with_depth is false if with_depth - base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct - read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) + base_cte = base_and_descendants_cte(with_depth: true).apply_to(unscoped_model.all).distinct + read_only(unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) else - base_cte = base_and_descendants_cte.apply_to(model.all) + base_cte = base_and_descendants_cte.apply_to(unscoped_model.all) if skip_ordering? base_cte = base_cte.distinct else base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct - base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) + base_cte = unscoped_model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) end read_only(base_cte) end else - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) + read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(unscoped_model.all)) end end # rubocop: enable CodeReuse/ActiveRecord @@ -154,16 +155,15 @@ module Gitlab ancestors_table = ancestors.alias_to(objects_table) descendants_table = descendants.alias_to(objects_table) - ancestors_scope = model.unscoped.from(ancestors_table) - descendants_scope = model.unscoped.from(descendants_table) + ancestors_scope = unscoped_model.from(ancestors_table) + descendants_scope = unscoped_model.from(descendants_table) if use_distinct? ancestors_scope = ancestors_scope.distinct descendants_scope = descendants_scope.distinct end - relation = model - .unscoped + relation = unscoped_model .with .recursive(ancestors.to_arel, descendants.to_arel) .from_union([ @@ -215,7 +215,7 @@ module Gitlab cte << base_query # Recursively get all the ancestors of the base set. - parent_query = model + parent_query = unscoped_model .from(from_tables(cte)) .where(ancestor_conditions(cte)) .except(:order) @@ -248,7 +248,7 @@ module Gitlab cte << base_query # Recursively get all the descendants of the base set. - descendants_query = model + descendants_query = unscoped_model .from(from_tables(cte)) .where(descendant_conditions(cte)) .except(:order) @@ -298,4 +298,4 @@ module Gitlab end end -Gitlab::ObjectHierarchy.prepend_if_ee('EE::Gitlab::ObjectHierarchy') +Gitlab::ObjectHierarchy.prepend_mod_with('Gitlab::ObjectHierarchy') diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 541f9b06842..3e14e1789bb 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -123,4 +123,4 @@ module Gitlab end end -Gitlab::OmniauthInitializer.prepend_if_ee('::EE::Gitlab::OmniauthInitializer') +Gitlab::OmniauthInitializer.prepend_mod_with('Gitlab::OmniauthInitializer') diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index 1d3200aa099..b65c8613d00 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -32,8 +32,8 @@ module Gitlab def rotate!(old_key:, new_key:) old_key ||= Gitlab::Application.secrets.otp_key_base - raise ArgumentError.new("Old key is the same as the new key") if old_key == new_key - raise ArgumentError.new("New key is too short! Must be 256 bits") if new_key.size < 64 + raise ArgumentError, "Old key is the same as the new key" if old_key == new_key + raise ArgumentError, "New key is too short! Must be 256 bits" if new_key.size < 64 write_csv do |csv| ActiveRecord::Base.transaction do diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb deleted file mode 100644 index 8f8667fafd9..00000000000 --- a/lib/gitlab/pages/migration_helper.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Pages - class MigrationHelper - def initialize(logger = nil) - @logger = logger - end - - def migrate_to_remote_storage - deployments = ::PagesDeployment.with_files_stored_locally - migrate(deployments, ObjectStorage::Store::REMOTE) - end - - def migrate_to_local_storage - deployments = ::PagesDeployment.with_files_stored_remotely - migrate(deployments, ObjectStorage::Store::LOCAL) - end - - private - - def batch_size - ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i - end - - def migrate(deployments, store) - deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord - deployment.file.migrate!(store) - - log_success(deployment, store) - rescue => e - log_error(e, deployment) - end - end - - def log_success(deployment, store) - logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage") - end - - def log_error(err, deployment) - logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}") - end - - def storage_label(store) - if store == ObjectStorage::Store::LOCAL - 'local' - else - 'object' - end - end - end - end -end diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index be71018e851..b35683c9dec 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -11,10 +11,6 @@ module Gitlab super end - def local_store - @local_store ||= ::Gitlab::Pages::Stores::LocalStore.new(super) - end - private def disk_access_denied? @@ -25,7 +21,7 @@ module Gitlab def report_denied_disk_access raise DiskAccessDenied if disk_access_denied? - rescue => e + rescue StandardError => e ::Gitlab::ErrorTracking.track_exception(e) end end diff --git a/lib/gitlab/pages/stores/local_store.rb b/lib/gitlab/pages/stores/local_store.rb deleted file mode 100644 index 68a7ebaceff..00000000000 --- a/lib/gitlab/pages/stores/local_store.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Pages - module Stores - class LocalStore < ::SimpleDelegator - def enabled - return false unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) - - super - end - end - end - end -end diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb new file mode 100644 index 00000000000..3bc8c0bf616 --- /dev/null +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class Iterator + def initialize(scope:, use_union_optimization: false) + @scope = scope + @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + @use_union_optimization = use_union_optimization + end + + # rubocop: disable CodeReuse/ActiveRecord + def each_batch(of: 1000) + cursor_attributes = {} + + loop do + current_scope = scope.dup.limit(of) + relation = order + .apply_cursor_conditions(current_scope, cursor_attributes, { use_union_optimization: @use_union_optimization }) + .reorder(order) + .limit(of) + + yield relation + + last_record = relation.last + break unless last_record + + cursor_attributes = order.cursor_attributes_for_node(last_record) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + attr_reader :scope, :order + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index e596e1bac9d..cef3a7b291a 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -135,7 +135,7 @@ module Gitlab # # (id < 3 AND created_at IS NULL) OR (created_at IS NOT NULL) def build_where_values(values) - return if values.blank? + return [] if values.blank? verify_incoming_values!(values) @@ -156,13 +156,26 @@ module Gitlab end end - build_or_query(where_values) + where_values + end + + def where_values_with_or_query(values) + build_or_query(build_where_values(values.with_indifferent_access)) end # rubocop: disable CodeReuse/ActiveRecord - def apply_cursor_conditions(scope, values = {}) + def apply_cursor_conditions(scope, values = {}, options = { use_union_optimization: false }) + values ||= {} + transformed_values = values.with_indifferent_access scope = apply_custom_projections(scope) - scope.where(build_where_values(values.with_indifferent_access)) + + where_values = build_where_values(transformed_values) + + if options[:use_union_optimization] && where_values.size > 1 + build_union_query(scope, where_values).reorder(self) + else + scope.where(build_or_query(where_values)) # rubocop: disable CodeReuse/ActiveRecord + end end # rubocop: enable CodeReuse/ActiveRecord @@ -170,6 +183,8 @@ module Gitlab self.class.build(column_definitions.map(&:reverse)) end + alias_method :to_sql, :to_s + private # Adds extra columns to the SELECT clause @@ -210,11 +225,19 @@ module Gitlab end def build_or_query(expressions) - or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) } + return [] if expressions.blank? + or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) } Arel::Nodes::Grouping.new(or_expression) end + def build_union_query(scope, where_values) + scopes = where_values.map do |where_value| + scope.dup.where(where_value).reorder(self) # rubocop: disable CodeReuse/ActiveRecord + end + scope.model.from_union(scopes, remove_duplicates: false, remove_order: false) + end + def to_sql_literal(column_definitions) column_definitions.map do |column_definition| if column_definition.order_expression.respond_to?(:to_sql) diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb new file mode 100644 index 00000000000..5ac5737c3be --- /dev/null +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # This class transforms the `order()` values from an Activerecord scope into a + # Gitlab::Pagination::Keyset::Order instance so the query later can be used in + # keyset pagination. + # + # Return values: + # [transformed_scope, true] # true indicates that the new scope was successfully built + # [orginal_scope, false] # false indicates that the order values are not supported in this class + class SimpleOrderBuilder + def self.build(scope) + new(scope: scope).build + end + + def initialize(scope:) + @scope = scope + @order_values = scope.order_values + @model_class = scope.model + @arel_table = @model_class.arel_table + @primary_key = @model_class.primary_key + end + + def build + order = if order_values.empty? + primary_key_descending_order + elsif ordered_by_primary_key? + primary_key_order + elsif ordered_by_other_column? + column_with_tie_breaker_order + elsif ordered_by_other_column_with_tie_breaker? + tie_breaker_attribute = order_values.second + + tie_breaker_column_order = Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: tie_breaker_attribute + ) + + column_with_tie_breaker_order(tie_breaker_column_order) + end + + order ? [scope.reorder!(order), true] : [scope, false] # [scope, success] + end + + private + + attr_reader :scope, :order_values, :model_class, :arel_table, :primary_key + + def primary_key_descending_order + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: arel_table[primary_key].desc + ) + ]) + end + + def primary_key_order + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: order_values.first + ) + ]) + end + + def column_with_tie_breaker_order(tie_breaker_column_order = default_tie_breaker_column_order) + order_expression = order_values.first + attribute_name = order_expression.expr.name + + column_nullable = model_class.columns.find { |column| column.name == attribute_name }.null + + nullable = if column_nullable && order_expression.is_a?(Arel::Nodes::Ascending) + :nulls_last + elsif column_nullable && order_expression.is_a?(Arel::Nodes::Descending) + :nulls_first + else + :not_nullable + end + + Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: attribute_name, + order_expression: order_expression, + nullable: nullable, + distinct: false + ), + tie_breaker_column_order + ]) + end + + def ordered_by_primary_key? + return unless order_values.one? + + attribute = order_values.first.try(:expr) + + return unless attribute + + arel_table[primary_key].to_s == attribute.to_s + end + + def ordered_by_other_column? + return unless order_values.one? + + attribute = order_values.first.try(:expr) + + return unless attribute + return unless attribute.try(:name) + + model_class.column_names.include?(attribute.name.to_s) + end + + def ordered_by_other_column_with_tie_breaker? + return unless order_values.size == 2 + + attribute = order_values.first.try(:expr) + tie_breaker_attribute = order_values.second.try(:expr) + + return unless attribute + return unless tie_breaker_attribute + + model_class.column_names.include?(attribute.name.to_s) && + arel_table[primary_key].to_s == tie_breaker_attribute.to_s + end + + def default_tie_breaker_column_order + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: model_class.primary_key, + order_expression: arel_table[primary_key].desc + ) + end + end + end + end +end diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb index f5fcd5c6093..61b25065e8f 100644 --- a/lib/gitlab/patch/draw_route.rb +++ b/lib/gitlab/patch/draw_route.rb @@ -10,7 +10,7 @@ module Gitlab def draw(routes_name) drawn_any = draw_ee(routes_name) | draw_ce(routes_name) - drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}")) + drawn_any || raise(RoutesNotFound, "Cannot find #{routes_name}") end def draw_ce(routes_name) @@ -37,4 +37,4 @@ module Gitlab end end -Gitlab::Patch::DrawRoute.prepend_if_ee('EE::Gitlab::Patch::DrawRoute') +Gitlab::Patch::DrawRoute.prepend_mod_with('Gitlab::Patch::DrawRoute') diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index dde78cd9178..1ed341e1c26 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -21,7 +21,12 @@ module Gitlab def prepend_features(base) return false if prepended?(base) - super + # Rails 6.1 allows prepending of the modules, but it doesn't + # work well when both modules extend ActiveSupport::Concern + # https://github.com/rails/rails/pull/42067 + # + # Let's keep our own implementation, until the issue is fixed + Module.instance_method(:prepend_features).bind(self).call(base) if const_defined?(:ClassMethods) klass_methods = const_get(:ClassMethods, false) diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 2ff23980ebd..8618d2da77c 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -288,4 +288,4 @@ module Gitlab end end -Gitlab::PathRegex.prepend_if_ee('EE::Gitlab::PathRegex') +Gitlab::PathRegex.prepend_mod_with('Gitlab::PathRegex') diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e26309b5dfd..10b13e7f55f 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -7,10 +7,10 @@ module Gitlab EXPIRY_TIME_L2_CACHE = 5.minutes def self.enabled_for_request? - Gitlab::SafeRequestStore[:peek_enabled] + !Gitlab::SafeRequestStore[:capturing_flamegraph] && Gitlab::SafeRequestStore[:peek_enabled] end - def self.enabled_for_user?(user = nil) + def self.allowed_for_user?(user = nil) return true if Rails.env.development? return true if user&.admin? return false unless user && allowed_group_id diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index c2a4602fd16..103cd65cb4b 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -20,7 +20,7 @@ module Gitlab return unless data log_sql_queries(id, data) - rescue => err + rescue StandardError => err logger.error(message: "failed to process request id #{id}: #{err.message}") end diff --git a/lib/gitlab/phabricator_import/conduit/client.rb b/lib/gitlab/phabricator_import/conduit/client.rb index 4469a3f5849..5945cde9618 100644 --- a/lib/gitlab/phabricator_import/conduit/client.rb +++ b/lib/gitlab/phabricator_import/conduit/client.rb @@ -13,7 +13,7 @@ module Gitlab Response.parse!(response) rescue *Gitlab::HTTP::HTTP_ERRORS => e # Wrap all errors from the API into an API-error. - raise ApiError.new(e) + raise ApiError, e end private diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb index 1b03cfa05e6..26037ba183e 100644 --- a/lib/gitlab/phabricator_import/conduit/response.rb +++ b/lib/gitlab/phabricator_import/conduit/response.rb @@ -18,7 +18,7 @@ module Gitlab response rescue JSON::JSONError => e - raise ResponseError.new(e) + raise ResponseError, e end def initialize(json) diff --git a/lib/gitlab/phabricator_import/importer.rb b/lib/gitlab/phabricator_import/importer.rb index ac85b96de08..0666fa0df01 100644 --- a/lib/gitlab/phabricator_import/importer.rb +++ b/lib/gitlab/phabricator_import/importer.rb @@ -22,7 +22,7 @@ module Gitlab schedule_first_tasks_page true - rescue => e + rescue StandardError => e fail_import(e.message) false diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 32d3eeb8cd2..8875e6320c7 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -87,4 +87,4 @@ module Gitlab end end -Gitlab::ProjectTemplate.prepend_if_ee('EE::Gitlab::ProjectTemplate') +Gitlab::ProjectTemplate.prepend_mod_with('Gitlab::ProjectTemplate') diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index 76e65d29c7a..45438d9bf7c 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -19,13 +19,11 @@ module Gitlab end def cluster_prometheus_adapter - if cluster&.integration_prometheus - return cluster.integration_prometheus - end - application = cluster&.application_prometheus + return application if application&.available? - application if application&.available? + integration = cluster&.integration_prometheus + integration if integration&.available? end private diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index ee3d98f3602..f5eb27b6916 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -14,7 +14,7 @@ module Gitlab private def validate!(obj) - raise ParsingError.new(obj.errors.full_messages.join('\n')) unless obj.valid? + raise ParsingError, obj.errors.full_messages.join('\n') unless obj.valid? end def group_from_entry(entry) diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 4a39260a340..020d4cf74a3 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -31,4 +31,4 @@ module Gitlab end end -Gitlab::Prometheus::MetricGroup.prepend_if_ee('EE::Gitlab::Prometheus::MetricGroup') +Gitlab::Prometheus::MetricGroup.prepend_mod_with('Gitlab::Prometheus::MetricGroup') diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index d24b98e790b..a870bb6bc5f 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -98,4 +98,4 @@ module Gitlab end end -Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_if_ee('EE::Gitlab::Prometheus::Queries::QueryAdditionalMetrics') +Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_mod_with('Gitlab::Prometheus::Queries::QueryAdditionalMetrics') diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 0fcf63d03fc..8182dbad4f8 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -47,7 +47,7 @@ module Gitlab # From Prometheus docs: This endpoint returns 200 when Prometheus is ready to serve traffic (i.e. respond to queries). response.code == 200 - rescue => e + rescue StandardError => e raise PrometheusClient::UnexpectedResponseError, "#{e.message}" end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 012e495502f..ff17ecf8024 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -267,7 +267,7 @@ module Gitlab private def zoom_link_service - Issues::ZoomLinkService.new(quick_action_target, current_user) + Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 6a404c34044..f3c6315cd6a 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -148,7 +148,7 @@ module Gitlab quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) end command :approve do - success = MergeRequests::ApprovalService.new(quick_action_target.project, current_user).execute(quick_action_target) + success = MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) next unless success diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 4a62e83e8e9..03b2a1086bb 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -19,7 +19,7 @@ module Gitlab def execute return if @spend_arg.blank? - return [get_time, DateTime.now.to_date] unless date_present? + return [get_time, DateTime.current] unless date_present? return unless valid_date? [get_time, get_date] diff --git a/lib/gitlab/quick_actions/substitution_definition.rb b/lib/gitlab/quick_actions/substitution_definition.rb index 24b4e3c62b3..2cc4a6d90ce 100644 --- a/lib/gitlab/quick_actions/substitution_definition.rb +++ b/lib/gitlab/quick_actions/substitution_definition.rb @@ -13,7 +13,7 @@ module Gitlab return unless content all_names.each do |a_name| - content = content.sub(%r{/#{a_name}(?![\S]) ?(.*)$}i, execute_block(action_block, context, '\1')) + content = content.sub(%r{/#{a_name}(?!\S) ?(.*)$}i, execute_block(action_block, context, '\1')) end content diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index ae3c89c3565..175f32bd4c6 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -83,16 +83,13 @@ module Gitlab def self.configure_throttles(rack_attack) throttle_or_track(rack_attack, 'throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req| - if !req.should_be_skipped? && - Gitlab::Throttle.settings.throttle_unauthenticated_enabled && - req.unauthenticated? + if req.throttle_unauthenticated? req.ip end end throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req| - if req.api_request? && - Gitlab::Throttle.settings.throttle_authenticated_api_enabled + if req.throttle_authenticated_api? req.throttled_user_id([:api]) end end @@ -107,40 +104,41 @@ module Gitlab end throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req| - if req.web_request? && - Gitlab::Throttle.settings.throttle_authenticated_web_enabled + if req.throttle_authenticated_web? req.throttled_user_id([:api, :rss, :ics]) end end throttle_or_track(rack_attack, 'throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req| - if req.post? && - !req.should_be_skipped? && - req.protected_path? && - Gitlab::Throttle.protected_paths_enabled? && - req.unauthenticated? + if req.throttle_unauthenticated_protected_paths? req.ip end end throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req| - if req.post? && - req.api_request? && - req.protected_path? && - Gitlab::Throttle.protected_paths_enabled? + if req.throttle_authenticated_protected_paths_api? req.throttled_user_id([:api]) end end throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req| - if req.post? && - req.web_request? && - req.protected_path? && - Gitlab::Throttle.protected_paths_enabled? + if req.throttle_authenticated_protected_paths_web? req.throttled_user_id([:api, :rss, :ics]) end end + throttle_or_track(rack_attack, 'throttle_unauthenticated_packages_api', Gitlab::Throttle.unauthenticated_packages_api_options) do |req| + if req.throttle_unauthenticated_packages_api? + req.ip + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_packages_api', Gitlab::Throttle.authenticated_packages_api_options) do |req| + if req.throttle_authenticated_packages_api? + req.throttled_user_id([:api]) + end + end + rack_attack.safelist('throttle_bypass_header') do |req| Gitlab::Throttle.bypass_header.present? && req.get_header(Gitlab::Throttle.bypass_header) == '1' @@ -173,4 +171,4 @@ module Gitlab end end end -::Gitlab::RackAttack.prepend_if_ee('::EE::Gitlab::RackAttack') +::Gitlab::RackAttack.prepend_mod_with('Gitlab::RackAttack') diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index bd6d2e016b4..7fee6a1b43d 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -58,6 +58,57 @@ module Gitlab path =~ protected_paths_regex end + def throttle_unauthenticated? + !should_be_skipped? && + !throttle_unauthenticated_packages_api? && + Gitlab::Throttle.settings.throttle_unauthenticated_enabled && + unauthenticated? + end + + def throttle_authenticated_api? + api_request? && + !throttle_authenticated_packages_api? && + Gitlab::Throttle.settings.throttle_authenticated_api_enabled + end + + def throttle_authenticated_web? + web_request? && + Gitlab::Throttle.settings.throttle_authenticated_web_enabled + end + + def throttle_unauthenticated_protected_paths? + post? && + !should_be_skipped? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + unauthenticated? + end + + def throttle_authenticated_protected_paths_api? + post? && + api_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? + end + + def throttle_authenticated_protected_paths_web? + post? && + web_request? && + protected_path? && + Gitlab::Throttle.protected_paths_enabled? + end + + def throttle_unauthenticated_packages_api? + packages_api_path? && + Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && + unauthenticated? + end + + def throttle_authenticated_packages_api? + packages_api_path? && + Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled + end + private def authenticated_user_id(request_formats) @@ -75,7 +126,11 @@ module Gitlab def protected_paths_regex Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ }) end + + def packages_api_path? + path =~ ::Gitlab::Regex::Packages::API_PATH_REGEX + end end end end -::Gitlab::RackAttack::Request.prepend_if_ee('::EE::Gitlab::RackAttack::Request') +::Gitlab::RackAttack::Request.prepend_mod_with('Gitlab::RackAttack::Request') diff --git a/lib/gitlab/redis/boolean.rb b/lib/gitlab/redis/boolean.rb index 9b0b20fc2be..cd0877c5b13 100644 --- a/lib/gitlab/redis/boolean.rb +++ b/lib/gitlab/redis/boolean.rb @@ -50,7 +50,7 @@ module Gitlab # @return [String] the encoded boolean # @raise [NotABooleanError] if the value isn't true or false def encode(value) - raise NotABooleanError.new(value) unless bool?(value) + raise NotABooleanError, value unless bool?(value) [LABEL, to_string(value)].join(DELIMITER) end @@ -61,11 +61,11 @@ module Gitlab # @return [Boolean] true or false # @raise [NotAnEncodedBooleanStringError] if the provided value isn't an encoded boolean def decode(value) - raise NotAnEncodedBooleanStringError.new(value.class) unless value.is_a?(String) + raise NotAnEncodedBooleanStringError, value.class unless value.is_a?(String) label, bool_str = *value.split(DELIMITER, 2) - raise NotAnEncodedBooleanStringError.new(label) unless label == LABEL + raise NotAnEncodedBooleanStringError, label unless label == LABEL from_string(bool_str) end @@ -99,7 +99,7 @@ module Gitlab end def from_string(str) - raise NotAnEncodedBooleanStringError.new(str) unless [TRUE_STR, FALSE_STR].include?(str) + raise NotAnEncodedBooleanStringError, str unless [TRUE_STR, FALSE_STR].include?(str) str == TRUE_STR end diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb index 010a6b59da5..0d04545688b 100644 --- a/lib/gitlab/redis/hll.rb +++ b/lib/gitlab/redis/hll.rb @@ -46,7 +46,7 @@ module Gitlab def validate_key!(key) return if KEY_REGEX.match?(key) - raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") + raise KeyFormatError, "Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 6f80c7d439f..94ab67ef08a 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -142,7 +142,7 @@ module Gitlab def fetch_config return false unless self.class._raw_config - yaml = YAML.load(self.class._raw_config) + yaml = YAML.safe_load(self.class._raw_config, aliases: true) # If the file has content but it's invalid YAML, `load` returns false if yaml diff --git a/lib/gitlab/reference_counter.rb b/lib/gitlab/reference_counter.rb index c2fa2e1330a..f41e42b9e9c 100644 --- a/lib/gitlab/reference_counter.rb +++ b/lib/gitlab/reference_counter.rb @@ -84,7 +84,7 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| yield(redis) } true - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("GitLab: An unexpected error occurred in writing to Redis: #{e}") false diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 488ba04f87c..ccb4f6e1097 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -6,6 +6,8 @@ module Gitlab CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze + API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze + def conan_package_reference_regex @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze end @@ -75,6 +77,10 @@ module Gitlab /x.freeze end + def terraform_module_package_name_regex + @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze + end + def pypi_version_regex # See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159 @@ -123,6 +129,18 @@ module Gitlab @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze end + def helm_channel_regex + @helm_channel_regex ||= %r{\A[-\.\_a-zA-Z0-9]+\z}.freeze + end + + def helm_package_regex + @helm_package_regex ||= %r{#{helm_channel_regex}}.freeze + end + + def helm_version_regex + @helm_version_regex ||= %r{#{prefixed_semver_regex}}.freeze + end + def unbounded_semver_regex # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -135,7 +153,7 @@ module Gitlab end def semver_regex - @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze end # These partial semver regexes are intended for use in composing other @@ -235,7 +253,7 @@ module Gitlab # used as a routing constraint. # def container_registry_tag_regex - @container_registry_tag_regex ||= /[\w][\w.-]{0,127}/ + @container_registry_tag_regex ||= /\w[\w.-]{0,127}/ end def environment_name_regex_chars diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb index e2cbe4b2de0..c2a73b7cfe5 100644 --- a/lib/gitlab/relative_positioning.rb +++ b/lib/gitlab/relative_positioning.rb @@ -13,7 +13,9 @@ module Gitlab MIN_GAP = 2 NoSpaceLeft = Class.new(StandardError) + InvalidPosition = Class.new(StandardError) IllegalRange = Class.new(ArgumentError) + IssuePositioningDisabled = Class.new(StandardError) def self.range(lhs, rhs) if lhs && rhs diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index 8f5495ece5e..1e738aef9b0 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -129,6 +129,14 @@ module Gitlab neighbour(sib) end + def at_position(position) + item = scoped_items.find_by(relative_position: position) + + raise InvalidPosition, 'No item found at the specified position' if item.nil? + + neighbour(item) + end + def shift_left move_sequence_before(true) object.reset_relative_position diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 46c84107e0f..42b94d5cf3b 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -110,4 +110,4 @@ module Gitlab end end -Gitlab::RepoPath.singleton_class.prepend_if_ee('EE::Gitlab::RepoPath::ClassMethods') +Gitlab::RepoPath.singleton_class.prepend_mod_with('Gitlab::RepoPath::ClassMethods') diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index 0ed31176dd8..2afc5e8d668 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -56,4 +56,4 @@ module Gitlab end end -Gitlab::RepositorySizeChecker.prepend_if_ee('EE::Gitlab::RepositorySizeChecker') +Gitlab::RepositorySizeChecker.prepend_mod_with('Gitlab::RepositorySizeChecker') diff --git a/lib/gitlab/repository_url_builder.rb b/lib/gitlab/repository_url_builder.rb index a2d0d50d20b..ed9a298ee8c 100644 --- a/lib/gitlab/repository_url_builder.rb +++ b/lib/gitlab/repository_url_builder.rb @@ -10,7 +10,7 @@ module Gitlab when :http http_url(path) else - raise NotImplementedError.new("No URL builder defined for protocol #{protocol}") + raise NotImplementedError, "No URL builder defined for protocol #{protocol}" end end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 7050aee3847..acdf8d4541f 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -90,7 +90,7 @@ module Gitlab File.open(file_path, 'wb') do |file| yield(file) end - rescue + rescue StandardError FileUtils.rm(file_path) end end diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb index a555bf1d812..65c7f4b39e5 100644 --- a/lib/gitlab/route_map.rb +++ b/lib/gitlab/route_map.rb @@ -7,7 +7,7 @@ module Gitlab def initialize(data) begin entries = YAML.safe_load(data) - rescue + rescue StandardError raise FormatError, 'Route map is not valid YAML' end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index cad127922df..fd9fb8ab7e2 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -30,7 +30,7 @@ module Gitlab rescue URI::InvalidURIError => e # If url is invalid, raise custom error, # which can be ignored by monitoring tools. - raise ActionController::RoutingError.new(e.message) + raise ActionController::RoutingError, e.message end end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 968ef06b085..b0bcea0ca69 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -26,13 +26,9 @@ module Gitlab if matches.one? matches.first elsif matches.none? - raise UnknownProcessError.new( - "Failed to identify runtime for process #{Process.pid} (#{$0})" - ) + raise UnknownProcessError, "Failed to identify runtime for process #{Process.pid} (#{$0})" else - raise AmbiguousProcessError.new( - "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" - ) + raise AmbiguousProcessError, "Ambiguous runtime #{matches} for process #{Process.pid} (#{$0})" end end @@ -91,7 +87,7 @@ module Gitlab def max_threads threads = 1 # main thread - if puma? + if puma? && Puma.respond_to?(:cli_config) threads += Puma.cli_config.options[:max_threads] elsif sidekiq? # An extra thread for the poller in Sidekiq Cron: diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index eec50deb61e..f607aff9d29 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -71,7 +71,7 @@ module Gitlab relation.find_each(**find_params) do |upload| clean(upload.retrieve_uploader, dry_run: dry_run) sleep sleep_time if sleep_time - rescue => err + rescue StandardError => err logger.error "failed to sanitize #{upload_ref(upload)}: #{err.message}" logger.debug err.backtrace.join("\n ") end diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb index 5d5d407c172..a397ce935cb 100644 --- a/lib/gitlab/search/parsed_query.rb +++ b/lib/gitlab/search/parsed_query.rb @@ -50,11 +50,11 @@ module Gitlab when :including then including when :excluding then excluding else - raise ArgumentError.new(type) + raise ArgumentError, type end end end end end -Gitlab::Search::ParsedQuery.prepend_if_ee('EE::Gitlab::Search::ParsedQuery') +Gitlab::Search::ParsedQuery.prepend_mod_with('Gitlab::Search::ParsedQuery') diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb index 0323220690a..04ef2be87f8 100644 --- a/lib/gitlab/search_context.rb +++ b/lib/gitlab/search_context.rb @@ -164,4 +164,4 @@ module Gitlab end end -Gitlab::SearchContext::Builder.prepend_ee_mod +Gitlab::SearchContext::Builder.prepend_mod diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index d0beb74c289..678c0b396ef 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -236,4 +236,4 @@ module Gitlab end end -Gitlab::SearchResults.prepend_if_ee('EE::Gitlab::SearchResults') +Gitlab::SearchResults.prepend_mod_with('Gitlab::SearchResults') diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 3419989c110..d26e1a34a9f 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -92,7 +92,7 @@ module Gitlab Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).rename("#{new_disk_path}.git") true - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, path: disk_path, new_path: new_disk_path, storage: storage) false @@ -115,7 +115,7 @@ module Gitlab Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).remove true - rescue => e + rescue StandardError => e Gitlab::AppLogger.warn("Repository does not exist: #{e} at: #{disk_path}.git") Gitlab::ErrorTracking.track_exception(e, path: disk_path, storage: storage) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 78d45b5f3f0..16a0619daf6 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -21,8 +21,18 @@ module Gitlab # invalid class name. We keep it in the YAML file for safety, just # in case anything does get scheduled to run there. DEFAULT_WORKERS = { - '_' => DummyWorker.new('default', weight: 1, tags: []), - 'ActionMailer::MailDeliveryJob' => DummyWorker.new('mailers', feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: []) + '_' => DummyWorker.new( + queue: 'default', + weight: 1, tags: [] + ), + 'ActionMailer::MailDeliveryJob' => DummyWorker.new( + name: 'ActionMailer::MailDeliveryJob', + queue: 'mailers', + feature_category: :issue_tracking, + urgency: 'low', + weight: 2, + tags: [] + ) }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze class << self diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index 7568840410b..ef0dce0cf84 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -4,9 +4,9 @@ module Gitlab module SidekiqConfig # For queues that don't have explicit workers - default and mailers class DummyWorker - attr_accessor :queue - ATTRIBUTE_METHODS = { + queue: :queue, + name: :name, feature_category: :get_feature_category, has_external_dependencies: :worker_has_external_dependencies?, urgency: :get_urgency, @@ -16,8 +16,7 @@ module Gitlab tags: :get_tags }.freeze - def initialize(queue, attributes = {}) - @queue = queue + def initialize(attributes = {}) @attributes = attributes end diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index 46fa0aa5be1..aea4209f631 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -6,10 +6,9 @@ module Gitlab include Comparable attr_reader :klass - delegate :feature_category_not_owned?, :get_feature_category, :get_tags, - :get_urgency, :get_weight, :get_worker_resource_boundary, - :idempotent?, :queue, :queue_namespace, - :worker_has_external_dependencies?, + delegate :feature_category_not_owned?, :get_feature_category, :get_sidekiq_options, + :get_tags, :get_urgency, :get_weight, :get_worker_resource_boundary, + :idempotent?, :queue, :queue_namespace, :worker_has_external_dependencies?, to: :klass def initialize(klass, ee:) @@ -47,6 +46,7 @@ module Gitlab def to_yaml { name: queue, + worker_name: klass.name, feature_category: get_feature_category, has_external_dependencies: worker_has_external_dependencies?, urgency: get_urgency, @@ -64,6 +64,10 @@ module Gitlab def queue_and_weight [queue, get_weight] end + + def retries + get_sidekiq_options['retry'] + end end end end diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb index fe5ac10c65a..d615d5ecba4 100644 --- a/lib/gitlab/sidekiq_config/worker_matcher.rb +++ b/lib/gitlab/sidekiq_config/worker_matcher.rb @@ -10,6 +10,7 @@ module Gitlab QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze QUERY_PREDICATES = { + worker_name: :to_s, feature_category: :to_sym, has_external_dependencies: lambda { |value| value == 'true' }, name: :to_s, @@ -50,7 +51,7 @@ module Gitlab def predicate_for_term(term) match = term.match(QUERY_TERM_REGEX) - raise InvalidTerm.new("Invalid term: #{term}") unless match + raise InvalidTerm, "Invalid term: #{term}" unless match _, lhs, op, rhs = *match @@ -66,14 +67,14 @@ module Gitlab else # This is unreachable because InvalidTerm will be raised instead, but # keeping it allows to guard against that changing in future. - raise UnknownOperator.new("Unknown operator: #{op}") + raise UnknownOperator, "Unknown operator: #{op}" end end def predicate_factory(lhs, values) values_block = QUERY_PREDICATES[lhs.to_sym] - raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block + raise UnknownPredicate, "Unknown predicate: #{lhs}" unless values_block lambda do |queue| comparator = Array(queue[lhs.to_sym]).to_set diff --git a/lib/gitlab/sidekiq_config/worker_router.rb b/lib/gitlab/sidekiq_config/worker_router.rb new file mode 100644 index 00000000000..946296a24d3 --- /dev/null +++ b/lib/gitlab/sidekiq_config/worker_router.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqConfig + class WorkerRouter + InvalidRoutingRuleError = Class.new(StandardError) + RuleEvaluator = Struct.new(:matcher, :queue_name) + + def self.queue_name_from_worker_name(worker_klass) + base_queue_name = + worker_klass.name + .delete_prefix('Gitlab::') + .delete_suffix('Worker') + .underscore + .tr('/', '_') + [worker_klass.queue_namespace, base_queue_name].compact.join(':') + end + + def self.global + @global_worker_router ||= new(::Gitlab.config.sidekiq.routing_rules) + rescue InvalidRoutingRuleError, ::Gitlab::SidekiqConfig::WorkerMatcher::UnknownPredicate => e + ::Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + + @global_worker_router = new([]) + end + + # call-seq: + # router = WorkerRouter.new([ + # ["resource_boundary=cpu", 'cpu_boundary'], + # ["feature_category=pages", nil], + # ["feature_category=source_code_management", ''], + # ["*", "default"] + # ]) + # router.route(ACpuBoundaryWorker) # Return "cpu_boundary" + # router.route(JustAPagesWorker) # Return "just_a_pages_worker" + # router.route(PostReceive) # Return "post_receive" + # router.route(RandomWorker) # Return "default" + # + # This class is responsible for routing a Sidekiq worker to a certain + # queue defined in the input routing rules. The input routing rules, as + # described above, is an order-matter array of tuples [query, queue_name]. + # + # - The query syntax is the same as the "queue selector" detailedly + # denoted in doc/administration/operations/extra_sidekiq_processes.md. + # + # - The queue_name must be a valid Sidekiq queue name. If the queue name + # is nil, or an empty string, the worker is routed to the queue generated + # by the name of the worker instead. + # + # Rules are evaluated from first to last, and as soon as we find a match + # for a given worker we stop processing for that worker (first match + # wins). If the worker doesn't match any rule, it falls back the queue + # name generated from the worker name + # + # For further information, please visit: + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1016 + # + def initialize(routing_rules) + @rule_evaluators = parse_routing_rules(routing_rules) + end + + def route(worker_klass) + # A medium representation to ensure the backward-compatibility of + # WorkerMatcher + worker_metadata = generate_worker_metadata(worker_klass) + @rule_evaluators.each do |evaluator| + if evaluator.matcher.match?(worker_metadata) + return evaluator.queue_name.presence || queue_name_from_worker_name(worker_klass) + end + end + + queue_name_from_worker_name(worker_klass) + end + + private + + def parse_routing_rules(routing_rules) + raise InvalidRoutingRuleError, 'The set of routing rule must be an array' unless routing_rules.is_a?(Array) + + routing_rules.map do |rule_tuple| + raise InvalidRoutingRuleError, "Routing rule `#{rule_tuple.inspect}` is invalid" unless valid_routing_rule?(rule_tuple) + + selector, destination_queue = rule_tuple + RuleEvaluator.new( + ::Gitlab::SidekiqConfig::WorkerMatcher.new(selector), + destination_queue + ) + end + end + + def valid_routing_rule?(rule_tuple) + rule_tuple.is_a?(Array) && rule_tuple.length == 2 + end + + def generate_worker_metadata(worker_klass) + # The ee indicator here is insignificant and irrelevant to the matcher. + # Plus, it's not easy to determine whether a worker is **only** + # available in EE. + ::Gitlab::SidekiqConfig::Worker.new(worker_klass, ee: false).to_yaml + end + + def queue_name_from_worker_name(worker_klass) + self.class.queue_name_from_worker_name(worker_klass) + end + end + end +end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 8793a672693..113076a6a75 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -73,7 +73,7 @@ module Gitlab begin sleep(CHECK_INTERVAL_SECONDS) restart_sidekiq unless rss_within_range? - rescue => e + rescue StandardError => e log_exception(e, __method__) rescue Exception => e # rubocop:disable Lint/RescueException log_exception(e, __method__ ) @@ -249,7 +249,7 @@ module Gitlab def get_job_options(job, key, default) job[:worker_class].sidekiq_options.fetch(key, default) - rescue + rescue StandardError default end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index b1fb3771c78..87fb36d04e9 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -30,7 +30,7 @@ module Gitlab Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) raise - rescue => job_exception + rescue StandardError => job_exception Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) raise @@ -39,7 +39,7 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - output_payload.merge!(job[:instrumentation].stringify_keys) + output_payload.merge!(job[:instrumentation].stringify_keys) if job[:instrumentation] end def add_logging_extras!(job, output_payload) @@ -70,6 +70,8 @@ module Gitlab message = base_message(payload) + payload['database_chosen'] = job[:database_chosen] if job[:database_chosen] + if job_exception payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" payload['job_status'] = 'fail' diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 563a105484d..c5b980769f0 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -44,4 +44,4 @@ module Gitlab end end -Gitlab::SidekiqMiddleware.singleton_class.prepend_if_ee('EE::Gitlab::SidekiqMiddleware') +Gitlab::SidekiqMiddleware.singleton_class.prepend_mod_with('Gitlab::SidekiqMiddleware') diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index f5fee8050ac..474afffcf93 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -119,4 +119,4 @@ module Gitlab end end -Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics') +Gitlab::SidekiqMiddleware::ServerMetrics.prepend_mod_with('Gitlab::SidekiqMiddleware::ServerMetrics') diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb index da6c903ccae..540159e8a72 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb @@ -13,7 +13,7 @@ module Gitlab @size = size @size_limit = size_limit - super "#{@worker_class} job exceeds payload size limit (#{size}/#{size_limit})" + super "#{@worker_class} job exceeds payload size limit" end def sentry_extra_data diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb new file mode 100644 index 00000000000..62d62bf82c4 --- /dev/null +++ b/lib/gitlab/sidekiq_migrate_jobs.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + class SidekiqMigrateJobs + LOG_FREQUENCY = 1_000 + + attr_reader :sidekiq_set, :logger + + def initialize(sidekiq_set, logger: nil) + @sidekiq_set = sidekiq_set + @logger = logger + end + + # mappings is a hash of WorkerClassName => target_queue_name + def execute(mappings) + source_queues_regex = Regexp.union(mappings.keys) + cursor = 0 + scanned = 0 + migrated = 0 + + estimated_size = Sidekiq.redis { |c| c.zcard(sidekiq_set) } + logger&.info("Processing #{sidekiq_set} set. Estimated size: #{estimated_size}.") + + begin + cursor, jobs = Sidekiq.redis { |c| c.zscan(sidekiq_set, cursor) } + + jobs.each do |(job, score)| + if scanned > 0 && scanned % LOG_FREQUENCY == 0 + logger&.info("In progress. Scanned records: #{scanned}. Migrated records: #{migrated}.") + end + + scanned += 1 + + next unless job.match?(source_queues_regex) + + job_hash = Sidekiq.load_json(job) + destination_queue = mappings[job_hash['class']] + + next unless mappings.has_key?(job_hash['class']) + next if job_hash['queue'] == destination_queue + + job_hash['queue'] = destination_queue + + migrated += migrate_job(job, score, job_hash) + end + end while cursor.to_i != 0 + + logger&.info("Done. Scanned records: #{scanned}. Migrated records: #{migrated}.") + + { + scanned: scanned, + migrated: migrated + } + end + + private + + def migrate_job(job, score, job_hash) + Sidekiq.redis do |connection| + removed = connection.zrem(sidekiq_set, job) + + if removed + connection.zadd(sidekiq_set, score, Sidekiq.dump_json(job_hash)) + + 1 + else + 0 + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 2293e2adee1..623fdd89456 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -66,7 +66,7 @@ module Gitlab def self.num_running(job_ids) responses = self.job_status(job_ids) - responses.select(&:present?).count + responses.count(&:present?) end # Returns the number of jobs that have completed. diff --git a/lib/gitlab/slash_commands/issue_close.rb b/lib/gitlab/slash_commands/issue_close.rb index 5fcc86e91c4..3dad7216983 100644 --- a/lib/gitlab/slash_commands/issue_close.rb +++ b/lib/gitlab/slash_commands/issue_close.rb @@ -29,7 +29,7 @@ module Gitlab private def close_issue(issue:) - Issues::CloseService.new(project, current_user).execute(issue) + Issues::CloseService.new(project: project, current_user: current_user).execute(issue) end def presenter(issue) diff --git a/lib/gitlab/slash_commands/issue_move.rb b/lib/gitlab/slash_commands/issue_move.rb index d2f1f130b38..0612663017c 100644 --- a/lib/gitlab/slash_commands/issue_move.rb +++ b/lib/gitlab/slash_commands/issue_move.rb @@ -29,7 +29,7 @@ module Gitlab return Gitlab::SlashCommands::Presenters::Access.new.not_found end - new_issue = Issues::MoveService.new(project, current_user) + new_issue = Issues::MoveService.new(project: project, current_user: current_user) .execute(old_issue, target_project) presenter(new_issue).present(old_issue) diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 48379031537..99a056c97fc 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -33,7 +33,7 @@ module Gitlab private def create_issue(title:, description:) - Issues::CreateService.new(project, current_user, title: title, description: description).execute + Issues::CreateService.new(project: project, current_user: current_user, params: { title: title, description: description }).execute end def presenter(issue) diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb index 017fb8a62c4..f6f6e3d7fc6 100644 --- a/lib/gitlab/slash_commands/presenters/issue_base.rb +++ b/lib/gitlab/slash_commands/presenters/issue_base.rb @@ -50,4 +50,4 @@ module Gitlab end end -Gitlab::SlashCommands::Presenters::IssueBase.prepend_if_ee('EE::Gitlab::SlashCommands::Presenters::IssueBase') +Gitlab::SlashCommands::Presenters::IssueBase.prepend_mod_with('Gitlab::SlashCommands::Presenters::IssueBase') diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 41ec19f0da8..581d6b738f3 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -45,4 +45,4 @@ module Gitlab end end -Gitlab::SnippetSearchResults.prepend_if_ee('::EE::Gitlab::SnippetSearchResults') +Gitlab::SnippetSearchResults.prepend_mod_with('Gitlab::SnippetSearchResults') diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb new file mode 100644 index 00000000000..6afc21be4e0 --- /dev/null +++ b/lib/gitlab/spamcheck/client.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true +require 'spamcheck' + +module Gitlab + module Spamcheck + class Client + include ::Spam::SpamConstants + DEFAULT_TIMEOUT_SECS = 2 + + VERDICT_MAPPING = { + ::Spamcheck::SpamVerdict::Verdict::ALLOW => ALLOW, + ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW => CONDITIONAL_ALLOW, + ::Spamcheck::SpamVerdict::Verdict::DISALLOW => DISALLOW, + ::Spamcheck::SpamVerdict::Verdict::BLOCK => BLOCK_USER, + ::Spamcheck::SpamVerdict::Verdict::NOOP => NOOP + }.freeze + + ACTION_MAPPING = { + create: ::Spamcheck::Action::CREATE, + update: ::Spamcheck::Action::UPDATE + }.freeze + + def initialize + @endpoint_url = Gitlab::CurrentSettings.current_application_settings.spam_check_endpoint_url + + # remove the `grpc://` as it's only useful to ensure we're expecting to + # connect with Spamcheck + @endpoint_url = @endpoint_url.gsub(%r(^grpc:\/\/), '') + + creds = + if Rails.env.development? || Rails.env.test? + :this_channel_is_insecure + else + GRPC::Core::ChannelCredentials.new + end + + @stub = ::Spamcheck::SpamcheckService::Stub.new(@endpoint_url, creds, + timeout: DEFAULT_TIMEOUT_SECS) + end + + def issue_spam?(spam_issue:, user:, context: {}) + issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) + + response = @stub.check_for_spam_issue(issue, + metadata: { 'authorization' => + Gitlab::CurrentSettings.spam_check_api_key }) + verdict = convert_verdict_to_gitlab_constant(response.verdict) + [verdict, response.extra_attributes.to_h, response.error] + end + + private + + def convert_verdict_to_gitlab_constant(verdict) + VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(verdict), verdict) + end + + def build_issue_protobuf(issue:, user:, context:) + issue_pb = ::Spamcheck::Issue.new + issue_pb.title = issue.spam_title || '' + issue_pb.description = issue.spam_description || '' + issue_pb.created_at = convert_to_pb_timestamp(issue.created_at) if issue.created_at + issue_pb.updated_at = convert_to_pb_timestamp(issue.updated_at) if issue.updated_at + issue_pb.user_in_project = user.authorized_project?(issue.project) + issue_pb.project = build_project_protobuf(issue) + issue_pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) + issue_pb.user = build_user_protobuf(user) + issue_pb + end + + def build_user_protobuf(user) + user_pb = ::Spamcheck::User.new + user_pb.username = user.username + user_pb.org = user.organization || '' + user_pb.created_at = convert_to_pb_timestamp(user.created_at) + + user_pb.emails << build_email(user.email, user.confirmed?) + + user.emails.each do |email| + user_pb.emails << build_email(email.email, email.confirmed?) + end + + user_pb + end + + def build_email(email, verified) + email_pb = ::Spamcheck::User::Email.new + email_pb.email = email + email_pb.verified = verified + email_pb + end + + def build_project_protobuf(issue) + project_pb = ::Spamcheck::Project.new + project_pb.project_id = issue.project_id + project_pb.project_path = issue.project.full_path + project_pb + end + + def convert_to_pb_timestamp(ar_timestamp) + Google::Protobuf::Timestamp.new(seconds: ar_timestamp.to_time.to_i, + nanos: ar_timestamp.to_time.nsec) + end + end + end +end diff --git a/lib/gitlab/stack_prof.rb b/lib/gitlab/stack_prof.rb new file mode 100644 index 00000000000..4b7d93c91ce --- /dev/null +++ b/lib/gitlab/stack_prof.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +# trigger stackprof by sending a SIGUSR2 signal +# +# Docs: https://docs.gitlab.com/ee/development/performance.html#production + +module Gitlab + class StackProf + DEFAULT_FILE_PREFIX = Dir.tmpdir + DEFAULT_TIMEOUT_SEC = 30 + DEFAULT_MODE = :cpu + # Sample interval as a frequency in microseconds (~99hz); appropriate for CPU profiles + DEFAULT_INTERVAL_US = 10_100 + # Sample interval in event occurrences (n = every nth event); appropriate for allocation profiles + DEFAULT_INTERVAL_EVENTS = 100 + + # this is a workaround for sidekiq, which defines its own SIGUSR2 handler. + # by defering to the sidekiq startup event, we get to set up our own + # handler late enough. + # see also: https://github.com/mperham/sidekiq/pull/4653 + def self.install + require 'stackprof' + require 'tmpdir' + + if Gitlab::Runtime.sidekiq? + Sidekiq.configure_server do |config| + config.on :startup do + on_worker_start + end + end + else + Gitlab::Cluster::LifecycleEvents.on_worker_start do + on_worker_start + end + end + end + + def self.on_worker_start + log_event('listening for SIGUSR2 signal') + + # create a pipe in order to propagate signal out of the signal handler + # see also: https://cr.yp.to/docs/selfpipe.html + read, write = IO.pipe + + # create a separate thread that polls for signals on the pipe. + # + # this way we do not execute in signal handler context, which + # lifts restrictions and also serializes the calls in a thread-safe + # manner. + # + # it's very similar to a goroutine and channel design. + # + # another nice benefit of this method is that we can timeout the + # IO.select call, allowing the profile to automatically stop after + # a given interval (by default 30 seconds), avoiding unbounded memory + # growth from a profile that was started and never stopped. + t = Thread.new do + timeout_s = ENV['STACKPROF_TIMEOUT_S']&.to_i || DEFAULT_TIMEOUT_SEC + current_timeout_s = nil + loop do + read.getbyte if IO.select([read], nil, nil, current_timeout_s) + + if ::StackProf.running? + stackprof_file_prefix = ENV['STACKPROF_FILE_PREFIX'] || DEFAULT_FILE_PREFIX + stackprof_out_file = "#{stackprof_file_prefix}/stackprof.#{Process.pid}.#{SecureRandom.hex(6)}.profile" + + log_event( + 'stopping profile', + profile_filename: stackprof_out_file, + profile_timeout_s: timeout_s + ) + + ::StackProf.stop + ::StackProf.results(stackprof_out_file) + current_timeout_s = nil + else + mode = ENV['STACKPROF_MODE']&.to_sym || DEFAULT_MODE + interval = ENV['STACKPROF_INTERVAL']&.to_i + interval ||= (mode == :object ? DEFAULT_INTERVAL_EVENTS : DEFAULT_INTERVAL_US) + + log_event( + 'starting profile', + profile_mode: mode, + profile_interval: interval, + profile_timeout: timeout_s + ) + + ::StackProf.start( + mode: mode, + raw: Gitlab::Utils.to_boolean(ENV['STACKPROF_RAW'] || 'true'), + interval: interval + ) + current_timeout_s = timeout_s + end + end + rescue StandardError => e + log_event("stackprof failed: #{e}") + end + t.abort_on_exception = true + + # in the case of puma, this will override the existing SIGUSR2 signal handler + # that can be used to trigger a restart. + # + # puma cluster has two types of restarts: + # * SIGUSR1: phased restart + # * SIGUSR2: restart + # + # phased restart is not supported in our configuration, because we use + # preload_app. this means we will always perform a normal restart. + # additionally, phased restart is not supported when sending a SIGUSR2 + # directly to a puma worker (as opposed to the master process). + # + # the result is that the behaviour of SIGUSR1 and SIGUSR2 is identical in + # our configuration, and we can always use a SIGUSR1 to perform a restart. + # + # thus, it is acceptable for us to re-appropriate the SIGUSR2 signal, and + # override the puma behaviour. + # + # see also: + # * https://github.com/puma/puma/blob/master/docs/signals.md#puma-signals + # * https://github.com/phusion/unicorn/blob/master/SIGNALS + # * https://github.com/mperham/sidekiq/wiki/Signals + Signal.trap('SIGUSR2') do + write.write('.') + end + end + + def self.log_event(event, labels = {}) + Gitlab::AppJsonLogger.info({ + event: 'stackprof', + message: event, + pid: Process.pid + }.merge(labels.compact)) + end + end +end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index 0a2cee75af7..1555c3469a5 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -42,11 +42,11 @@ module Gitlab end def supported_content? - master_branch? && extension_supported? && file_exists? + branch_supported? && extension_supported? && file_exists? end - def master_branch? - ref == 'master' + def branch_supported? + ref.in?(%w[master main]) end def extension_supported? diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 3072210d7c8..ab2e1404cd2 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -9,8 +9,13 @@ module Gitlab def self.subscriptions_url ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url) end + + def self.payment_form_url + "#{self.subscriptions_url}/payment_forms/cc_validation" + end end end -Gitlab::SubscriptionPortal.prepend_if_jh('JH::Gitlab::SubscriptionPortal') +Gitlab::SubscriptionPortal.prepend_mod Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze +Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb index f9a635734a3..53885cdbf19 100644 --- a/lib/gitlab/suggestions/suggestion_set.rb +++ b/lib/gitlab/suggestions/suggestion_set.rb @@ -39,6 +39,10 @@ module Gitlab @file_paths ||= suggestions.map(&:file_path).uniq end + def authors + suggestions.map { |suggestion| suggestion.note.author }.uniq + end + private def first_suggestion diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index db3c058184c..1ceccc64ec0 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -109,7 +109,7 @@ module Gitlab def run_command!(command) output, status = Gitlab::Popen.popen(command) - raise Gitlab::TaskFailedError.new(output) unless status == 0 + raise Gitlab::TaskFailedError, output unless status == 0 output end diff --git a/lib/gitlab/tcp_checker.rb b/lib/gitlab/tcp_checker.rb index f37a044b607..a07a2e8786b 100644 --- a/lib/gitlab/tcp_checker.rb +++ b/lib/gitlab/tcp_checker.rb @@ -30,7 +30,7 @@ module Gitlab end true - rescue => err + rescue StandardError => err @error = err false diff --git a/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb b/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb deleted file mode 100644 index 3bf3a28d3c5..00000000000 --- a/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Template - class GitlabCiSyntaxYmlTemplate < BaseTemplate - class << self - def extension - '.gitlab-ci.yml' - end - - def categories - { - 'General' => '' - } - end - - def base_dir - Rails.root.join('lib/gitlab/ci/syntax_templates') - end - - def finder(project = nil) - Gitlab::Template::Finders::GlobalTemplateFinder.new( - self.base_dir, self.extension, self.categories - ) - end - end - end - end -end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 01158cafc4f..e1ca4b5ff6a 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -59,4 +59,4 @@ module Gitlab end end -Gitlab::Template::GitlabCiYmlTemplate.prepend_if_ee('::EE::Gitlab::Template::GitlabCiYmlTemplate') +Gitlab::Template::GitlabCiYmlTemplate.prepend_mod_with('Gitlab::Template::GitlabCiYmlTemplate') diff --git a/lib/gitlab/terraform_registry_token.rb b/lib/gitlab/terraform_registry_token.rb new file mode 100644 index 00000000000..ae7df49835f --- /dev/null +++ b/lib/gitlab/terraform_registry_token.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + class TerraformRegistryToken < JWTToken + class << self + def from_token(token) + new.tap do |terraform_registry_token| + terraform_registry_token['token'] = token.try(:token).presence || token.try(:id).presence + end + end + end + end +end diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb index 520075012e8..8f045021088 100644 --- a/lib/gitlab/throttle.rb +++ b/lib/gitlab/throttle.rb @@ -49,6 +49,20 @@ module Gitlab { limit: limit_proc, period: period_proc } end + def self.unauthenticated_packages_api_options + limit_proc = proc { |req| settings.throttle_unauthenticated_packages_api_requests_per_period } + period_proc = proc { |req| settings.throttle_unauthenticated_packages_api_period_in_seconds.seconds } + + { limit: limit_proc, period: period_proc } + end + + def self.authenticated_packages_api_options + limit_proc = proc { |req| settings.throttle_authenticated_packages_api_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_packages_api_period_in_seconds.seconds } + + { limit: limit_proc, period: period_proc } + end + def self.rate_limiting_response_text (settings.rate_limiting_response_text.presence || DEFAULT_RATE_LIMITING_RESPONSE_TEXT) + "\n" end diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index b15cb85dde0..bfdfb01093f 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -15,7 +15,7 @@ module Gitlab ChronicDuration.parse( string, CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours')) - rescue + rescue StandardError nil end @@ -30,7 +30,7 @@ module Gitlab format: :short, limit_to_hours: limit_to_hours_setting, weeks: true)) - rescue + rescue StandardError nil end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index b16ae39bcee..5fb360296b7 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -14,7 +14,7 @@ module Gitlab snowplow.event(category, action, label: label, property: property, value: value, context: contexts) product_analytics.event(category, action, label: label, property: property, value: value, context: contexts) - rescue => error + rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end diff --git a/lib/gitlab/tracking/docs/helper.rb b/lib/gitlab/tracking/docs/helper.rb new file mode 100644 index 00000000000..81874aac9a5 --- /dev/null +++ b/lib/gitlab/tracking/docs/helper.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Docs + # Helper with functions to be used by HAML templates + module Helper + def auto_generated_comment + <<-MARKDOWN.strip_heredoc + --- + stage: Growth + group: Product Intelligence + info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers + --- + + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. + ---> + + <!-- vale gitlab.Spelling = NO --> + MARKDOWN + end + + def render_description(object) + return 'Missing description' unless object.description.present? + + object.description + end + + def render_event_taxonomy(object) + headers = %w[category action label property value] + values = %i[category action label property_description value_description] + values = values.map { |key| backtick(object.attributes[key]) } + values = values.join(" | ") + + [ + "| #{headers.join(" | ")} |", + "#{'|---' * headers.size}|", + "| #{values} |" + ].join("\n") + end + + def md_link_to(anchor_text, url) + "[#{anchor_text}](#{url})" + end + + def render_owner(object) + "Owner: #{backtick(object.product_group)}" + end + + def render_tiers(object) + "Tiers: #{object.tiers.map(&method(:backtick)).join(', ')}" + end + + def render_yaml_definition_path(object) + "YAML definition: #{backtick(object.yaml_path)}" + end + + def backtick(string) + "`#{string}`" + end + end + end + end +end diff --git a/lib/gitlab/tracking/docs/renderer.rb b/lib/gitlab/tracking/docs/renderer.rb new file mode 100644 index 00000000000..184b935c2ba --- /dev/null +++ b/lib/gitlab/tracking/docs/renderer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Docs + class Renderer + include Gitlab::Tracking::Docs::Helper + DICTIONARY_PATH = Rails.root.join('doc', 'development', 'snowplow') + TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'tracking', 'docs', 'templates', 'default.md.haml') + + def initialize(event_definitions) + @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) + @event_definitions = event_definitions.sort + end + + def contents + # Render and remove an extra trailing new line + @contents ||= @layout.render(self, event_definitions: @event_definitions).sub!(/\n(?=\Z)/, '') + end + + def write + filename = DICTIONARY_PATH.join('dictionary.md').to_s + + FileUtils.mkdir_p(DICTIONARY_PATH) + File.write(filename, contents) + + filename + end + end + end + end +end diff --git a/lib/gitlab/tracking/docs/templates/default.md.haml b/lib/gitlab/tracking/docs/templates/default.md.haml new file mode 100644 index 00000000000..568f56590fa --- /dev/null +++ b/lib/gitlab/tracking/docs/templates/default.md.haml @@ -0,0 +1,35 @@ += auto_generated_comment + +:plain + # Event Dictionary + + This file is autogenerated, please do not edit it directly. + + To generate these files from the GitLab repository, run: + + ```shell + bundle exec rake gitlab:snowplow:generate_event_dictionary + ``` + + The Event Dictionary is based on the following event definition YAML files: + + - [`config/events`](https://gitlab.com/gitlab-org/gitlab/-/tree/f9a404301ca22d038e7b9a9eb08d9c1bbd6c4d84/config/events) + - [`ee/config/events`](https://gitlab.com/gitlab-org/gitlab/-/tree/f9a404301ca22d038e7b9a9eb08d9c1bbd6c4d84/ee/config/events) + + ## Event definitions + +\ +- event_definitions.each do |_path, object| + + = "### `#{object.category} #{object.action}`" + \ + = render_event_taxonomy(object) + \ + = render_description(object) + \ + = render_yaml_definition_path(object) + \ + = render_owner(object) + \ + = render_tiers(object) + \ diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb new file mode 100644 index 00000000000..8f70c8ecab7 --- /dev/null +++ b/lib/gitlab/tracking/event_definition.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + InvalidEventError = Class.new(RuntimeError) + + class EventDefinition + EVENT_SCHEMA_PATH = Rails.root.join('config', 'events', 'schema.json') + BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' + SCHEMA = ::JSONSchemer.schema(Pathname.new(EVENT_SCHEMA_PATH)) + + attr_reader :path + attr_reader :attributes + + class << self + def paths + @paths ||= [Rails.root.join('config', 'events', '*.yml'), Rails.root.join('ee', 'config', 'events', '*.yml')] + end + + def definitions + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + + private + + def load_from_file(path) + definition = File.read(path) + definition = YAML.safe_load(definition) + definition.deep_symbolize_keys! + + self.new(path, definition).tap(&:validate!) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(e.message)) + end + + def load_all_from_path!(definitions, glob_path) + Dir.glob(glob_path).each do |path| + definition = load_from_file(path) + definitions[definition.path] = definition + end + end + end + + def initialize(path, opts = {}) + @path = path + @attributes = opts + end + + def to_h + attributes + end + alias_method :to_dictionary, :to_h + + def yaml_path + path.delete_prefix(Rails.root.to_s) + end + + def validate! + SCHEMA.validate(attributes.stringify_keys).each do |error| + error_message = <<~ERROR_MSG + Error type: #{error['type']} + Data: #{error['data']} + Path: #{error['data_pointer']} + Details: #{error['details']} + Definition file: #{path} + ERROR_MSG + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(error_message)) + end + end + + private + + def method_missing(method, *args) + attributes[method] || super + end + end + end +end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index da030649f76..7902f96dfa6 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,10 +3,12 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-4' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-5' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) + @namespace = namespace + @plan = @namespace&.actual_plan_name @extra = extra end @@ -36,6 +38,7 @@ module Gitlab { environment: environment, source: source, + plan: @plan, extra: @extra } end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 86cd91f0a32..85f0ba1fd25 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -161,4 +161,4 @@ module Gitlab end end -Gitlab::TreeSummary.prepend_if_ee('::EE::Gitlab::TreeSummary') +Gitlab::TreeSummary.prepend_mod_with('Gitlab::TreeSummary') diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 706c0925302..09236a7f1f0 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -22,7 +22,7 @@ module Gitlab @regexp = RE2::Regexp.new(pattern, log_errors: false) - raise RegexpError.new(regexp.error) unless regexp.ok? + raise RegexpError, regexp.error unless regexp.ok? end def replace_all(text, rewrite) diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index b610d2a10c6..deab2cd43a6 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -76,4 +76,4 @@ module Gitlab end end -Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') +Gitlab::Uploads::MigrationHelper.prepend_mod_with('Gitlab::Uploads::MigrationHelper') diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index f98c488bbe5..a242f718b16 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -49,7 +49,7 @@ module Gitlab when ::DesignManagement::Design design_url(object, **options) else - raise NotImplementedError.new("No URL builder defined for #{object.inspect}") + raise NotImplementedError, "No URL builder defined for #{object.inspect}" end end # rubocop:enable Metrics/CyclomaticComplexity @@ -127,4 +127,4 @@ module Gitlab end end -::Gitlab::UrlBuilder.prepend_if_ee('EE::Gitlab::UrlBuilder') +::Gitlab::UrlBuilder.prepend_mod_with('Gitlab::UrlBuilder') diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index 6b185a5a1e9..c2e5d467dbb 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -18,8 +18,6 @@ module Gitlab Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. ---> - - <!-- vale gitlab.Spelling = NO --> MARKDOWN end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 26f1aa4396d..8911ac2ed1a 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -19,6 +19,10 @@ Each table includes a `milestone`, which corresponds to the GitLab version when the metric was released. + <!-- vale off --> + <!-- Docs linting disabled after this line. --> + <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests --> + ## Metrics Definitions \ diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 9c4255a7c92..ccd2c69e2e7 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -26,11 +26,11 @@ module Gitlab def json_schema_path return '' unless has_json_schema? - "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}" + "#{BASE_REPO_PATH}/#{attributes[:value_json_schema]}" end def has_json_schema? - attributes[:value_type] == 'object' && attributes[:object_json_schema].present? + attributes[:value_type] == 'object' && attributes[:value_json_schema].present? end def yaml_path @@ -65,6 +65,10 @@ module Gitlab @definitions ||= load_all! end + def all + @all ||= definitions.map { |_key_path, definition| definition } + end + def schemer @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) end @@ -87,7 +91,7 @@ module Gitlab definition.deep_symbolize_keys! self.new(path, definition).tap(&:validate!) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message)) end @@ -117,4 +121,4 @@ module Gitlab end end -Gitlab::Usage::MetricDefinition.prepend_if_ee('EE::Gitlab::Usage::MetricDefinition') +Gitlab::Usage::MetricDefinition.prepend_mod_with('Gitlab::Usage::MetricDefinition') diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index f77c8cab39c..4c40bfbc06f 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -83,7 +83,7 @@ module Gitlab when UNION_OF_AGGREGATED_METRICS source.calculate_metrics_union(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) when INTERSECTION_OF_AGGREGATED_METRICS - calculate_metrics_intersections(source: source, metric_names: aggregation[:events], start_date: start_date, end_date: end_date) + source.calculate_metrics_intersections(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) else Gitlab::ErrorTracking .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) @@ -94,67 +94,6 @@ module Gitlab Gitlab::Utils::UsageData::FALLBACK end - # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle - # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 - def calculate_metrics_intersections(source:, metric_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) - # calculate power of intersection of all given metrics from inclusion exclusion principle - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - - # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... - subset_powers_data = subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) - - # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_metrics = begin - subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ - source.calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) - end - - # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, - # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - subset_powers_size_even = subset_powers_data.size.even? - - # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => - sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) - - # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| - sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_metrics : -power_of_union_of_all_metrics) - end - - def sum_subset_powers(subset_powers_data, subset_powers_size_even) - sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| - (index + 1).odd? ? value : -value - end - - (subset_powers_size_even ? -1 : 1) * sum_without_sign - end - - def subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) - subset_sizes = (1...metric_names.size) - - subset_sizes.map do |subset_size| - if subset_size > 1 - # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) - metric_names.combination(subset_size).sum do |metrics_subset| - subset_powers_cache[subset_size][metrics_subset.join('_&_')] ||= - calculate_metrics_intersections(source: source, metric_names: metrics_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) - end - else - # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... - metric_names.sum do |metric| - subset_powers_cache[subset_size][metric] ||= \ - source.calculate_metrics_union(metric_names: metric, start_date: start_date, end_date: end_date, recorded_at: recorded_at) - end - end - end - end - def load_metrics(wildcard) Dir[wildcard].each_with_object([]) do |path, metrics| metrics.push(*load_yaml_from_path(path)) @@ -170,4 +109,4 @@ module Gitlab end end -Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_if_ee('EE::Gitlab::Usage::Metrics::Aggregates::Aggregate') +Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_mod_with('Gitlab::Usage::Metrics::Aggregates::Aggregate') diff --git a/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb new file mode 100644 index 00000000000..dabf757c8a7 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources/calculations/intersection.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + module Calculations + module Intersection + def calculate_metrics_intersections(metric_names:, start_date:, end_date:, recorded_at:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(metric_names, start_date, end_date, recorded_at, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_metrics = begin + subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ + calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_metrics : -power_of_union_of_all_metrics) + end + + private + + def subsets_intersection_powers(metric_names, start_date, end_date, recorded_at, subset_powers_cache) + subset_sizes = (1...metric_names.size) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + metric_names.combination(subset_size).sum do |metrics_subset| + subset_powers_cache[subset_size][metrics_subset.join('_&_')] ||= + calculate_metrics_intersections(metric_names: metrics_subset, start_date: start_date, end_date: end_date, recorded_at: recorded_at, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + metric_names.sum do |metric| + subset_powers_cache[subset_size][metric] ||= \ + calculate_metrics_union(metric_names: metric, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + end + end + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb index a01efbdb1a6..3069afab147 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -6,6 +6,7 @@ module Gitlab module Aggregates module Sources class PostgresHll + extend Calculations::Intersection class << self def calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at:) time_period = start_date && end_date ? (start_date..end_date) : nil diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb index f3a4dcf1e31..009b8e62543 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb @@ -8,6 +8,7 @@ module Gitlab UnionNotAvailable = Class.new(AggregatedMetricError) class RedisHll + extend Calculations::Intersection def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil) union = Gitlab::UsageDataCounters::HLLRedisCounter .calculate_events_union(event_names: metric_names, start_date: start_date, end_date: end_date) diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb new file mode 100644 index 00000000000..29b44f2bd0a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BaseMetric + include Gitlab::Utils::UsageData + + attr_reader :time_frame + + def initialize(time_frame:) + @time_frame = time_frame + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb new file mode 100644 index 00000000000..4e1ba027bca --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_boards_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountBoardsMetric < DatabaseMetric + operation :count + + relation { Board } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb new file mode 100644 index 00000000000..34247f4f6dd --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountIssuesMetric < DatabaseMetric + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + relation { Issue } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_creating_issues_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_creating_issues_metric.rb new file mode 100644 index 00000000000..c8331ce5b31 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_creating_issues_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersCreatingIssuesMetric < DatabaseMetric + operation :distinct_count, column: :author_id + + relation { Issue } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb new file mode 100644 index 00000000000..9c92f2e9595 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_using_approve_quick_action_metric.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersUsingApproveQuickActionMetric < RedisHLLMetric + event_names :i_quickactions_approve + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb new file mode 100644 index 00000000000..f83f90dea03 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DatabaseMetric < BaseMetric + # Usage Example + # + # class CountUsersCreatingIssuesMetric < DatabaseMetric + # operation :distinct_count, column: :author_id + # + # relation do |database_time_constraints| + # ::Issue.where(database_time_constraints) + # end + # end + class << self + def start(&block) + @metric_start = block + end + + def finish(&block) + @metric_finish = block + end + + def relation(&block) + @metric_relation = block + end + + def operation(symbol, column: nil) + @metric_operation = symbol + @column = column + end + + attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column + end + + def value + method(self.class.metric_operation) + .call(relation, + self.class.column, + start: self.class.metric_start&.call, + finish: self.class.metric_finish&.call) + end + + def relation + self.class.metric_relation.call.where(time_constraints) + end + + private + + def time_constraints + case time_frame + when '28d' + { created_at: 30.days.ago..2.days.ago } + when 'all' + {} + when 'none' + nil + else + raise "Unknown time frame: #{time_frame} for DatabaseMetric" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb new file mode 100644 index 00000000000..7c97cc37d17 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GenericMetric < BaseMetric + # Usage example + # + # class UuidMetric < GenericMetric + # value do + # Gitlab::CurrentSettings.uuid + # end + # end + class << self + def value(&block) + @metric_value = block + end + + attr_reader :metric_value + end + + def value + alt_usage_data do + self.class.metric_value.call + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/hostname_metric.rb b/lib/gitlab/usage/metrics/instrumentations/hostname_metric.rb new file mode 100644 index 00000000000..3364c330cca --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/hostname_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class HostnameMetric < GenericMetric + value do + Gitlab.config.gitlab.host + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb new file mode 100644 index 00000000000..140d56f0d42 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class RedisHLLMetric < BaseMetric + # Usage example + # + # class CountUsersVisitingAnalyticsValuestreamMetric < RedisHLLMetric + # event_names :g_analytics_valuestream + # end + class << self + def event_names(events = nil) + @metric_events = events + end + + attr_reader :metric_events + end + + def value + redis_usage_data do + event_params = time_constraints.merge(event_names: self.class.metric_events) + + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(**event_params) + end + end + + private + + def time_constraints + case time_frame + when '28d' + { start_date: 4.weeks.ago.to_date, end_date: Date.current } + when '7d' + { start_date: 7.days.ago.to_date, end_date: Date.current } + else + raise "Unknown time frame: #{time_frame} for TimeConstraint" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/uuid_metric.rb b/lib/gitlab/usage/metrics/instrumentations/uuid_metric.rb new file mode 100644 index 00000000000..58547b5383a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/uuid_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class UuidMetric < GenericMetric + value do + Gitlab::CurrentSettings.uuid + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/key_path_processor.rb b/lib/gitlab/usage/metrics/key_path_processor.rb new file mode 100644 index 00000000000..dbe574d5838 --- /dev/null +++ b/lib/gitlab/usage/metrics/key_path_processor.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class KeyPathProcessor + class << self + def process(key_path, value) + unflatten(key_path.split('.'), value) + end + + private + + def unflatten(keys, value) + loop do + value = { keys.pop.to_sym => value } + + break if keys.blank? + end + + value + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b36ca38cd64..b1ba529d4a4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -98,7 +98,6 @@ module Gitlab ci_external_pipelines: count(::Ci::Pipeline.external), ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), - ci_runners: count(::Ci::Runner), ci_triggers: count(::Ci::Trigger), ci_pipeline_schedules: count(::Ci::PipelineSchedule), auto_devops_enabled: count(::ProjectAutoDevops.enabled), @@ -164,7 +163,6 @@ module Gitlab projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_service_enabled: count(Service.active.where(type: 'AlertsService')), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -186,12 +184,14 @@ module Gitlab merge_requests: count(MergeRequest), notes: count(Note) }.merge( + runners_usage, services_usage, usage_counters, user_preferences_usage, ingress_modsecurity_usage, container_expiration_policies_usage, - service_desk_counts + service_desk_counts, + email_campaign_counts ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -199,6 +199,18 @@ module Gitlab end # rubocop: enable Metrics/AbcSize + def runners_usage + { + ci_runners: count(::Ci::Runner), + ci_runners_instance_type_active: count(::Ci::Runner.instance_type.active), + ci_runners_group_type_active: count(::Ci::Runner.group_type.active), + ci_runners_project_type_active: count(::Ci::Runner.project_type.active), + ci_runners_instance_type_active_online: count(::Ci::Runner.instance_type.active.online), + ci_runners_group_type_active_online: count(::Ci::Runner.group_type.active.online), + ci_runners_project_type_active_online: count(::Ci::Runner.project_type.active.online) + } + end + def snowplow_event_counts(time_period) return {} unless report_snowplow_events? @@ -243,7 +255,8 @@ module Gitlab { settings: { ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, - operating_system: alt_usage_data(fallback: nil) { operating_system } + operating_system: alt_usage_data(fallback: nil) { operating_system }, + gitaly_apdex: alt_usage_data { gitaly_apdex } } } end @@ -414,13 +427,15 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: - Service.available_services_names.each_with_object({}) do |service_name, response| - response["projects_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where(type: "#{service_name}_service".camelize)) - response["groups_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where(type: "#{service_name}_service".camelize)) - response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize)) - response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize)) - response["projects_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) - response["groups_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) + Integration.available_services_names(include_dev: false).each_with_object({}) do |service_name, response| + service_type = Integration.service_name_to_type(service_name) + + response["projects_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where(type: service_type)) + response["groups_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where(type: service_type)) + response["templates_#{service_name}_active".to_sym] = count(Integration.active.where(template: true, type: service_type)) + response["instances_#{service_name}_active".to_sym] = count(Integration.active.where(instance: true, type: service_type)) + response["projects_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: service_type)) + response["groups_inheriting_#{service_name}_active".to_sym] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: service_type)) end.merge(jira_usage, jira_import_usage) # rubocop: enable UsageData/LargeTable: end @@ -435,18 +450,10 @@ module Gitlab projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - # rubocop: disable UsageData/LargeTable: - JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| - counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - service_url = service.data_fields&.url || (service.properties && service.properties['url']) - service_url&.include?('.atlassian.net') ? :cloud : :server - end + jira_service_data_hash = jira_service_data + results[:projects_jira_server_active] = jira_service_data_hash[:projects_jira_server_active] + results[:projects_jira_cloud_active] = jira_service_data_hash[:projects_jira_cloud_active] - results[:projects_jira_server_active] += counts[:server].size if counts[:server] - results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] - end - # rubocop: enable UsageData/LargeTable: results rescue ActiveRecord::StatementInvalid { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } @@ -570,7 +577,11 @@ module Gitlab projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))), projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), - snippets: distinct_count(::Snippet.where(time_period), :author_id) + snippets: distinct_count(::Snippet.where(time_period), :author_id), + suggestions: distinct_count(::Note.with_suggestions.where(time_period), + :author_id, + start: minimum_id(::User), + finish: maximum_id(::User)) }.tap do |h| if time_period.present? h[:merge_requests_users] = merge_requests_users(time_period) @@ -597,7 +608,7 @@ module Gitlab unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { gitlab: DEPRECATED_VALUE, - gitlab_v1: count(::BulkImport.where(time_period, source_type: :gitlab)) + gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, project_imports: project_imports(time_period), issue_imports: issue_imports(time_period), @@ -767,6 +778,16 @@ module Gitlab private + def gitaly_apdex + with_prometheus_client(verify: false, fallback: FALLBACK) do |client| + result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first + + break FALLBACK unless result + + result['value'].last.to_f + end + end + def aggregated_metrics @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new(recorded_at) end @@ -825,6 +846,28 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord + def email_campaign_counts + # rubocop:disable UsageData/LargeTable + sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) + clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) + + series_amount = Namespaces::InProductMarketingEmailsService::INTERVAL_DAYS.count + + Users::InProductMarketingEmail.tracks.keys.each_with_object({}) do |track, result| + # rubocop: enable UsageData/LargeTable: + 0.upto(series_amount - 1).map do |series| + # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. + sent_count = sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails + clicked_count = clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails + + result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count + result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + def unique_visit_service strong_memoize(:unique_visit_service) do ::Gitlab::Analytics::UniqueVisits.new @@ -955,4 +998,4 @@ module Gitlab end end -Gitlab::UsageData.prepend_if_ee('EE::Gitlab::UsageData') +Gitlab::UsageData.prepend_mod_with('Gitlab::UsageData') diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb index 7f7854c3eb1..b823d6cc2bf 100644 --- a/lib/gitlab/usage_data/topology.rb +++ b/lib/gitlab/usage_data/topology.rb @@ -47,7 +47,7 @@ module Gitlab nodes: topology_node_data(client) }.compact end - rescue => e + rescue StandardError => e @failures << CollectionFailure.new('other', e.class.to_s) {} @@ -183,7 +183,7 @@ module Gitlab @failures << CollectionFailure.new(query_name, 'empty_result') fallback - rescue => e + rescue StandardError => e @failures << CollectionFailure.new(query_name, e.class.to_s) fallback end diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index e1648245f3f..dd66a40a48f 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -47,3 +47,6 @@ - i_package_tag_delete_package - i_package_tag_pull_package - i_package_tag_push_package +- i_package_terraform_module_delete_package +- i_package_terraform_module_pull_package +- i_package_terraform_module_push_package diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index bef3fc7b504..bc0126cd893 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -50,7 +50,6 @@ module Gitlab private def track_unique_action(action, author, time) - return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true) return unless author Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index a8691169fb8..833eebd5d04 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -132,7 +132,7 @@ module Gitlab return unless feature_enabled?(event) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) - rescue => e + rescue StandardError => e # Ignore any exceptions unless is dev or test env # The application flow should not be blocked by erros in tracking Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) @@ -232,8 +232,8 @@ module Gitlab # Compose the key in order to store events daily or weekly def redis_key(event, time, context = '') - raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) - raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) + raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) key = apply_slot(event) key = apply_time_aggregation(key, time, event) @@ -277,4 +277,4 @@ module Gitlab end end -Gitlab::UsageDataCounters::HLLRedisCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::HLLRedisCounter') +Gitlab::UsageDataCounters::HLLRedisCounter.prepend_mod_with('Gitlab::UsageDataCounters::HLLRedisCounter') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 6f5f878501f..083de402175 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -154,4 +154,4 @@ module Gitlab end end -Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::IssueActivityUniqueCounter') +Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_mod_with('Gitlab::UsageDataCounters::IssueActivityUniqueCounter') diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml new file mode 100644 index 00000000000..e4f20b61901 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -0,0 +1,85 @@ +- name: users_viewing_analytics_group_devops_adoption + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_dev_ops_adoption + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_dev_ops_score + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_merge_request + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_instance_statistics + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_contribution + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_insights + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_issues + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_productivity + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: g_analytics_valuestream + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_pipelines + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_code_reviews + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_valuestream + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_insights + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_issues + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: p_analytics_repo + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits +- name: i_analytics_cohorts + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 18c5dc73de2..cc89fbd5caf 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -3,204 +3,219 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_mr_diffs - name: i_code_review_user_single_file_diffs redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_single_file_diffs - name: i_code_review_mr_single_file_diffs redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_mr_single_file_diffs - name: i_code_review_user_toggled_task_item_status redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_toggled_task_item_status - name: i_code_review_user_create_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr - name: i_code_review_user_close_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_close_mr - name: i_code_review_user_reopen_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_reopen_mr - name: i_code_review_user_approve_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approve_mr - name: i_code_review_user_unapprove_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_unapprove_mr - name: i_code_review_user_resolve_thread redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_resolve_thread - name: i_code_review_user_unresolve_thread redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_unresolve_thread - name: i_code_review_edit_mr_title redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_edit_mr_title - name: i_code_review_edit_mr_desc redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_edit_mr_desc - name: i_code_review_user_merge_mr redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_merge_mr - name: i_code_review_user_create_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr_comment - name: i_code_review_user_edit_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_mr_comment - name: i_code_review_user_remove_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_mr_comment - name: i_code_review_user_create_review_note redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_review_note - name: i_code_review_user_publish_review redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_publish_review - name: i_code_review_user_create_multiline_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment - name: i_code_review_user_edit_multiline_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment - name: i_code_review_user_remove_multiline_mr_comment redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment - name: i_code_review_user_add_suggestion redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_add_suggestion - name: i_code_review_user_apply_suggestion redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_apply_suggestion - name: i_code_review_user_assigned redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_assigned - name: i_code_review_user_marked_as_draft redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_marked_as_draft - name: i_code_review_user_unmarked_as_draft redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_unmarked_as_draft - name: i_code_review_user_review_requested redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_review_requested - name: i_code_review_user_approval_rule_added redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approval_rule_added - name: i_code_review_user_approval_rule_deleted redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approval_rule_deleted - name: i_code_review_user_approval_rule_edited redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_approval_rule_edited - name: i_code_review_user_vs_code_api_request redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_vs_code_api_request - name: i_code_review_user_create_mr_from_issue redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr_from_issue - name: i_code_review_user_mr_discussion_locked redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_mr_discussion_locked - name: i_code_review_user_mr_discussion_unlocked redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_mr_discussion_unlocked - name: i_code_review_user_time_estimate_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_time_estimate_changed - name: i_code_review_user_time_spent_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_time_spent_changed - name: i_code_review_user_assignees_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_assignees_changed - name: i_code_review_user_reviewers_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_reviewers_changed - name: i_code_review_user_milestone_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_milestone_changed - name: i_code_review_user_labels_changed redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_labels_changed +# Diff settings events +- name: i_code_review_click_single_file_mode_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_click_file_browser_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_click_whitespace_setting + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_view_inline + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_view_parallel + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_file_browser_tree_view + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_file_browser_list_view + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_show_whitespace + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_hide_whitespace + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_single_file + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data +- name: i_code_review_diff_multiple_files + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_settings_usage_data diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 077864032e8..f2504396cc4 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -24,92 +24,6 @@ category: compliance redis_slot: compliance aggregation: weekly - feature_flag: usage_data_a_compliance_audit_events_api -# Analytics category -- name: g_analytics_contribution - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_productivity - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_valuestream - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_pipelines - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_code_reviews - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_valuestream - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_insights - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_issues - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_repo - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_cohorts - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_dev_ops_score - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_dev_ops_adoption - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: g_analytics_merge_request - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: p_analytics_merge_request - category: analytics - redis_slot: analytics - aggregation: weekly - feature_flag: track_unique_visits -- name: i_analytics_instance_statistics - category: analytics - redis_slot: analytics - aggregation: weekly feature_flag: track_unique_visits - name: g_edit_by_web_ide category: ide_edit @@ -139,17 +53,14 @@ category: search redis_slot: search aggregation: weekly - feature_flag: search_track_unique_users - name: i_search_advanced category: search redis_slot: search aggregation: weekly - feature_flag: search_track_unique_users - name: i_search_paid category: search redis_slot: search aggregation: weekly - feature_flag: search_track_unique_users - name: wiki_action category: source_code aggregation: daily @@ -175,52 +86,42 @@ redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_alert_status_changed - name: incident_management_alert_assigned redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_alert_assigned - name: incident_management_alert_todo redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_alert_todo - name: incident_management_incident_created redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_created - name: incident_management_incident_reopened redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_reopened - name: incident_management_incident_closed redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_closed - name: incident_management_incident_assigned redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_assigned - name: incident_management_incident_todo redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_todo - name: incident_management_incident_comment redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_comment - name: incident_management_incident_zoom_meeting redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_zoom_meeting - name: incident_management_incident_published redis_slot: incident_management category: incident_management @@ -230,23 +131,19 @@ redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_relate - name: incident_management_incident_unrelate redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_unrelate - name: incident_management_incident_change_confidential redis_slot: incident_management category: incident_management aggregation: weekly - feature_flag: usage_data_incident_management_incident_change_confidential # Incident management alerts - name: incident_management_alert_create_incident redis_slot: incident_management category: incident_management_alerts aggregation: weekly - feature_flag: usage_data_incident_management_alert_create_incident # Incident management on-call - name: i_incident_management_oncall_notification_sent redis_slot: incident_management diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 1c765bb1830..adc5ba36ad7 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -24,45 +24,35 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_push_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_deployment_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_wiki_page_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_merge_request_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_note_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_tag_push_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_confidential_note_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - name: i_ecosystem_slack_service_confidential_issue_notification category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_slack_service - diff --git a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml new file mode 100644 index 00000000000..281db441829 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml @@ -0,0 +1,22 @@ +# Epic board events +# +# We are using the same slot of issue events 'project_management' for +# epic events to allow data aggregation. +# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 +- name: g_project_management_users_creating_epic_boards + category: epic_boards_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epic_boards_activity + +- name: g_project_management_users_viewing_epic_boards + category: epic_boards_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epic_boards_activity + +- name: g_project_management_users_updating_epic_board_names + category: epic_boards_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epic_boards_activity diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml index 80460dbe4d2..d1864cd569b 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -9,6 +9,20 @@ aggregation: daily feature_flag: track_epics_activity +# content change events + +- name: project_management_users_unchecking_epic_task + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: project_management_users_checking_epic_task + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + - name: g_project_management_users_updating_epic_titles category: epics_usage redis_slot: project_management @@ -41,6 +55,20 @@ aggregation: daily feature_flag: track_epics_activity +# emoji + +- name: g_project_management_users_awarding_epic_emoji + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_removing_epic_emoji + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + # start date events - name: g_project_management_users_setting_epic_start_date_as_fixed @@ -81,6 +109,8 @@ aggregation: daily feature_flag: track_epics_activity +# relationships + - name: g_project_management_epic_issue_added category: epics_usage redis_slot: project_management @@ -99,6 +129,12 @@ aggregation: daily feature_flag: track_epics_activity +- name: g_project_management_users_updating_epic_parent + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + - name: g_project_management_epic_closed category: epics_usage redis_slot: project_management @@ -140,3 +176,9 @@ redis_slot: project_management aggregation: daily feature_flag: track_epics_activity + +- name: g_project_management_epic_cross_referenced + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index b7e583003c8..d8ad2b538d6 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -95,3 +95,11 @@ category: user_packages aggregation: weekly redis_slot: package +- name: i_package_terraform_module_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package +- name: i_package_terraform_module_user + category: user_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 0fe65afb237..c1eabb352f7 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -3,334 +3,267 @@ category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_single category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_multiple category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_self category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_assign_reviewer category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_award category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_board_move category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_child_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_clear_weight category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_clone category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_close category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_confidential category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_copy_metadata_merge_request category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_copy_metadata_issue category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_create_merge_request category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_done category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_draft category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_due category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_duplicate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_estimate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_iteration category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_label category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_lock category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_merge category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_milestone category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_move category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_parent_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_promote category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_publish category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_reassign category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_reassign_reviewer category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_rebase category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_relabel category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_relate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_child_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_due_date category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_estimate category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_iteration category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_milestone category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_parent_epic category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_time_spent category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_remove_zoom category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_reopen category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_shrug category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_spend_subtract category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_spend_add category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_submit_review category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_subscribe category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_tableflip category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_tag category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_target_branch category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_title category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_todo category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unassign_specific category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unassign_all category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unassign_reviewer category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unlabel_specific category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unlabel_all category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unlock category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_unsubscribe category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_weight category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_wip category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_zoom category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_invite_email_single category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions - name: i_quickactions_invite_email_multiple category: quickactions redis_slot: quickactions aggregation: weekly - feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index eae42bdc4a1..8b9ca0fc220 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -4,17 +4,27 @@ module Gitlab module UsageDataCounters class KubernetesAgentCounter < BaseCounter PREFIX = 'kubernetes_agent' - KNOWN_EVENTS = %w[gitops_sync].freeze + KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request].freeze class << self - def increment_gitops_sync(incr) - raise ArgumentError, 'must be greater than or equal to zero' if incr < 0 + def increment_event_counts(events) + validate!(events) - # rather then hitting redis for this no-op, we return early - # note: redis returns the increment, so we mimic this here - return 0 if incr == 0 + events.each do |event, incr| + # rather then hitting redis for this no-op, we return early + next if incr == 0 - increment_by(redis_key(:gitops_sync), incr) + increment_by(redis_key(event), incr) + end + end + + private + + def validate!(events) + events.each do |event, incr| + raise ArgumentError, "unknown event #{event}" unless event.in?(KNOWN_EVENTS) + raise ArgumentError, "#{event} count must be greater than or equal to zero" if incr < 0 + end end end end diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index ed3df7dcf75..557179ad57a 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -7,7 +7,6 @@ module Gitlab # Tracks the quick action with name `name`. # `args` is expected to be a single string, will be split internally when necessary. def track_unique_action(name, args:, user:) - return unless Feature.enabled?(:usage_data_track_quickactions, default_enabled: :yaml) return unless user args ||= '' diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb new file mode 100644 index 00000000000..e181da01229 --- /dev/null +++ b/lib/gitlab/usage_data_metrics.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + class UsageDataMetrics + class << self + # Build the Usage Ping JSON payload from metrics YAML definitions which have instrumentation class set + def uncached_data + ::Gitlab::Usage::MetricDefinition.all.map do |definition| + instrumentation_class = definition.attributes[:instrumentation_class] + + if instrumentation_class.present? + metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new(time_frame: definition.attributes[:time_frame]).value + + metric_payload(definition.key_path, metric_value) + else + {} + end + end.reduce({}, :deep_merge) + end + + private + + def metric_payload(key_path, value) + ::Gitlab::Usage::Metrics::KeyPathProcessor.process(key_path, value) + end + end + end +end diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 1f72bf4ce26..bc72a96a468 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -25,10 +25,17 @@ module Gitlab SQL_METRIC_DEFAULT end - def maximum_id(model) + def maximum_id(model, column = nil) end - def minimum_id(model) + def minimum_id(model, column = nil) + end + + def jira_service_data + { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c0dfae88fc7..1c776501fdb 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -25,6 +25,27 @@ module Gitlab relation.select(relation.all.table[column].sum).to_sql end + # rubocop: disable CodeReuse/ActiveRecord + def histogram(relation, column, buckets:, bucket_size: buckets.size) + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + bucket_segments = bucket_size - 1 + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + query.to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + # For estimated distinct count use exact query instead of hll # buckets query, because it can't be used to obtain estimations without # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter @@ -36,10 +57,21 @@ module Gitlab 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') end - def maximum_id(model) + def maximum_id(model, column = nil) + end + + def minimum_id(model, column = nil) + end + + def jira_service_data + { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } end - def minimum_id(model) + def epics_deepest_relationship_level + { epics_deepest_relationship_level: 0 } end private diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index c1a57566640..d70e5c3594c 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,7 +16,7 @@ module Gitlab path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/ if path.match?(path_regex) - raise PathTraversalAttackError.new('Invalid path') + raise PathTraversalAttackError, 'Invalid path' end path diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index c92865636d0..39670a835a6 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -43,12 +43,12 @@ module Gitlab instance_method_defined?(parent, method_name) end - raise NotImplementedError.new("#{klass}\##{method_name} doesn't exist!") unless overridden_parent + raise NotImplementedError, "#{klass}\##{method_name} doesn't exist!" unless overridden_parent super_method_arity = find_direct_method(overridden_parent, method_name).arity unless arity_compatible?(sub_method_arity, super_method_arity) - raise NotImplementedError.new("#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}") + raise NotImplementedError, "#{subject}\##{method_name} has arity of #{sub_method_arity}, but #{overridden_parent}\##{method_name} has arity of #{super_method_arity}" end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index efa2f7a943f..b1ccdcb1df0 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -121,7 +121,7 @@ module Gitlab count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) - # For example, 9 segements gives 10 buckets + # For example, 9 segments gives 10 buckets bucket_segments = bucket_size - 1 width_bucket = Arel::Nodes::NamedFunction @@ -171,7 +171,7 @@ module Gitlab else value end - rescue + rescue StandardError fallback end @@ -188,7 +188,7 @@ module Gitlab return fallback unless client yield client - rescue + rescue StandardError fallback end @@ -210,20 +210,54 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end - def maximum_id(model) - key = :"#{model.name.downcase}_maximum_id" + def maximum_id(model, column = nil) + key = :"#{model.name.downcase.gsub('::', '_')}_maximum_id" + column_to_read = column || :id + strong_memoize(key) do - model.maximum(:id) + model.maximum(column_to_read) end end - def minimum_id(model) - key = :"#{model.name.downcase}_minimum_id" + # rubocop: disable UsageData/LargeTable: + def jira_service_data + data = { + projects_jira_server_active: 0, + projects_jira_cloud_active: 0 + } + + # rubocop: disable CodeReuse/ActiveRecord + JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| + counts = services.group_by do |service| + # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 + service_url = service.data_fields&.url || (service.properties && service.properties['url']) + service_url&.include?('.atlassian.net') ? :cloud : :server + end + + data[:projects_jira_server_active] += counts[:server].size if counts[:server] + data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] + end + + data + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: enable UsageData/LargeTable: + + def minimum_id(model, column = nil) + key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id" + column_to_read = column || :id + strong_memoize(key) do - model.minimum(:id) + model.minimum(column_to_read) end end + def epics_deepest_relationship_level + # rubocop: disable UsageData/LargeTable + { epics_deepest_relationship_level: ::Epic.deepest_relationship_level.to_i } + # rubocop: enable UsageData/LargeTable + end + private def prometheus_client(verify:) @@ -237,7 +271,7 @@ module Gitlab api_url = "#{scheme}://#{server_address}" client = Gitlab::PrometheusClient.new(api_url, allow_local_requests: true, verify: verify) break client if client.ready? - rescue + rescue StandardError nil end end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index fc114a4e9dd..71d106db742 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -24,11 +24,11 @@ module Gitlab end def name - raise NotImplementedError.new + raise NotImplementedError end def describe(_object) - raise NotImplementedError.new + raise NotImplementedError end private @@ -39,7 +39,7 @@ module Gitlab def verify(object) local?(object) ? verify_local(object) : verify_remote(object) - rescue => err + rescue StandardError => err failure(object, err.inspect) end @@ -77,27 +77,27 @@ module Gitlab # This should return an ActiveRecord::Relation suitable for calling #in_batches on def all_relation - raise NotImplementedError.new + raise NotImplementedError end # Should return true if the object is stored locally def local?(_object) - raise NotImplementedError.new + raise NotImplementedError end # The checksum we expect the object to have def expected_checksum(_object) - raise NotImplementedError.new + raise NotImplementedError end # The freshly-recalculated checksum of the object def actual_checksum(_object) - raise NotImplementedError.new + raise NotImplementedError end # Be sure to perform a hard check of the remote object (don't just check DB value) def remote_object_exists?(object) - raise NotImplementedError.new + raise NotImplementedError end end end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb index 4a90ab758fb..d14f8cc4e5e 100644 --- a/lib/gitlab/view/presenter/delegated.rb +++ b/lib/gitlab/view/presenter/delegated.rb @@ -11,7 +11,7 @@ module Gitlab attributes.each do |key, value| if subject.respond_to?(key) - raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!") + raise CannotOverrideMethodError, "#{subject} already respond to #{key}!" end define_singleton_method(key) { value } diff --git a/lib/gitlab/web_ide/config/entry/global.rb b/lib/gitlab/web_ide/config/entry/global.rb index 2c67c7d02d4..2939095fd0f 100644 --- a/lib/gitlab/web_ide/config/entry/global.rb +++ b/lib/gitlab/web_ide/config/entry/global.rb @@ -30,4 +30,4 @@ module Gitlab end end -::Gitlab::WebIde::Config::Entry::Global.prepend_if_ee('EE::Gitlab::WebIde::Config::Entry::Global') +::Gitlab::WebIde::Config::Entry::Global.prepend_mod_with('Gitlab::WebIde::Config::Entry::Global') diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index 9c967d99e3a..b73c2ebb578 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -102,13 +102,13 @@ module Gitlab rescue OpenSSL::SSL::SSLError, EOFError => e ssl_status = Gitlab.config.webpack.dev_server.https ? ' over SSL' : '' raise ManifestLoadError.new("Could not connect to webpack-dev-server at #{uri}#{ssl_status}.\n\nIs SSL enabled? Check that settings in `gitlab.yml` and webpack-dev-server match.", e) - rescue => e + rescue StandardError => e raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri}.\n\nIs webpack-dev-server running? Try running `gdk status webpack` or `gdk tail webpack`.", e) end def load_static_manifest File.read(static_manifest_path) - rescue => e + rescue StandardError => e raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path}.\n\nHave you run `rake gitlab:assets:compile`?", e) end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index edff1540cb3..c83213e973b 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -72,7 +72,7 @@ module Gitlab pkcs7_text = pkcs7_text.sub('-----END SIGNED MESSAGE-----', '-----END PKCS7-----') OpenSSL::PKCS7.new(pkcs7_text) - rescue + rescue StandardError nil end end @@ -87,7 +87,7 @@ module Gitlab def valid_signature? p7.verify([], cert_store, signed_text, OpenSSL::PKCS7::NOVERIFY) - rescue + rescue StandardError nil end @@ -104,7 +104,7 @@ module Gitlab else nil end - rescue + rescue StandardError nil end diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb index 48582c17764..ad85b200130 100644 --- a/lib/gitlab/x509/tag.rb +++ b/lib/gitlab/x509/tag.rb @@ -23,7 +23,7 @@ module Gitlab def signature_text @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) - rescue + rescue StandardError nil end diff --git a/lib/grafana/client.rb b/lib/grafana/client.rb index 7c0e56b61c8..44808f8bb5a 100644 --- a/lib/grafana/client.rb +++ b/lib/grafana/client.rb @@ -62,7 +62,7 @@ module Grafana raise_error 'Grafana returned invalid SSL data' rescue Errno::ECONNREFUSED raise_error 'Connection refused' - rescue => e + rescue StandardError => e raise_error "Grafana request failed due to #{e.class}" end diff --git a/lib/grafana/time_window.rb b/lib/grafana/time_window.rb index 111e3ab7de2..6cc757d77c5 100644 --- a/lib/grafana/time_window.rb +++ b/lib/grafana/time_window.rb @@ -109,7 +109,7 @@ module Grafana def from_ms_since_epoch(time) return if time.nil? - raise Error.new('Expected milliseconds since epoch') unless ms_since_epoch?(time) + raise Error, 'Expected milliseconds since epoch' unless ms_since_epoch?(time) new(cast_ms_to_time(time)) end diff --git a/lib/learn_gitlab.rb b/lib/learn_gitlab.rb deleted file mode 100644 index abceb80bd30..00000000000 --- a/lib/learn_gitlab.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -class LearnGitlab - PROJECT_NAME = 'Learn GitLab' - BOARD_NAME = 'GitLab onboarding' - LABEL_NAME = 'Novice' - - def initialize(current_user) - @current_user = current_user - end - - def available? - project && board && label - end - - def project - @project ||= current_user.projects.find_by_name(PROJECT_NAME) - end - - def board - return unless project - - @board ||= project.boards.find_by_name(BOARD_NAME) - end - - def label - return unless project - - @label ||= project.labels.find_by_name(LABEL_NAME) - end - - private - - attr_reader :current_user -end diff --git a/lib/learn_gitlab/onboarding.rb b/lib/learn_gitlab/onboarding.rb new file mode 100644 index 00000000000..38ffa9eb2e6 --- /dev/null +++ b/lib/learn_gitlab/onboarding.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module LearnGitlab + class Onboarding + include Gitlab::Utils::StrongMemoize + + ACTION_ISSUE_IDS = { + issue_created: 4, + git_write: 6, + pipeline_created: 7, + merge_request_created: 9, + user_added: 8, + trial_started: 2, + required_mr_approvals_enabled: 11, + code_owners_enabled: 10 + }.freeze + + ACTION_DOC_URLS = { + security_scan_enabled: 'https://docs.gitlab.com/ee/user/application_security/security_dashboard/#gitlab-security-dashboard-security-center-and-vulnerability-reports' + }.freeze + + def initialize(namespace) + @namespace = namespace + end + + def completed_percentage + return 0 unless onboarding_progress + + attributes = onboarding_progress.attributes.symbolize_keys + + total_actions = action_columns.count + completed_actions = action_columns.count { |column| attributes[column].present? } + + (completed_actions.to_f / total_actions.to_f * 100).round + end + + private + + def onboarding_progress + strong_memoize(:onboarding_progress) do + OnboardingProgress.find_by(namespace: namespace) # rubocop: disable CodeReuse/ActiveRecord + end + end + + def action_columns + strong_memoize(:action_columns) do + tracked_actions.map { |action_key| OnboardingProgress.column_name(action_key) } + end + end + + def tracked_actions + ACTION_ISSUE_IDS.keys + ACTION_DOC_URLS.keys + end + + attr_reader :namespace + end +end diff --git a/lib/learn_gitlab/project.rb b/lib/learn_gitlab/project.rb new file mode 100644 index 00000000000..599f9940e53 --- /dev/null +++ b/lib/learn_gitlab/project.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module LearnGitlab + class Project + PROJECT_NAME = 'Learn GitLab' + BOARD_NAME = 'GitLab onboarding' + LABEL_NAME = 'Novice' + + def initialize(current_user) + @current_user = current_user + end + + def available? + project && board && label + end + + def project + @project ||= current_user.projects.find_by_name(PROJECT_NAME) + end + + def board + return unless project + + @board ||= project.boards.find_by_name(BOARD_NAME) + end + + def label + return unless project + + @label ||= project.labels.find_by_name(LABEL_NAME) + end + + private + + attr_reader :current_user + end +end diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index a9551ffbd30..7fb959a149c 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -52,12 +52,12 @@ module Mattermost json_response = Gitlab::Json.parse(response.body, legacy_mode: true) unless response.success? - raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error') + raise Mattermost::ClientError, json_response['message'] || 'Undefined error' end json_response rescue JSON::JSONError - raise Mattermost::ClientError.new('Cannot parse response') + raise Mattermost::ClientError, 'Cannot parse response' end end end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index b349b46dc18..523d82f9161 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -174,9 +174,9 @@ module Mattermost def handle_exceptions yield rescue Gitlab::HTTP::Error => e - raise Mattermost::ConnectionError.new(e.message) + raise Mattermost::ConnectionError, e.message rescue Errno::ECONNREFUSED => e - raise Mattermost::ConnectionError.new(e.message) + raise Mattermost::ConnectionError, e.message end def parse_cookie(response) diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 7f1c30e574d..7fbf01f3768 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -79,7 +79,7 @@ module ObjectStorage Provider: 'AWS', S3Config: { Bucket: bucket_name, - Region: credentials[:region], + Region: credentials[:region] || ::Fog::AWS::Storage::DEFAULT_REGION, Endpoint: credentials[:endpoint], PathStyle: config.use_path_style?, UseIamProfile: config.use_iam_profile?, diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 4040bed50a9..774e4768597 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -67,4 +67,4 @@ module Peek end end -Peek::Views::ActiveRecord.prepend_if_ee('EE::Peek::Views::ActiveRecord') +Peek::Views::ActiveRecord.prepend_mod_with('Peek::Views::ActiveRecord') diff --git a/lib/quality/seeders/issues.rb b/lib/quality/seeders/issues.rb index ae19e86546a..ea2db2aa5fe 100644 --- a/lib/quality/seeders/issues.rb +++ b/lib/quality/seeders/issues.rb @@ -30,7 +30,7 @@ module Quality labels: labels.join(',') } params[:closed_at] = params[:created_at] + rand(35).days if params[:state] == 'closed' - issue = ::Issues::CreateService.new(project, team.sample, params).execute + issue = ::Issues::CreateService.new(project: project, current_user: team.sample, params: params).execute if issue.persisted? created_issues_count += 1 diff --git a/lib/safe_zip/entry.rb b/lib/safe_zip/entry.rb index 664e2f52f91..52d70e83154 100644 --- a/lib/safe_zip/entry.rb +++ b/lib/safe_zip/entry.rb @@ -44,7 +44,7 @@ module SafeZip end rescue SafeZip::Extract::Error raise - rescue => e + rescue StandardError => e raise SafeZip::Extract::ExtractError, e.message end @@ -90,7 +90,7 @@ module SafeZip def expand_symlink(source_path) ::File.realpath(source_path, path_dir) - rescue + rescue StandardError raise SafeZip::Extract::SymlinkSourceDoesNotExistError, "Symlink source #{source_path} does not exist" end end diff --git a/lib/security/ci_configuration/base_build_action.rb b/lib/security/ci_configuration/base_build_action.rb new file mode 100644 index 00000000000..b169d780cad --- /dev/null +++ b/lib/security/ci_configuration/base_build_action.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Security + module CiConfiguration + class BaseBuildAction + def initialize(auto_devops_enabled, existing_gitlab_ci_content) + @auto_devops_enabled = auto_devops_enabled + @existing_gitlab_ci_content = existing_gitlab_ci_content || {} + end + + def generate + action = @existing_gitlab_ci_content.present? ? 'update' : 'create' + + update_existing_content! + + { action: action, file_path: '.gitlab-ci.yml', content: prepare_existing_content, default_values_overwritten: @default_values_overwritten } + end + + private + + def generate_includes + includes = @existing_gitlab_ci_content['include'] || [] + includes = Array.wrap(includes) + includes << { 'template' => template } + includes.uniq + end + + def prepare_existing_content + content = @existing_gitlab_ci_content.to_yaml + content = remove_document_delimiter(content) + + content.prepend(comment) + end + + def remove_document_delimiter(content) + content.gsub(/^---\n/, '') + end + + def comment + <<~YAML + # You can override the included template(s) by including variable overrides + # SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings + # Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings + # Note that environment variables can be set in several places + # See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables + YAML + end + end + end +end diff --git a/lib/security/ci_configuration/sast_build_actions.rb b/lib/security/ci_configuration/sast_build_action.rb index b2d684bc1e1..23dd4bd6d14 100644 --- a/lib/security/ci_configuration/sast_build_actions.rb +++ b/lib/security/ci_configuration/sast_build_action.rb @@ -2,33 +2,19 @@ module Security module CiConfiguration - class SastBuildActions - SAST_DEFAULT_ANALYZERS = 'bandit, brakeman, eslint, flawfinder, gosec, kubesec, nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, sobelow, spotbugs' + class SastBuildAction < BaseBuildAction + SAST_DEFAULT_ANALYZERS = 'bandit, brakeman, eslint, flawfinder, gosec, kubesec, nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, semgrep, sobelow, spotbugs' def initialize(auto_devops_enabled, params, existing_gitlab_ci_content) - @auto_devops_enabled = auto_devops_enabled + super(auto_devops_enabled, existing_gitlab_ci_content) @variables = variables(params) - @existing_gitlab_ci_content = existing_gitlab_ci_content || {} @default_sast_values = default_sast_values(params) @default_values_overwritten = false end - def generate - action = @existing_gitlab_ci_content.present? ? 'update' : 'create' - - update_existing_content! - - [{ action: action, file_path: '.gitlab-ci.yml', content: prepare_existing_content, default_values_overwritten: @default_values_overwritten }] - end - private def variables(params) - # This early return is necessary for supporting REST API. - # Will be removed during the implementation of - # https://gitlab.com/gitlab-org/gitlab/-/issues/246737 - return params unless params['global'].present? - collect_values(params, 'value') end @@ -71,19 +57,12 @@ module Security @existing_gitlab_ci_content['stages'] = set_stages @existing_gitlab_ci_content['variables'] = set_variables(global_variables, @existing_gitlab_ci_content) @existing_gitlab_ci_content['sast'] = set_sast_block - @existing_gitlab_ci_content['include'] = set_includes + @existing_gitlab_ci_content['include'] = generate_includes @existing_gitlab_ci_content.select! { |k, v| v.present? } @existing_gitlab_ci_content['sast'].select! { |k, v| v.present? } end - def set_includes - includes = @existing_gitlab_ci_content['include'] || [] - includes = includes.is_a?(Array) ? includes : [includes] - includes << { 'template' => template } - includes.uniq - end - def set_stages existing_stages = @existing_gitlab_ci_content['stages'] || [] base_stages = @auto_devops_enabled ? auto_devops_stages : ['test'] @@ -121,26 +100,6 @@ module Security sast_content.select { |k, v| v.present? } end - def prepare_existing_content - content = @existing_gitlab_ci_content.to_yaml - content = remove_document_delimeter(content) - - content.prepend(sast_comment) - end - - def remove_document_delimeter(content) - content.gsub(/^---\n/, '') - end - - def sast_comment - <<~YAML - # You can override the included template(s) by including variable overrides - # See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings - # Note that environment variables can be set in several places - # See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables - YAML - end - def template return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled diff --git a/lib/security/ci_configuration/secret_detection_build_action.rb b/lib/security/ci_configuration/secret_detection_build_action.rb new file mode 100644 index 00000000000..5d513bf5547 --- /dev/null +++ b/lib/security/ci_configuration/secret_detection_build_action.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Security + module CiConfiguration + class SecretDetectionBuildAction < BaseBuildAction + private + + def update_existing_content! + @existing_gitlab_ci_content['include'] = generate_includes + end + + def template + return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled + + 'Security/Secret-Detection.gitlab-ci.yml' + end + end + end +end diff --git a/lib/sidebars/concerns/container_with_html_options.rb b/lib/sidebars/concerns/container_with_html_options.rb new file mode 100644 index 00000000000..873cb5b0de9 --- /dev/null +++ b/lib/sidebars/concerns/container_with_html_options.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Sidebars + module Concerns + module ContainerWithHtmlOptions + # The attributes returned from this method + # will be applied to helper methods like + # `link_to` or the div containing the container. + def container_html_options + { + aria: { label: title } + }.merge(extra_container_html_options) + end + + # Classes will override mostly this method + # and not `container_html_options`. + def extra_container_html_options + {} + end + + # The attributes returned from this method + # will be applied to helper methods like + # `link_to` or the div containing the container + # when it is collapsed. + def collapsed_container_html_options + { + aria: { label: title } + }.merge(extra_collapsed_container_html_options) + end + + # Classes should mostly override this method + # and not `collapsed_container_html_options`. + def extra_collapsed_container_html_options + {} + end + + # Attributes to pass to the html_options attribute + # in the helper method that sets the active class + # on each element. + def nav_link_html_options + {} + end + + def title + raise NotImplementedError + end + + # The attributes returned from this method + # will be applied right next to the title, + # for example in the span that renders the title. + def title_html_options + {} + end + + def link + raise NotImplementedError + end + end + end +end diff --git a/lib/sidebars/concerns/has_active_routes.rb b/lib/sidebars/concerns/has_active_routes.rb new file mode 100644 index 00000000000..50c9f8c85a1 --- /dev/null +++ b/lib/sidebars/concerns/has_active_routes.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Sidebars + module Concerns + module HasActiveRoutes + # This method will indicate for which paths or + # controllers, the menu or menu item should + # be set as active. + # + # The returned values are passed to the `nav_link` helper method, + # so the params can be either `path`, `page`, `controller`. + # Param 'action' is not supported. + def active_routes + {} + end + end + end +end diff --git a/lib/sidebars/concerns/has_hint.rb b/lib/sidebars/concerns/has_hint.rb new file mode 100644 index 00000000000..dc4f765e974 --- /dev/null +++ b/lib/sidebars/concerns/has_hint.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# This module has the necessary methods to store +# hints for menus. Hints are elements displayed +# when the user hover the menu item. +module Sidebars + module Concerns + module HasHint + def show_hint? + false + end + + def hint_html_options + {} + end + end + end +end diff --git a/lib/sidebars/concerns/has_icon.rb b/lib/sidebars/concerns/has_icon.rb new file mode 100644 index 00000000000..afff466239d --- /dev/null +++ b/lib/sidebars/concerns/has_icon.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# This module has the necessary methods to show +# sprites or images next to the menu item. +module Sidebars + module Concerns + module HasIcon + def sprite_icon + nil + end + + def sprite_icon_html_options + {} + end + + def image_path + nil + end + + def image_html_options + {} + end + + def icon_or_image? + sprite_icon || image_path + end + end + end +end diff --git a/lib/sidebars/concerns/has_pill.rb b/lib/sidebars/concerns/has_pill.rb new file mode 100644 index 00000000000..5082ed477e6 --- /dev/null +++ b/lib/sidebars/concerns/has_pill.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# This module introduces the logic to show the "pill" element +# next to the menu item, indicating the a count. +module Sidebars + module Concerns + module HasPill + def has_pill? + false + end + + # In this method we will need to provide the query + # to retrieve the elements count + def pill_count + raise NotImplementedError + end + + def pill_html_options + {} + end + end + end +end diff --git a/lib/sidebars/concerns/positionable_list.rb b/lib/sidebars/concerns/positionable_list.rb new file mode 100644 index 00000000000..0bbe1d918e5 --- /dev/null +++ b/lib/sidebars/concerns/positionable_list.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +# This module handles element positions in a list. +module Sidebars + module Concerns + module PositionableList + def add_element(list, element) + return unless element + + list << element + end + + def insert_element_before(list, before_element, new_element) + return unless new_element + + index = index_of(list, before_element) + + if index + list.insert(index, new_element) + else + list.unshift(new_element) + end + end + + def insert_element_after(list, after_element, new_element) + return unless new_element + + index = index_of(list, after_element) + + if index + list.insert(index + 1, new_element) + else + add_element(list, new_element) + end + end + + def replace_element(list, element_to_replace, new_element) + return unless new_element + + index = index_of(list, element_to_replace) + + return unless index + + list[index] = new_element + end + + private + + # Classes including this method will have to define + # the way to identify elements through this method + def index_of(list, element) + raise NotImplementedError + end + end + end +end diff --git a/lib/sidebars/concerns/renderable.rb b/lib/sidebars/concerns/renderable.rb new file mode 100644 index 00000000000..750efa2fcb8 --- /dev/null +++ b/lib/sidebars/concerns/renderable.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Sidebars + module Concerns + module Renderable + # This method will control whether the menu or menu_item + # should be rendered. It will be overriden by specific + # classes. + def render? + true + end + end + end +end diff --git a/lib/sidebars/context.rb b/lib/sidebars/context.rb new file mode 100644 index 00000000000..d9ac2705aaf --- /dev/null +++ b/lib/sidebars/context.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# This class stores all the information needed to display and +# render the sidebar and menus. +# It usually stores information regarding the context and calculated +# values where the logic is in helpers. +module Sidebars + class Context + attr_reader :current_user, :container + + def initialize(current_user:, container:, **args) + @current_user = current_user + @container = container + + args.each do |key, value| + singleton_class.public_send(:attr_reader, key) # rubocop:disable GitlabSecurity/PublicSend + instance_variable_set("@#{key}", value) + end + end + end +end diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb new file mode 100644 index 00000000000..d81e413f4a9 --- /dev/null +++ b/lib/sidebars/menu.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Sidebars + class Menu + extend ::Gitlab::Utils::Override + include ::Gitlab::Routing + include GitlabRoutingHelper + include Gitlab::Allowable + include ::Sidebars::Concerns::HasPill + include ::Sidebars::Concerns::HasIcon + include ::Sidebars::Concerns::PositionableList + include ::Sidebars::Concerns::Renderable + include ::Sidebars::Concerns::ContainerWithHtmlOptions + include ::Sidebars::Concerns::HasActiveRoutes + + attr_reader :context + delegate :current_user, :container, to: :@context + + def initialize(context) + @context = context + @items = [] + + configure_menu_items + end + + def configure_menu_items + true + end + + override :render? + def render? + has_renderable_items? + end + + # Menus might have or not a link + override :link + def link + nil + end + + # This method normalizes the information retrieved from the submenus and this menu + # Value from menus is something like: [{ path: 'foo', path: 'bar', controller: :foo }] + # This method filters the information and returns: { path: ['foo', 'bar'], controller: :foo } + def all_active_routes + @all_active_routes ||= begin + ([active_routes] + renderable_items.map(&:active_routes)).flatten.each_with_object({}) do |pairs, hash| + pairs.each do |k, v| + hash[k] ||= [] + hash[k] += Array(v) + hash[k].uniq! + end + + hash + end + end + end + + # Returns whether the menu has any menu item, no + # matter whether it is renderable or not + def has_items? + @items.any? + end + + # Returns all renderable menu items + def renderable_items + @renderable_items ||= @items.select(&:render?) + end + + # Returns whether the menu has any renderable menu item + def has_renderable_items? + renderable_items.any? + end + + def add_item(item) + add_element(@items, item) + end + + def insert_item_before(before_item, new_item) + insert_element_before(@items, before_item, new_item) + end + + def insert_item_after(after_item, new_item) + insert_element_after(@items, after_item, new_item) + end + + private + + override :index_of + def index_of(list, element) + list.index { |e| e.item_id == element } + end + end +end diff --git a/lib/sidebars/menu_item.rb b/lib/sidebars/menu_item.rb new file mode 100644 index 00000000000..b0a12e769dc --- /dev/null +++ b/lib/sidebars/menu_item.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Sidebars + class MenuItem + attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options + + def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}) + @title = title + @link = link + @active_routes = active_routes + @item_id = item_id + @container_html_options = { aria: { label: title } }.merge(container_html_options) + @sprite_icon = sprite_icon + @sprite_icon_html_options = sprite_icon_html_options + @hint_html_options = hint_html_options + end + + def show_hint? + hint_html_options.present? + end + + def render? + true + end + end +end diff --git a/lib/sidebars/nil_menu_item.rb b/lib/sidebars/nil_menu_item.rb new file mode 100644 index 00000000000..9ff7fd0d6d6 --- /dev/null +++ b/lib/sidebars/nil_menu_item.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Sidebars + class NilMenuItem < MenuItem + extend ::Gitlab::Utils::Override + + def initialize(item_id:) + super(item_id: item_id, title: nil, link: nil, active_routes: {}) + end + + override :render? + def render? + false + end + end +end diff --git a/lib/sidebars/panel.rb b/lib/sidebars/panel.rb new file mode 100644 index 00000000000..75b3ba65729 --- /dev/null +++ b/lib/sidebars/panel.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Sidebars + class Panel + extend ::Gitlab::Utils::Override + include ::Sidebars::Concerns::PositionableList + + attr_reader :context, :scope_menu, :hidden_menu + + def initialize(context) + @context = context + @scope_menu = nil + @hidden_menu = nil + @menus = [] + + configure_menus + end + + def configure_menus + # No-op + end + + def add_menu(menu) + add_element(@menus, menu) + end + + def insert_menu_before(before_menu, new_menu) + insert_element_before(@menus, before_menu, new_menu) + end + + def insert_menu_after(after_menu, new_menu) + insert_element_after(@menus, after_menu, new_menu) + end + + def replace_menu(menu_to_replace, new_menu) + replace_element(@menus, menu_to_replace, new_menu) + end + + def set_scope_menu(scope_menu) + @scope_menu = scope_menu + end + + def set_hidden_menu(hidden_menu) + @hidden_menu = hidden_menu + end + + def aria_label + raise NotImplementedError + end + + def has_renderable_menus? + renderable_menus.any? + end + + def renderable_menus + @renderable_menus ||= @menus.select(&:render?) + end + + def container + context.container + end + + # Auxiliar method that helps with the migration from + # regular views to the new logic + def render_raw_scope_menu_partial + # No-op + end + + # Auxiliar method that helps with the migration from + # regular views to the new logic. + # + # Any menu inside this partial will be added after + # all the menus added in the `configure_menus` + # method. + def render_raw_menus_partial + # No-op + end + + private + + override :index_of + def index_of(list, element) + list.index { |e| e.is_a?(element) } + end + end +end diff --git a/lib/sidebars/projects/context.rb b/lib/sidebars/projects/context.rb new file mode 100644 index 00000000000..4c82309035d --- /dev/null +++ b/lib/sidebars/projects/context.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + class Context < ::Sidebars::Context + def initialize(current_user:, container:, **args) + super(current_user: current_user, container: container, project: container, **args) + end + end + end +end diff --git a/lib/sidebars/projects/menus/analytics_menu.rb b/lib/sidebars/projects/menus/analytics_menu.rb new file mode 100644 index 00000000000..660965005c3 --- /dev/null +++ b/lib/sidebars/projects/menus/analytics_menu.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class AnalyticsMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :configure_menu_items + def configure_menu_items + return false unless can?(context.current_user, :read_analytics, context.project) + + add_item(ci_cd_analytics_menu_item) + add_item(repository_analytics_menu_item) + add_item(cycle_analytics_menu_item) + + true + end + + override :link + def link + return cycle_analytics_menu_item.link if cycle_analytics_menu_item.render? + + renderable_items.first.link + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-analytics' + } + end + + override :title + def title + _('Analytics') + end + + override :sprite_icon + def sprite_icon + 'chart' + end + + private + + def ci_cd_analytics_menu_item + if !context.project.feature_available?(:builds, context.current_user) || + !can?(context.current_user, :read_build, context.project) || + context.project.empty_repo? + return ::Sidebars::NilMenuItem.new(item_id: :ci_cd_analytics) + end + + ::Sidebars::MenuItem.new( + title: _('CI/CD'), + link: charts_project_pipelines_path(context.project), + active_routes: { path: 'pipelines#charts' }, + item_id: :ci_cd_analytics + ) + end + + def repository_analytics_menu_item + if context.project.empty_repo? + return ::Sidebars::NilMenuItem.new(item_id: :repository_analytics) + end + + ::Sidebars::MenuItem.new( + title: _('Repository'), + link: charts_project_graph_path(context.project, context.current_ref), + container_html_options: { class: 'shortcuts-repository-charts' }, + active_routes: { path: 'graphs#charts' }, + item_id: :repository_analytics + ) + end + + def cycle_analytics_menu_item + strong_memoize(:cycle_analytics_menu_item) do + unless can?(context.current_user, :read_cycle_analytics, context.project) + next ::Sidebars::NilMenuItem.new(item_id: :cycle_analytics) + end + + ::Sidebars::MenuItem.new( + title: _('Value Stream'), + link: project_cycle_analytics_path(context.project), + container_html_options: { class: 'shortcuts-project-cycle-analytics' }, + active_routes: { path: 'cycle_analytics#show' }, + item_id: :cycle_analytics + ) + end + end + end + end + end +end + +Sidebars::Projects::Menus::AnalyticsMenu.prepend_mod_with('Sidebars::Projects::Menus::AnalyticsMenu') diff --git a/lib/sidebars/projects/menus/ci_cd_menu.rb b/lib/sidebars/projects/menus/ci_cd_menu.rb new file mode 100644 index 00000000000..042ad17fdfc --- /dev/null +++ b/lib/sidebars/projects/menus/ci_cd_menu.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class CiCdMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return unless can?(context.current_user, :read_build, context.project) + + add_item(pipelines_menu_item) + add_item(pipelines_editor_menu_item) + add_item(jobs_menu_item) + add_item(artifacts_menu_item) + add_item(pipeline_schedules_menu_item) + end + + override :link + def link + project_pipelines_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-pipelines rspec-link-pipelines' + } + end + + override :title + def title + _('CI/CD') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-pipelines-link' + } + end + + override :sprite_icon + def sprite_icon + 'rocket' + end + + private + + def pipelines_menu_item + ::Sidebars::MenuItem.new( + title: _('Pipelines'), + link: project_pipelines_path(context.project), + container_html_options: { class: 'shortcuts-pipelines' }, + active_routes: { path: pipelines_routes }, + item_id: :pipelines + ) + end + + def pipelines_routes + %w[ + pipelines#index + pipelines#show + pipelines#new + ] + end + + def pipelines_editor_menu_item + unless context.can_view_pipeline_editor + return ::Sidebars::NilMenuItem.new(item_id: :pipelines_editor) + end + + ::Sidebars::MenuItem.new( + title: s_('Pipelines|Editor'), + link: project_ci_pipeline_editor_path(context.project), + active_routes: { path: 'projects/ci/pipeline_editor#show' }, + item_id: :pipelines_editor + ) + end + + def jobs_menu_item + ::Sidebars::MenuItem.new( + title: _('Jobs'), + link: project_jobs_path(context.project), + container_html_options: { class: 'shortcuts-builds' }, + active_routes: { controller: :jobs }, + item_id: :jobs + ) + end + + def artifacts_menu_item + unless Feature.enabled?(:artifacts_management_page, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :artifacts) + end + + ::Sidebars::MenuItem.new( + title: _('Artifacts'), + link: project_artifacts_path(context.project), + container_html_options: { class: 'shortcuts-builds' }, + active_routes: { path: 'artifacts#index' }, + item_id: :artifacts + ) + end + + def pipeline_schedules_menu_item + ::Sidebars::MenuItem.new( + title: _('Schedules'), + link: pipeline_schedules_path(context.project), + container_html_options: { class: 'shortcuts-builds' }, + active_routes: { controller: :pipeline_schedules }, + item_id: :pipeline_schedules + ) + end + end + end + end +end + +Sidebars::Projects::Menus::CiCdMenu.prepend_mod_with('Sidebars::Projects::Menus::CiCdMenu') diff --git a/lib/sidebars/projects/menus/confluence_menu.rb b/lib/sidebars/projects/menus/confluence_menu.rb new file mode 100644 index 00000000000..0d83238fa82 --- /dev/null +++ b/lib/sidebars/projects/menus/confluence_menu.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class ConfluenceMenu < ::Sidebars::Menu + override :link + def link + project_wikis_confluence_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-confluence' + } + end + + override :title + def title + _('Confluence') + end + + override :image_path + def image_path + 'confluence.svg' + end + + override :image_html_options + def image_html_options + { + alt: title + } + end + + override :render? + def render? + context.project.has_confluence? + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb new file mode 100644 index 00000000000..f3d13e12258 --- /dev/null +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class DeploymentsMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return false if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + + add_item(feature_flags_menu_item) + add_item(environments_menu_item) + add_item(releases_menu_item) + + true + end + + override :link + def link + renderable_items.first.link + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-deployments' + } + end + + override :title + def title + _('Deployments') + end + + override :sprite_icon + def sprite_icon + 'environment' + end + + private + + def feature_flags_menu_item + unless can?(context.current_user, :read_feature_flag, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :feature_flags) + end + + ::Sidebars::MenuItem.new( + title: _('Feature Flags'), + link: project_feature_flags_path(context.project), + active_routes: { controller: :feature_flags }, + container_html_options: { class: 'shortcuts-feature-flags' }, + item_id: :feature_flags + ) + end + + def environments_menu_item + unless can?(context.current_user, :read_environment, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :environments) + end + + ::Sidebars::MenuItem.new( + title: _('Environments'), + link: project_environments_path(context.project), + active_routes: { controller: :environments }, + container_html_options: { class: 'shortcuts-environments' }, + item_id: :environments + ) + end + + def releases_menu_item + if !can?(context.current_user, :read_release, context.project) || + context.project.empty_repo? + return ::Sidebars::NilMenuItem.new(item_id: :releases) + end + + ::Sidebars::MenuItem.new( + title: _('Releases'), + link: project_releases_path(context.project), + item_id: :releases, + active_routes: { controller: :releases }, + container_html_options: { class: 'shortcuts-deployments-releases' } + ) + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb new file mode 100644 index 00000000000..136d30f38c3 --- /dev/null +++ b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class ExternalIssueTrackerMenu < ::Sidebars::Menu + override :link + def link + external_issue_tracker.issue_tracker_path + end + + override :extra_container_html_options + def extra_container_html_options + { + target: '_blank', + rel: 'noopener noreferrer', + class: 'shortcuts-external_tracker' + } + end + + override :extra_collapsed_container_html_options + def extra_collapsed_container_html_options + { + target: '_blank', + rel: 'noopener noreferrer' + } + end + + override :title + def title + external_issue_tracker.title + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-issues-link' + } + end + + override :sprite_icon + def sprite_icon + 'external-link' + end + + override :render? + def render? + external_issue_tracker.present? + end + + private + + def external_issue_tracker + @external_issue_tracker ||= context.project.external_issue_tracker + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/external_wiki_menu.rb b/lib/sidebars/projects/menus/external_wiki_menu.rb new file mode 100644 index 00000000000..825f0ca5e8b --- /dev/null +++ b/lib/sidebars/projects/menus/external_wiki_menu.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class ExternalWikiMenu < ::Sidebars::Menu + override :link + def link + external_wiki.external_wiki_url + end + + override :extra_container_html_options + def extra_container_html_options + { + target: '_blank', + rel: 'noopener noreferrer', + class: 'shortcuts-external_wiki' + } + end + + override :extra_collapsed_container_html_options + def extra_collapsed_container_html_options + { + target: '_blank', + rel: 'noopener noreferrer' + } + end + + override :title + def title + s_('ExternalWikiService|External wiki') + end + + override :sprite_icon + def sprite_icon + 'external-link' + end + + override :render? + def render? + external_wiki.present? + end + + private + + def external_wiki + @external_wiki ||= context.project.external_wiki + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/hidden_menu.rb b/lib/sidebars/projects/menus/hidden_menu.rb new file mode 100644 index 00000000000..c273ee8b74f --- /dev/null +++ b/lib/sidebars/projects/menus/hidden_menu.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class HiddenMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(activity_menu_item) + add_item(graph_menu_item) + add_item(new_issue_menu_item) + add_item(jobs_menu_item) + add_item(commits_menu_item) + add_item(issue_boards_menu_item) + + true + end + + private + + def activity_menu_item + ::Sidebars::MenuItem.new( + title: _('Activity'), + link: activity_project_path(context.project), + active_routes: {}, + container_html_options: { class: 'shortcuts-project-activity' }, + item_id: :activity + ) + end + + def graph_menu_item + if !can?(context.current_user, :download_code, context.project) || + context.project.empty_repo? + return ::Sidebars::NilMenuItem.new(item_id: :graph) + end + + ::Sidebars::MenuItem.new( + title: _('Graph'), + link: project_network_path(context.project, context.current_ref), + active_routes: {}, + container_html_options: { class: 'shortcuts-network' }, + item_id: :graph + ) + end + + def new_issue_menu_item + unless can?(context.current_user, :read_issue, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :new_issue) + end + + ::Sidebars::MenuItem.new( + title: _('Create a new issue'), + link: new_project_issue_path(context.project), + active_routes: {}, + container_html_options: { class: 'shortcuts-new-issue' }, + item_id: :new_issue + ) + end + + def jobs_menu_item + unless can?(context.current_user, :read_build, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :jobs) + end + + ::Sidebars::MenuItem.new( + title: _('Jobs'), + link: project_jobs_path(context.project), + active_routes: {}, + container_html_options: { class: 'shortcuts-builds' }, + item_id: :jobs + ) + end + + def commits_menu_item + if !can?(context.current_user, :download_code, context.project) || + context.project.empty_repo? + return ::Sidebars::NilMenuItem.new(item_id: :commits) + end + + ::Sidebars::MenuItem.new( + title: _('Commits'), + link: project_commits_path(context.project), + active_routes: {}, + container_html_options: { class: 'shortcuts-commits' }, + item_id: :commits + ) + end + + def issue_boards_menu_item + unless can?(context.current_user, :read_issue, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :issue_boards) + end + + ::Sidebars::MenuItem.new( + title: _('Issue Boards'), + link: project_boards_path(context.project), + active_routes: {}, + container_html_options: { class: 'shortcuts-issue-boards' }, + item_id: :issue_boards + ) + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb new file mode 100644 index 00000000000..75b6cae295f --- /dev/null +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class InfrastructureMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return false if Feature.disabled?(:sidebar_refactor, context.current_user) + return false unless context.project.feature_available?(:operations, context.current_user) + + add_item(kubernetes_menu_item) + add_item(serverless_menu_item) + add_item(terraform_menu_item) + + true + end + + override :link + def link + project_clusters_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-infrastructure' + } + end + + override :title + def title + _('Infrastructure') + end + + override :sprite_icon + def sprite_icon + 'cloud-gear' + end + + private + + def kubernetes_menu_item + unless can?(context.current_user, :read_cluster, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :kubernetes) + end + + ::Sidebars::MenuItem.new( + title: _('Kubernetes clusters'), + link: project_clusters_path(context.project), + active_routes: { controller: [:cluster_agents, :clusters] }, + container_html_options: { class: 'shortcuts-kubernetes' }, + hint_html_options: kubernetes_hint_html_options, + item_id: :kubernetes + ) + end + + def kubernetes_hint_html_options + return {} unless context.show_cluster_hint + + { disabled: true, + data: { trigger: 'manual', + container: 'body', + placement: 'right', + highlight: UserCalloutsHelper::GKE_CLUSTER_INTEGRATION, + highlight_priority: UserCallout.feature_names[:GKE_CLUSTER_INTEGRATION], + dismiss_endpoint: user_callouts_path, + auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } + end + + def serverless_menu_item + unless can?(context.current_user, :read_cluster, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :serverless) + end + + ::Sidebars::MenuItem.new( + title: _('Serverless platform'), + link: project_serverless_functions_path(context.project), + active_routes: { controller: :functions }, + item_id: :serverless + ) + end + + def terraform_menu_item + unless can?(context.current_user, :read_terraform_state, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :terraform) + end + + ::Sidebars::MenuItem.new( + title: _('Terraform'), + link: project_terraform_index_path(context.project), + active_routes: { controller: :terraform }, + item_id: :terraform + ) + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb new file mode 100644 index 00000000000..9840f644179 --- /dev/null +++ b/lib/sidebars/projects/menus/issues_menu.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class IssuesMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :configure_menu_items + def configure_menu_items + return unless can?(context.current_user, :read_issue, context.project) + + add_item(list_menu_item) + add_item(boards_menu_item) + add_item(labels_menu_item) + add_item(service_desk_menu_item) + add_item(milestones_menu_item) + + true + end + + override :link + def link + project_issues_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-issues' + } + end + + override :title + def title + _('Issues') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-issues-link' + } + end + + override :sprite_icon + def sprite_icon + 'issues' + end + + override :active_routes + def active_routes + { controller: 'projects/issues' } + end + + override :has_pill? + def has_pill? + strong_memoize(:has_pill) do + context.project.issues_enabled? + end + end + + override :pill_count + def pill_count + strong_memoize(:pill_count) do + context.project.open_issues_count(context.current_user) + end + end + + override :pill_html_options + def pill_html_options + { + class: 'issue_counter' + } + end + + private + + def list_menu_item + ::Sidebars::MenuItem.new( + title: _('List'), + link: project_issues_path(context.project), + active_routes: { path: 'projects/issues#index' }, + container_html_options: { aria: { label: _('Issues') } }, + item_id: :issue_list + ) + end + + def boards_menu_item + title = context.project.multiple_issue_boards_available? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board') + + ::Sidebars::MenuItem.new( + title: title, + link: project_boards_path(context.project), + active_routes: { controller: :boards }, + item_id: :boards + ) + end + + def labels_menu_item + if Feature.enabled?(:sidebar_refactor, context.current_user) + return ::Sidebars::NilMenuItem.new(item_id: :labels) + end + + ::Sidebars::MenuItem.new( + title: _('Labels'), + link: project_labels_path(context.project), + active_routes: { controller: :labels }, + item_id: :labels + ) + end + + def service_desk_menu_item + ::Sidebars::MenuItem.new( + title: _('Service Desk'), + link: service_desk_project_issues_path(context.project), + active_routes: { path: 'issues#service_desk' }, + item_id: :service_desk + ) + end + + def milestones_menu_item + ::Sidebars::MenuItem.new( + title: _('Milestones'), + link: project_milestones_path(context.project), + active_routes: { controller: :milestones }, + item_id: :milestones + ) + end + end + end + end +end + +Sidebars::Projects::Menus::IssuesMenu.prepend_mod_with('Sidebars::Projects::Menus::IssuesMenu') diff --git a/lib/sidebars/projects/menus/labels_menu.rb b/lib/sidebars/projects/menus/labels_menu.rb new file mode 100644 index 00000000000..12cf0444994 --- /dev/null +++ b/lib/sidebars/projects/menus/labels_menu.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class LabelsMenu < ::Sidebars::Menu + override :link + def link + project_labels_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-labels' + } + end + + override :title + def title + _('Labels') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-labels-link' + } + end + + override :active_routes + def active_routes + { controller: :labels } + end + + override :sprite_icon + def sprite_icon + 'label' + end + + override :render? + def render? + return false if Feature.enabled?(:sidebar_refactor, context.current_user) + + can?(context.current_user, :read_label, context.project) && !context.project.issues_enabled? + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/learn_gitlab_menu.rb b/lib/sidebars/projects/menus/learn_gitlab_menu.rb new file mode 100644 index 00000000000..e3fcd8f25d5 --- /dev/null +++ b/lib/sidebars/projects/menus/learn_gitlab_menu.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class LearnGitlabMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :link + def link + project_learn_gitlab_path(context.project) + end + + override :active_routes + def active_routes + { controller: :learn_gitlab } + end + + override :title + def title + _('Learn GitLab') + end + + override :has_pill? + def has_pill? + context.learn_gitlab_experiment_enabled + end + + override :pill_count + def pill_count + strong_memoize(:pill_count) do + percentage = LearnGitlab::Onboarding.new(context.project.namespace).completed_percentage + + "#{percentage}%" + end + end + + override :extra_container_html_options + def nav_link_html_options + { + class: 'home', + data: { + track_action: 'click_menu', + track_property: context.learn_gitlab_experiment_tracking_category, + track_label: 'learn_gitlab' + } + } + end + + override :image_path + def image_path + 'learn_gitlab/graduation_hat.svg' + end + + override :render? + def render? + context.learn_gitlab_experiment_enabled + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/members_menu.rb b/lib/sidebars/projects/menus/members_menu.rb new file mode 100644 index 00000000000..498bfa74261 --- /dev/null +++ b/lib/sidebars/projects/menus/members_menu.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class MembersMenu < ::Sidebars::Menu + override :link + def link + project_project_members_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + id: 'js-onboarding-members-link' + } + end + + override :title + def title + _('Members') + end + + override :sprite_icon + def sprite_icon + 'users' + end + + override :render? + def render? + return false if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + + can?(context.current_user, :read_project_member, context.project) + end + + override :active_routes + def active_routes + { controller: :project_members } + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/merge_requests_menu.rb b/lib/sidebars/projects/menus/merge_requests_menu.rb new file mode 100644 index 00000000000..fe501667d37 --- /dev/null +++ b/lib/sidebars/projects/menus/merge_requests_menu.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class MergeRequestsMenu < ::Sidebars::Menu + override :link + def link + project_merge_requests_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-merge_requests' + } + end + + override :title + def title + _('Merge requests') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-mr-link' + } + end + + override :sprite_icon + def sprite_icon + 'git-merge' + end + + override :render? + def render? + can?(context.current_user, :read_merge_request, context.project) && + context.project.repo_exists? + end + + override :has_pill? + def has_pill? + true + end + + override :pill_count + def pill_count + @pill_count ||= context.project.open_merge_requests_count + end + + override :pill_html_options + def pill_html_options + { + class: 'merge_counter js-merge-counter' + } + end + + override :active_routes + def active_routes + if context.project.issues_enabled? + { controller: :merge_requests } + else + { controller: [:merge_requests, :milestones] } + end + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/monitor_menu.rb b/lib/sidebars/projects/menus/monitor_menu.rb new file mode 100644 index 00000000000..18c990d0e1f --- /dev/null +++ b/lib/sidebars/projects/menus/monitor_menu.rb @@ -0,0 +1,246 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class MonitorMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return false unless context.project.feature_available?(:operations, context.current_user) + + add_item(metrics_dashboard_menu_item) + add_item(logs_menu_item) + add_item(tracing_menu_item) + add_item(error_tracking_menu_item) + add_item(alert_management_menu_item) + add_item(incidents_menu_item) + add_item(serverless_menu_item) + add_item(terraform_menu_item) + add_item(kubernetes_menu_item) + add_item(environments_menu_item) + add_item(feature_flags_menu_item) + add_item(product_analytics_menu_item) + + true + end + + override :link + def link + if can?(context.current_user, :read_environment, context.project) + metrics_project_environments_path(context.project) + else + project_feature_flags_path(context.project) + end + end + + override :extra_container_html_options + def extra_container_html_options + { + class: Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? 'shortcuts-monitor' : 'shortcuts-operations' + } + end + + override :title + def title + Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? _('Monitor') : _('Operations') + end + + override :sprite_icon + def sprite_icon + Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? 'monitor' : 'cloud-gear' + end + + override :active_routes + def active_routes + { controller: [:user, :gcp] } + end + + private + + def metrics_dashboard_menu_item + unless can?(context.current_user, :metrics_dashboard, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :metrics) + end + + ::Sidebars::MenuItem.new( + title: _('Metrics'), + link: project_metrics_dashboard_path(context.project), + active_routes: { path: 'metrics_dashboard#show' }, + container_html_options: { class: 'shortcuts-metrics' }, + item_id: :metrics + ) + end + + def logs_menu_item + if !can?(context.current_user, :read_environment, context.project) || + !can?(context.current_user, :read_pod_logs, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :logs) + end + + ::Sidebars::MenuItem.new( + title: _('Logs'), + link: project_logs_path(context.project), + active_routes: { path: 'logs#index' }, + item_id: :logs + ) + end + + def tracing_menu_item + if !can?(context.current_user, :read_environment, context.project) || + !can?(context.current_user, :admin_project, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :tracing) + end + + ::Sidebars::MenuItem.new( + title: _('Tracing'), + link: project_tracing_path(context.project), + active_routes: { path: 'tracings#show' }, + item_id: :tracing + ) + end + + def error_tracking_menu_item + unless can?(context.current_user, :read_sentry_issue, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :error_tracking) + end + + ::Sidebars::MenuItem.new( + title: _('Error Tracking'), + link: project_error_tracking_index_path(context.project), + active_routes: { controller: :error_tracking }, + item_id: :error_tracking + ) + end + + def alert_management_menu_item + unless can?(context.current_user, :read_alert_management_alert, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :alert_management) + end + + ::Sidebars::MenuItem.new( + title: _('Alerts'), + link: project_alert_management_index_path(context.project), + active_routes: { controller: :alert_management }, + item_id: :alert_management + ) + end + + def incidents_menu_item + unless can?(context.current_user, :read_issue, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :incidents) + end + + ::Sidebars::MenuItem.new( + title: _('Incidents'), + link: project_incidents_path(context.project), + active_routes: { controller: [:incidents, :incident_management] }, + item_id: :incidents + ) + end + + def serverless_menu_item + if Feature.enabled?(:sidebar_refactor, context.current_user) || + !can?(context.current_user, :read_cluster, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :serverless) + end + + ::Sidebars::MenuItem.new( + title: _('Serverless'), + link: project_serverless_functions_path(context.project), + active_routes: { controller: :functions }, + item_id: :serverless + ) + end + + def terraform_menu_item + if Feature.enabled?(:sidebar_refactor, context.current_user) || + !can?(context.current_user, :read_terraform_state, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :terraform) + end + + ::Sidebars::MenuItem.new( + title: _('Terraform'), + link: project_terraform_index_path(context.project), + active_routes: { controller: :terraform }, + item_id: :terraform + ) + end + + def kubernetes_menu_item + if Feature.enabled?(:sidebar_refactor, context.current_user) || + !can?(context.current_user, :read_cluster, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :kubernetes) + end + + ::Sidebars::MenuItem.new( + title: _('Kubernetes'), + link: project_clusters_path(context.project), + active_routes: { controller: [:cluster_agents, :clusters] }, + container_html_options: { class: 'shortcuts-kubernetes' }, + hint_html_options: kubernetes_hint_html_options, + item_id: :kubernetes + ) + end + + def kubernetes_hint_html_options + return {} unless context.show_cluster_hint + + { disabled: true, + data: { trigger: 'manual', + container: 'body', + placement: 'right', + highlight: UserCalloutsHelper::GKE_CLUSTER_INTEGRATION, + highlight_priority: UserCallout.feature_names[:GKE_CLUSTER_INTEGRATION], + dismiss_endpoint: user_callouts_path, + auto_devops_help_path: help_page_path('topics/autodevops/index.md') } } + end + + def environments_menu_item + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || + !can?(context.current_user, :read_environment, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :environments) + end + + ::Sidebars::MenuItem.new( + title: _('Environments'), + link: project_environments_path(context.project), + active_routes: { controller: :environments }, + container_html_options: { class: 'shortcuts-environments' }, + item_id: :environments + ) + end + + def feature_flags_menu_item + if Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) || + !can?(context.current_user, :read_feature_flag, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :feature_flags) + end + + ::Sidebars::MenuItem.new( + title: _('Feature Flags'), + link: project_feature_flags_path(context.project), + active_routes: { controller: :feature_flags }, + container_html_options: { class: 'shortcuts-feature-flags' }, + item_id: :feature_flags + ) + end + + def product_analytics_menu_item + if Feature.disabled?(:product_analytics, context.project) || + !can?(context.current_user, :read_product_analytics, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :product_analytics) + end + + ::Sidebars::MenuItem.new( + title: _('Product Analytics'), + link: project_product_analytics_path(context.project), + active_routes: { controller: :product_analytics }, + item_id: :product_analytics + ) + end + end + end + end +end + +Sidebars::Projects::Menus::MonitorMenu.prepend_mod_with('Sidebars::Projects::Menus::MonitorMenu') diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb new file mode 100644 index 00000000000..7087916bb04 --- /dev/null +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class PackagesRegistriesMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(packages_registry_menu_item) + add_item(container_registry_menu_item) + add_item(infrastructure_registry_menu_item) + + true + end + + override :link + def link + renderable_items.first.link + end + + override :title + def title + _('Packages & Registries') + end + + override :sprite_icon + def sprite_icon + 'package' + end + + private + + def packages_registry_menu_item + if !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :packages_registry) + end + + ::Sidebars::MenuItem.new( + title: _('Package Registry'), + link: project_packages_path(context.project), + active_routes: { controller: :packages }, + item_id: :packages_registry, + container_html_options: { class: 'shortcuts-container-registry' } + ) + end + + def container_registry_menu_item + if !::Gitlab.config.registry.enabled || !can?(context.current_user, :read_container_image, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :container_registry) + end + + ::Sidebars::MenuItem.new( + title: _('Container Registry'), + link: project_container_registry_index_path(context.project), + active_routes: { controller: :repositories }, + item_id: :container_registry + ) + end + + def infrastructure_registry_menu_item + if Feature.disabled?(:infrastructure_registry_page, context.current_user) + return ::Sidebars::NilMenuItem.new(item_id: :infrastructure_registry) + end + + ::Sidebars::MenuItem.new( + title: _('Infrastructure Registry'), + link: project_infrastructure_registry_index_path(context.project), + active_routes: { controller: :infrastructure_registry }, + item_id: :infrastructure_registry + ) + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb new file mode 100644 index 00000000000..cbb34714087 --- /dev/null +++ b/lib/sidebars/projects/menus/project_information_menu.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class ProjectInformationMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(details_menu_item) + add_item(activity_menu_item) + add_item(releases_menu_item) + add_item(labels_menu_item) + add_item(members_menu_item) + + true + end + + override :link + def link + project_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-project rspec-project-link' + } + end + + override :nav_link_html_options + def nav_link_html_options + { class: 'home' } + end + + override :title + def title + if Feature.enabled?(:sidebar_refactor, context.current_user) + _('Project information') + else + _('Project overview') + end + end + + override :sprite_icon + def sprite_icon + if Feature.enabled?(:sidebar_refactor, context.current_user) + 'project' + else + 'home' + end + end + + override :active_routes + def active_routes + return {} if Feature.disabled?(:sidebar_refactor, context.current_user) + + { path: 'projects#show' } + end + + private + + def details_menu_item + return if Feature.enabled?(:sidebar_refactor, context.current_user) + + ::Sidebars::MenuItem.new( + title: _('Details'), + link: project_path(context.project), + active_routes: { path: 'projects#show' }, + item_id: :project_overview, + container_html_options: { + aria: { label: _('Project details') }, + class: 'shortcuts-project' + } + ) + end + + def activity_menu_item + ::Sidebars::MenuItem.new( + title: _('Activity'), + link: activity_project_path(context.project), + active_routes: { path: 'projects#activity' }, + item_id: :activity, + container_html_options: { class: 'shortcuts-project-activity' } + ) + end + + def releases_menu_item + return ::Sidebars::NilMenuItem.new(item_id: :releases) unless show_releases? + + ::Sidebars::MenuItem.new( + title: _('Releases'), + link: project_releases_path(context.project), + item_id: :releases, + active_routes: { controller: :releases }, + container_html_options: { class: 'shortcuts-project-releases' } + ) + end + + def show_releases? + Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) && + can?(context.current_user, :read_release, context.project) && + !context.project.empty_repo? + end + + def labels_menu_item + if Feature.disabled?(:sidebar_refactor, context.current_user) + return ::Sidebars::NilMenuItem.new(item_id: :labels) + end + + ::Sidebars::MenuItem.new( + title: _('Labels'), + link: project_labels_path(context.project), + active_routes: { controller: :labels }, + item_id: :labels + ) + end + + def members_menu_item + if Feature.disabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) + return ::Sidebars::NilMenuItem.new(item_id: :members) + end + + ::Sidebars::MenuItem.new( + title: _('Members'), + link: project_project_members_path(context.project), + active_routes: { controller: :project_members }, + item_id: :members, + container_html_options: { + id: 'js-onboarding-members-link' + } + ) + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb new file mode 100644 index 00000000000..a784aecc3dc --- /dev/null +++ b/lib/sidebars/projects/menus/repository_menu.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class RepositoryMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return false unless can?(context.current_user, :download_code, context.project) + return false if context.project.empty_repo? + + add_item(files_menu_item) + add_item(commits_menu_item) + add_item(branches_menu_item) + add_item(tags_menu_item) + add_item(contributors_menu_item) + add_item(graphs_menu_item) + add_item(compare_menu_item) + + true + end + + override :link + def link + project_tree_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-tree' + } + end + + override :title + def title + _('Repository') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-repo-link' + } + end + + override :sprite_icon + def sprite_icon + 'doc-text' + end + + private + + def files_menu_item + ::Sidebars::MenuItem.new( + title: _('Files'), + link: project_tree_path(context.project, context.current_ref), + active_routes: { controller: %w[tree blob blame edit_tree new_tree find_file] }, + item_id: :files + ) + end + + def commits_menu_item + ::Sidebars::MenuItem.new( + title: _('Commits'), + link: project_commits_path(context.project, context.current_ref), + active_routes: { controller: %w(commit commits) }, + item_id: :commits, + container_html_options: { id: 'js-onboarding-commits-link' } + ) + end + + def branches_menu_item + ::Sidebars::MenuItem.new( + title: _('Branches'), + link: project_branches_path(context.project), + active_routes: { controller: :branches }, + item_id: :branches, + container_html_options: { id: 'js-onboarding-branches-link' } + ) + end + + def tags_menu_item + ::Sidebars::MenuItem.new( + title: _('Tags'), + link: project_tags_path(context.project), + item_id: :tags, + active_routes: { controller: :tags } + ) + end + + def contributors_menu_item + ::Sidebars::MenuItem.new( + title: _('Contributors'), + link: project_graph_path(context.project, context.current_ref), + active_routes: { path: 'graphs#show' }, + item_id: :contributors + ) + end + + def graphs_menu_item + ::Sidebars::MenuItem.new( + title: _('Graph'), + link: project_network_path(context.project, context.current_ref), + active_routes: { controller: :network }, + item_id: :graphs + ) + end + + def compare_menu_item + ::Sidebars::MenuItem.new( + title: _('Compare'), + link: project_compare_index_path(context.project, from: context.project.repository.root_ref, to: context.current_ref), + active_routes: { controller: :compare }, + item_id: :compare + ) + end + end + end + end +end + +Sidebars::Projects::Menus::RepositoryMenu.prepend_mod_with('Sidebars::Projects::Menus::RepositoryMenu') diff --git a/lib/sidebars/projects/menus/scope_menu.rb b/lib/sidebars/projects/menus/scope_menu.rb new file mode 100644 index 00000000000..1d1cf11b271 --- /dev/null +++ b/lib/sidebars/projects/menus/scope_menu.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class ScopeMenu < ::Sidebars::Menu + override :link + def link + project_path(context.project) + end + + override :title + def title + context.project.name + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/security_compliance_menu.rb b/lib/sidebars/projects/menus/security_compliance_menu.rb new file mode 100644 index 00000000000..6c9fb8312bd --- /dev/null +++ b/lib/sidebars/projects/menus/security_compliance_menu.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class SecurityComplianceMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :configure_menu_items + def configure_menu_items + return false unless can?(context.current_user, :access_security_and_compliance, context.project) + + add_item(configuration_menu_item) + + true + end + + override :link + def link + project_security_configuration_path(context.project) + end + + override :title + def title + _('Security & Compliance') + end + + override :sprite_icon + def sprite_icon + 'shield' + end + + private + + def configuration_menu_item + strong_memoize(:configuration_menu_item) do + unless render_configuration_menu_item? + next ::Sidebars::NilMenuItem.new(item_id: :configuration) + end + + ::Sidebars::MenuItem.new( + title: _('Configuration'), + link: project_security_configuration_path(context.project), + active_routes: { path: configuration_menu_item_paths }, + item_id: :configuration + ) + end + end + + def render_configuration_menu_item? + can?(context.current_user, :read_security_configuration, context.project) + end + + def configuration_menu_item_paths + %w[ + projects/security/configuration#show + ] + end + end + end + end +end + +Sidebars::Projects::Menus::SecurityComplianceMenu.prepend_mod_with('Sidebars::Projects::Menus::SecurityComplianceMenu') diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb new file mode 100644 index 00000000000..4ea6f5e298a --- /dev/null +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -0,0 +1,154 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class SettingsMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return false unless can?(context.current_user, :admin_project, context.project) + + add_item(general_menu_item) + add_item(integrations_menu_item) + add_item(webhooks_menu_item) + add_item(access_tokens_menu_item) + add_item(repository_menu_item) + add_item(ci_cd_menu_item) + add_item(monitor_menu_item) + add_item(pages_menu_item) + add_item(packages_and_registries_menu_item) + + true + end + + override :link + def link + edit_project_path(context.project) + end + + override :title + def title + _('Settings') + end + + override :title_html_options + def title_html_options + { + id: 'js-onboarding-settings-link' + } + end + + override :sprite_icon + def sprite_icon + 'settings' + end + + private + + def general_menu_item + ::Sidebars::MenuItem.new( + title: _('General'), + link: edit_project_path(context.project), + active_routes: { path: 'projects#edit' }, + item_id: :general + ) + end + + def integrations_menu_item + ::Sidebars::MenuItem.new( + title: _('Integrations'), + link: project_settings_integrations_path(context.project), + active_routes: { path: %w[integrations#show services#edit] }, + item_id: :integrations + ) + end + + def webhooks_menu_item + ::Sidebars::MenuItem.new( + title: _('Webhooks'), + link: project_hooks_path(context.project), + active_routes: { path: %w[hooks#index hooks#edit hook_logs#show] }, + item_id: :webhooks + ) + end + + def access_tokens_menu_item + unless can?(context.current_user, :read_resource_access_tokens, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :access_tokens) + end + + ::Sidebars::MenuItem.new( + title: _('Access Tokens'), + link: project_settings_access_tokens_path(context.project), + active_routes: { path: 'access_tokens#index' }, + item_id: :access_tokens + ) + end + + def repository_menu_item + ::Sidebars::MenuItem.new( + title: _('Repository'), + link: project_settings_repository_path(context.project), + active_routes: { path: 'repository#show' }, + item_id: :repository + ) + end + + def ci_cd_menu_item + if context.project.archived? || !context.project.feature_available?(:builds, context.current_user) + return ::Sidebars::NilMenuItem.new(item_id: :ci_cd) + end + + ::Sidebars::MenuItem.new( + title: _('CI/CD'), + link: project_settings_ci_cd_path(context.project), + active_routes: { path: 'ci_cd#show' }, + item_id: :ci_cd + ) + end + + def monitor_menu_item + if context.project.archived? || !can?(context.current_user, :admin_operations, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :monitor) + end + + title = Feature.enabled?(:sidebar_refactor, context.current_user, default_enabled: :yaml) ? _('Monitor') : _('Operations') + ::Sidebars::MenuItem.new( + title: title, + link: project_settings_operations_path(context.project), + active_routes: { path: 'operations#show' }, + item_id: :monitor + ) + end + + def pages_menu_item + unless context.project.pages_available? + return ::Sidebars::NilMenuItem.new(item_id: :pages) + end + + ::Sidebars::MenuItem.new( + title: _('Pages'), + link: project_pages_path(context.project), + active_routes: { path: 'pages#show' }, + item_id: :pages + ) + end + + def packages_and_registries_menu_item + if !Gitlab.config.registry.enabled || + Feature.disabled?(:sidebar_refactor, context.current_user) || + !can?(context.current_user, :destroy_container_image, context.project) + return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) + end + + ::Sidebars::MenuItem.new( + title: _('Packages & Registries'), + link: project_settings_packages_and_registries_path(context.project), + active_routes: { path: 'packages_and_registries#index' }, + item_id: :packages_and_registries + ) + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/snippets_menu.rb b/lib/sidebars/projects/menus/snippets_menu.rb new file mode 100644 index 00000000000..060341b3c51 --- /dev/null +++ b/lib/sidebars/projects/menus/snippets_menu.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class SnippetsMenu < ::Sidebars::Menu + override :link + def link + project_snippets_path(context.project) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-snippets' + } + end + + override :title + def title + _('Snippets') + end + + override :sprite_icon + def sprite_icon + 'snippet' + end + + override :render? + def render? + can?(context.current_user, :read_snippet, context.project) + end + + override :active_routes + def active_routes + { controller: :snippets } + end + end + end + end +end diff --git a/lib/sidebars/projects/menus/wiki_menu.rb b/lib/sidebars/projects/menus/wiki_menu.rb new file mode 100644 index 00000000000..3980b193fd1 --- /dev/null +++ b/lib/sidebars/projects/menus/wiki_menu.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + module Menus + class WikiMenu < ::Sidebars::Menu + override :link + def link + wiki_path(context.project.wiki) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-wiki' + } + end + + override :title + def title + _('Wiki') + end + + override :sprite_icon + def sprite_icon + 'book' + end + + override :render? + def render? + can?(context.current_user, :read_wiki, context.project) + end + + override :active_routes + def active_routes + { controller: :wikis } + end + end + end + end +end diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb new file mode 100644 index 00000000000..ac7c043a96e --- /dev/null +++ b/lib/sidebars/projects/panel.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Sidebars + module Projects + class Panel < ::Sidebars::Panel + override :configure_menus + def configure_menus + set_scope_menu(Sidebars::Projects::Menus::ScopeMenu.new(context)) + set_hidden_menu(Sidebars::Projects::Menus::HiddenMenu.new(context)) + add_menus + end + + override :aria_label + def aria_label + _('Project navigation') + end + + private + + def add_menus + add_menu(Sidebars::Projects::Menus::ProjectInformationMenu.new(context)) + add_menu(Sidebars::Projects::Menus::LearnGitlabMenu.new(context)) + add_menu(Sidebars::Projects::Menus::RepositoryMenu.new(context)) + add_menu(Sidebars::Projects::Menus::IssuesMenu.new(context)) + add_menu(Sidebars::Projects::Menus::ExternalIssueTrackerMenu.new(context)) + add_menu(Sidebars::Projects::Menus::LabelsMenu.new(context)) + add_menu(Sidebars::Projects::Menus::MergeRequestsMenu.new(context)) + add_menu(Sidebars::Projects::Menus::CiCdMenu.new(context)) + add_menu(Sidebars::Projects::Menus::SecurityComplianceMenu.new(context)) + add_menu(Sidebars::Projects::Menus::DeploymentsMenu.new(context)) + add_menu(Sidebars::Projects::Menus::MonitorMenu.new(context)) + add_menu(Sidebars::Projects::Menus::InfrastructureMenu.new(context)) + add_menu(Sidebars::Projects::Menus::PackagesRegistriesMenu.new(context)) + add_menu(Sidebars::Projects::Menus::AnalyticsMenu.new(context)) + add_menu(confluence_or_wiki_menu) + add_menu(Sidebars::Projects::Menus::ExternalWikiMenu.new(context)) + add_menu(Sidebars::Projects::Menus::SnippetsMenu.new(context)) + add_menu(Sidebars::Projects::Menus::MembersMenu.new(context)) + add_menu(Sidebars::Projects::Menus::SettingsMenu.new(context)) + end + + def confluence_or_wiki_menu + confluence_menu = ::Sidebars::Projects::Menus::ConfluenceMenu.new(context) + + confluence_menu.render? ? confluence_menu : Sidebars::Projects::Menus::WikiMenu.new(context) + end + end + end +end + +Sidebars::Projects::Panel.prepend_mod_with('Sidebars::Projects::Panel') diff --git a/lib/system_check/incoming_email/imap_authentication_check.rb b/lib/system_check/incoming_email/imap_authentication_check.rb index 056021d460c..61719abc991 100644 --- a/lib/system_check/incoming_email/imap_authentication_check.rb +++ b/lib/system_check/incoming_email/imap_authentication_check.rb @@ -35,7 +35,7 @@ module SystemCheck imap.login(mailbox[:email], mailbox[:password]) end true - rescue => error + rescue StandardError => error @error = error false end @@ -52,7 +52,7 @@ module SystemCheck def load_config erb = ERB.new(File.read(mail_room_config_path)) erb.filename = mail_room_config_path - config_file = YAML.load(erb.result) + config_file = YAML.safe_load(erb.result) config_file[:mailboxes] end diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb index 99c93edd12d..f7d2bf86c78 100644 --- a/lib/system_check/rake_task/app_task.rb +++ b/lib/system_check/rake_task/app_task.rb @@ -40,4 +40,4 @@ module SystemCheck end end -SystemCheck::RakeTask::AppTask.prepend_if_ee('EE::SystemCheck::RakeTask::AppTask') +SystemCheck::RakeTask::AppTask.prepend_mod_with('SystemCheck::RakeTask::AppTask') diff --git a/lib/system_check/rake_task/gitlab_task.rb b/lib/system_check/rake_task/gitlab_task.rb index ae2a97c98e9..233acfefb4e 100644 --- a/lib/system_check/rake_task/gitlab_task.rb +++ b/lib/system_check/rake_task/gitlab_task.rb @@ -32,4 +32,4 @@ module SystemCheck end end -SystemCheck::RakeTask::GitlabTask.prepend_if_ee('EE::SystemCheck::RakeTask::GitlabTask') +SystemCheck::RakeTask::GitlabTask.prepend_mod_with('SystemCheck::RakeTask::GitlabTask') diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 94867e1a16a..4c312ea492b 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -8,30 +8,24 @@ namespace :gitlab do namespace :artifacts do task migrate: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts to remote storage') - helper = Gitlab::Artifacts::MigrationHelper.new + helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger) begin - helper.migrate_to_remote_storage do |artifact| - logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage") - end - rescue => e + helper.migrate_to_remote_storage + rescue StandardError => e logger.error(e.message) end end task migrate_to_local: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts to local storage') - helper = Gitlab::Artifacts::MigrationHelper.new + helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger) begin - helper.migrate_to_local_storage do |artifact| - logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage") - end - rescue => e + helper.migrate_to_local_storage + rescue StandardError => e logger.error(e.message) end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 3baf4e7b7c6..bbfdf598e42 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -191,7 +191,7 @@ namespace :gitlab do ActiveRecord::Base.logger = Logger.new(STDOUT) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) Gitlab::Database::Reindexing.perform(indexes) - rescue => e + rescue StandardError => e Gitlab::AppLogger.error(e) raise end @@ -217,9 +217,11 @@ namespace :gitlab do end desc 'Run migrations with instrumentation' - task :migration_testing, [:result_file] => :environment do |_, args| - result_file = args[:result_file] || raise("Please specify result_file argument") - raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file) + task migration_testing: :environment do + result_dir = Gitlab::Database::Migrations::Instrumentation::RESULT_DIR + raise "Directory exists already, won't overwrite: #{result_dir}" if File.exist?(result_dir) + + Dir.mkdir(result_dir) verbose_was = ActiveRecord::Migration.verbose ActiveRecord::Migration.verbose = true @@ -240,7 +242,7 @@ namespace :gitlab do end ensure if instrumentation - File.open(result_file, 'wb+') do |io| + File.open(File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME), 'wb+') do |io| io << instrumentation.observations.to_json end end @@ -248,5 +250,19 @@ namespace :gitlab do ActiveRecord::Base.clear_cache! ActiveRecord::Migration.verbose = verbose_was end + + desc 'Run all pending batched migrations' + task execute_batched_migrations: :environment do + Gitlab::Database::BackgroundMigration::BatchedMigration.active.queue_order.each do |migration| + Gitlab::AppLogger.info("Executing batched migration #{migration.id} inline") + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(migration) + end + end + + # Only for development environments, + # we execute pending data migrations inline for convenience. + Rake::Task['db:migrate'].enhance do + Rake::Task['gitlab:db:execute_batched_migrations'].invoke if Rails.env.development? + end end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 9c5549b4a54..8587ef4afdb 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -9,7 +9,7 @@ namespace :gitlab do begin project.repository.fsck - rescue => e + rescue StandardError => e failures << "#{project.full_path} on #{project.repository_storage}: #{e}" end @@ -51,7 +51,7 @@ namespace :gitlab do next unless project.repo_exists? result = project.repository.checksum - rescue => e + rescue StandardError => e result = "Ignored error: #{e.message}".squish.truncate(255) ensure puts "#{project.id},#{result}" diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 05249a126bc..a173de7c5c7 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -14,7 +14,7 @@ namespace :gitlab do lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") - rescue => e + rescue StandardError => e logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end @@ -28,7 +28,7 @@ namespace :gitlab do lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") - rescue => e + rescue StandardError => e logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index 4a6a014acc5..d24535d85b6 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -9,7 +9,7 @@ namespace :gitlab do task generate: :environment do Rake::Task["gitlab:packages:events:generate_counts"].invoke Rake::Task["gitlab:packages:events:generate_unique"].invoke - rescue => e + rescue StandardError => e logger.error("Error building events list: #{e}") end @@ -21,7 +21,7 @@ namespace :gitlab do File.open(path, "w") { |file| file << counter_events_list.to_yaml } logger.info("Events file `#{path}` generated successfully") - rescue => e + rescue StandardError => e logger.error("Error building events list: #{e}") end @@ -33,7 +33,7 @@ namespace :gitlab do File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml } logger.info("Events file `#{path}` generated successfully") - rescue => e + rescue StandardError => e logger.error("Error building events list: #{e}") end diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake index 20a8c51db66..febc3e7fa2d 100644 --- a/lib/tasks/gitlab/packages/migrate.rake +++ b/lib/tasks/gitlab/packages/migrate.rake @@ -17,7 +17,7 @@ namespace :gitlab do package_file.file.migrate!(::Packages::PackageFileUploader::Store::REMOTE) logger.info("Transferred package file #{package_file.id} of size #{package_file.size.to_i.bytes} to object storage") - rescue => e + rescue StandardError => e logger.error("Failed to transfer package file #{package_file.id} with error: #{e.message}") end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index ee2931f0c4f..684d62d1367 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -61,26 +61,24 @@ namespace :gitlab do namespace :deployments do task migrate_to_object_storage: :gitlab_environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of pages deployments to remote storage') - helper = Gitlab::Pages::MigrationHelper.new(logger) + helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger) begin helper.migrate_to_remote_storage - rescue => e + rescue StandardError => e logger.error(e.message) end end task migrate_to_local: :gitlab_environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of Pages deployments to local storage') - helper = Gitlab::Pages::MigrationHelper.new(logger) + helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger) begin helper.migrate_to_local_storage - rescue => e + rescue StandardError => e logger.error(e.message) end end diff --git a/lib/tasks/gitlab/praefect.rake b/lib/tasks/gitlab/praefect.rake index 346df3e0c75..28b70f8986e 100644 --- a/lib/tasks/gitlab/praefect.rake +++ b/lib/tasks/gitlab/praefect.rake @@ -44,7 +44,7 @@ namespace :gitlab do row = [project.name] << replicas_resp.primary.checksum row.concat(sorted_replicas.map {|r| r.checksum}) - rescue + rescue StandardError puts 'Something went wrong when getting replicas.' next end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 6f00db42d78..6f5c3a86dd3 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -8,6 +8,29 @@ namespace :gitlab do File.write(path, banner + YAML.dump(object).gsub(/ *$/m, '')) end + namespace :migrate_jobs do + def mappings + ::Gitlab::SidekiqConfig + .workers + .reject { |worker| worker.klass.is_a?(Gitlab::SidekiqConfig::DummyWorker) } + .to_h { |worker| [worker.klass.to_s, ::Gitlab::SidekiqConfig::WorkerRouter.global.route(worker.klass)] } + end + + desc 'GitLab | Sidekiq | Migrate jobs in the scheduled set to new queue names' + task schedule: :environment do + ::Gitlab::SidekiqMigrateJobs + .new('schedule', logger: Logger.new($stdout)) + .execute(mappings) + end + + desc 'GitLab | Sidekiq | Migrate jobs in the retry set to new queue names' + task retry: :environment do + ::Gitlab::SidekiqMigrateJobs + .new('retry', logger: Logger.new($stdout)) + .execute(mappings) + end + end + namespace :all_queues_yml do desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions' task generate: :environment do diff --git a/lib/tasks/gitlab/snowplow.rake b/lib/tasks/gitlab/snowplow.rake new file mode 100644 index 00000000000..278ba4a471c --- /dev/null +++ b/lib/tasks/gitlab/snowplow.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :snowplow do + desc 'GitLab | Snowplow | Generate event dictionary' + task generate_event_dictionary: :environment do + items = Gitlab::Tracking::EventDefinition.definitions + Gitlab::Tracking::Docs::Renderer.new(items).write + end + end +end diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake index a9c16049240..2bf9ec9537a 100644 --- a/lib/tasks/gitlab/terraform/migrate.rake +++ b/lib/tasks/gitlab/terraform/migrate.rake @@ -15,7 +15,7 @@ namespace :gitlab do logger.info(message) end - rescue => e + rescue StandardError => e logger.error("Failed to migrate: #{e.message}") end end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 95072444fcf..0ad50c0fa53 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -29,5 +29,10 @@ namespace :gitlab do items = Gitlab::Usage::MetricDefinition.definitions Gitlab::Usage::Docs::Renderer.new(items).write end + + desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON' + task generate_from_yaml: :environment do + puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data) + end end end diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index e0666a87656..f5db94d5336 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -11,7 +11,7 @@ task migrate_iids: :environment do else print 'F' end - rescue + rescue StandardError print 'F' end @@ -25,7 +25,7 @@ task migrate_iids: :environment do else print 'F' end - rescue + rescue StandardError print 'F' end @@ -39,7 +39,7 @@ task migrate_iids: :environment do else print 'F' end - rescue + rescue StandardError print 'F' end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index b0ff2cce482..74baa11c314 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -1,9 +1,9 @@ # frozen_string_literal: true -require_relative '../../app/models/concerns/token_authenticatable.rb' -require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb' -require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb' -require_relative '../../app/models/concerns/token_authenticatable_strategies/digest.rb' +require_relative '../../app/models/concerns/token_authenticatable' +require_relative '../../app/models/concerns/token_authenticatable_strategies/base' +require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure' +require_relative '../../app/models/concerns/token_authenticatable_strategies/digest' namespace :tokens do desc "Reset all GitLab incoming email tokens" diff --git a/lib/version_check.rb b/lib/version_check.rb index c9f102f6b19..a8b7c7371ca 100644 --- a/lib/version_check.rb +++ b/lib/version_check.rb @@ -12,10 +12,12 @@ class VersionCheck def self.url encoded_data = Base64.urlsafe_encode64(data.to_json) - "#{host}?gitlab_info=#{encoded_data}" + "#{host}/check.svg?gitlab_info=#{encoded_data}" end def self.host - 'https://version.gitlab.com/check.svg' + 'https://version.gitlab.com' end end + +VersionCheck.prepend_mod |