diff options
| author | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 13:34:23 -0600 |
|---|---|---|
| committer | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 13:34:23 -0600 |
| commit | 6438df3a1e0fb944485cebf07976160184697d72 (patch) | |
| tree | 00b09bfd170e77ae9391b1a2f5a93ef6839f2597 /lib | |
| parent | 42bcd54d971da7ef2854b896a7b34f4ef8601067 (diff) | |
| download | gitlab-ce-6438df3a1e0fb944485cebf07976160184697d72.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42
Diffstat (limited to 'lib')
260 files changed, 4804 insertions, 2070 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 06c2b46a2f2..ada0da28749 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -59,6 +59,7 @@ module API project: -> { @project }, namespace: -> { @group }, caller_id: route.origin, + remote_ip: request.ip, feature_category: feature_category ) end @@ -212,6 +213,7 @@ module API mount ::API::GroupPackages mount ::API::PackageFiles mount ::API::NugetProjectPackages + mount ::API::NugetGroupPackages mount ::API::PypiPackages mount ::API::ComposerPackages mount ::API::ConanProjectPackages @@ -251,6 +253,7 @@ module API mount ::API::Services mount ::API::Settings mount ::API::SidekiqMetrics + mount ::API::SnippetRepositoryStorageMoves mount ::API::Snippets mount ::API::Statistics mount ::API::Submodules diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 0a486307653..8641271f2df 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -69,10 +69,15 @@ module API def find_user_from_sources strong_memoize(:find_user_from_sources) do - deploy_token_from_request || - find_user_from_bearer_token || - find_user_from_job_token || - user_from_warden + if try(:namespace_inheritable, :authentication) + user_from_namespace_inheritable || + user_from_warden + else + deploy_token_from_request || + find_user_from_bearer_token || + find_user_from_job_token || + user_from_warden + end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index e2d30dd7c2b..5fd4ca3546c 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -7,10 +7,10 @@ module API prepend_if_ee('EE::API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule - before { authenticate! } - feature_category :boards + before { authenticate! } + helpers do def board_parent user_project diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 89355c84401..5a30de1f766 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -80,10 +80,20 @@ module API requires :label_id, type: Integer, desc: 'The ID of an existing label' end - params :update_params do + params :update_params_ce do + optional :name, type: String, desc: 'The board name' + optional :hide_backlog_list, type: Grape::API::Boolean, desc: 'Hide the Open list' + optional :hide_closed_list, type: Grape::API::Boolean, desc: 'Hide the Closed list' + end + + params :update_params_ee do # Configurable issue boards are not available in CE/EE Core. # https://docs.gitlab.com/ee/user/project/issue_board.html#configurable-issue-boards - optional :name, type: String, desc: 'The board name' + end + + params :update_params do + use :update_params_ce + use :update_params_ee end end end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 86e1a939df1..5cfb65e1fbb 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -180,6 +180,7 @@ module API optional :checksum, type: String, desc: %q(Job's trace CRC32 checksum) optional :bytesize, type: Integer, desc: %q(Job's trace size in bytes) end + optional :exit_code, type: Integer, desc: %q(Job's exit code) end put '/:id' do job = authenticate_job! diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index 1a03a6a6dad..53b778875fc 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -19,44 +19,49 @@ module API included do helpers do - def find_packages - packages = package_finder.execute + def find_packages(package_name) + packages = package_finder(package_name).execute not_found!('Packages') unless packages.exists? packages end - def find_package - package = package_finder(package_version: params[:package_version]).execute - .first + def find_package(package_name, package_version) + package = package_finder(package_name, package_version).execute + .first not_found!('Package') unless package package end - def package_finder(finder_params = {}) + def package_finder(package_name, package_version = nil) ::Packages::Nuget::PackageFinder.new( - authorized_user_project, - **finder_params.merge(package_name: params[:package_name]) + current_user, + project_or_group, + package_name: package_name, + package_version: package_version ) end + + def search_packages(search_term, search_options) + ::Packages::Nuget::SearchService + .new(current_user, project_or_group, params[:q], search_options) + .execute + end end # https://docs.microsoft.com/en-us/nuget/api/service-index desc 'The NuGet Service Index' do detail 'This feature was introduced in GitLab 12.6' end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - get 'index', format: :json do - authorize_read_package!(authorized_user_project) + authorize_read_package!(project_or_group) track_package_event('cli_metadata', :nuget, category: 'API::NugetPackages') - present ::Packages::Nuget::ServiceIndexPresenter.new(authorized_user_project), - with: ::API::Entities::Nuget::ServiceIndex + present ::Packages::Nuget::ServiceIndexPresenter.new(project_or_group), + with: ::API::Entities::Nuget::ServiceIndex end # https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource @@ -64,18 +69,15 @@ module API requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX end namespace '/metadata/*package_name' do - before do - authorize_read_package!(authorized_user_project) + after_validation do + authorize_read_package!(project_or_group) end desc 'The NuGet Metadata Service - Package name level' do detail 'This feature was introduced in GitLab 12.8' end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - get 'index', format: :json do - present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages), + present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages(params[:package_name])), with: ::API::Entities::Nuget::PackagesMetadata end @@ -85,11 +87,8 @@ module API params do requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - get '*package_version', format: :json do - present ::Packages::Nuget::PackageMetadataPresenter.new(find_package), + present ::Packages::Nuget::PackageMetadataPresenter.new(find_package(params[:package_name], params[:package_version])), with: ::API::Entities::Nuget::PackageMetadata end end @@ -102,30 +101,26 @@ module API optional :prerelease, type: ::Grape::API::Boolean, desc: 'Include prerelease versions', default: true end namespace '/query' do - before do - authorize_read_package!(authorized_user_project) + after_validation do + authorize_read_package!(project_or_group) end desc 'The NuGet Search Service' do detail 'This feature was introduced in GitLab 12.8' end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - get format: :json do search_options = { include_prerelease_versions: params[:prerelease], per_page: params[:take], padding: params[:skip] } - search = ::Packages::Nuget::SearchService - .new(authorized_user_project, params[:q], search_options) - .execute + + results = search_packages(params[:q], search_options) track_package_event('search_package', :nuget, category: 'API::NugetPackages') - present ::Packages::Nuget::SearchResultsPresenter.new(search), - with: ::API::Entities::Nuget::SearchResults + present ::Packages::Nuget::SearchResultsPresenter.new(results), + with: ::API::Entities::Nuget::SearchResults end end end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index bcb4e8c8cbc..f8129c18dff 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -21,6 +21,8 @@ module API end namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do + content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' @@ -42,10 +44,9 @@ module API # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth - post 'authorize' do + put 'authorize' do authorize_workhorse!( subject: authorized_user_project, - has_length: false, maximum_size: authorized_user_project.actual_limits.debian_max_file_size ) end diff --git a/lib/api/entities/basic_repository_storage_move.rb b/lib/api/entities/basic_repository_storage_move.rb new file mode 100644 index 00000000000..3ee112fb9a2 --- /dev/null +++ b/lib/api/entities/basic_repository_storage_move.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + class BasicRepositoryStorageMove < Grape::Entity + expose :id + expose :created_at + expose :human_state_name, as: :state + expose :source_storage_name + expose :destination_storage_name + end + end +end diff --git a/lib/api/entities/basic_snippet.rb b/lib/api/entities/basic_snippet.rb new file mode 100644 index 00000000000..26297514798 --- /dev/null +++ b/lib/api/entities/basic_snippet.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module API + module Entities + class BasicSnippet < Grape::Entity + expose :id, :title, :description, :visibility + expose :updated_at, :created_at + expose :project_id + expose :web_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + end + expose :raw_url do |snippet| + Gitlab::UrlBuilder.build(snippet, raw: true) + end + expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.repository_exists? } + end + end +end diff --git a/lib/api/entities/board.rb b/lib/api/entities/board.rb index b7a50408313..fe0182ad772 100644 --- a/lib/api/entities/board.rb +++ b/lib/api/entities/board.rb @@ -5,6 +5,8 @@ module API class Board < Grape::Entity expose :id expose :name + expose :hide_backlog_list + expose :hide_closed_list expose :project, using: Entities::BasicProjectDetails expose :lists, using: Entities::List do |board| diff --git a/lib/api/entities/note.rb b/lib/api/entities/note.rb index 9a60c04220d..a597aa7bb4a 100644 --- a/lib/api/entities/note.rb +++ b/lib/api/entities/note.rb @@ -23,6 +23,7 @@ module API expose :resolvable?, as: :resolvable expose :resolved?, as: :resolved, if: ->(note, options) { note.resolvable? } expose :resolved_by, using: Entities::UserBasic, if: ->(note, options) { note.resolvable? } + expose :resolved_at, if: ->(note, options) { note.resolvable? } expose :confidential?, as: :confidential diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 317caefe0a1..6ad6123a20e 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -100,6 +100,7 @@ module API end expose :only_allow_merge_if_pipeline_succeeds expose :allow_merge_on_skipped_pipeline + expose :restrict_user_defined_variables expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved expose :remove_source_branch_after_merge diff --git a/lib/api/entities/project_repository_storage_move.rb b/lib/api/entities/project_repository_storage_move.rb index 25643651a14..191bbaf19d7 100644 --- a/lib/api/entities/project_repository_storage_move.rb +++ b/lib/api/entities/project_repository_storage_move.rb @@ -2,12 +2,7 @@ module API module Entities - class ProjectRepositoryStorageMove < Grape::Entity - expose :id - expose :created_at - expose :human_state_name, as: :state - expose :source_storage_name - expose :destination_storage_name + class ProjectRepositoryStorageMove < BasicRepositoryStorageMove expose :project, using: Entities::ProjectIdentity end end diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index 44a46c5861e..f6c3dd5a509 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -16,7 +16,12 @@ module API expose :author, using: Entities::UserBasic, if: -> (release, _) { release.author.present? } expose :commit, using: Entities::Commit, if: ->(_, _) { can_download_code? } expose :upcoming_release?, as: :upcoming_release - expose :milestones, using: Entities::MilestoneWithStats, if: -> (release, _) { release.milestones.present? && can_read_milestone? } + expose :milestones, + using: Entities::MilestoneWithStats, + if: -> (release, _) { release.milestones.present? && can_read_milestone? } do |release, _| + release.milestones.order_by_dates_and_title + end + expose :commit_path, expose_nil: false expose :tag_path, expose_nil: false diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb index 85148c03d18..f05e593a302 100644 --- a/lib/api/entities/snippet.rb +++ b/lib/api/entities/snippet.rb @@ -2,18 +2,8 @@ module API module Entities - class Snippet < Grape::Entity - expose :id, :title, :description, :visibility + class Snippet < BasicSnippet expose :author, using: Entities::UserBasic - expose :updated_at, :created_at - expose :project_id - expose :web_url do |snippet| - Gitlab::UrlBuilder.build(snippet) - end - expose :raw_url do |snippet| - Gitlab::UrlBuilder.build(snippet, raw: true) - end - expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.repository_exists? } expose :file_name do |snippet| snippet.file_name_on_repo || snippet.file_name end diff --git a/lib/api/entities/snippet_repository_storage_move.rb b/lib/api/entities/snippet_repository_storage_move.rb new file mode 100644 index 00000000000..ee86816bd14 --- /dev/null +++ b/lib/api/entities/snippet_repository_storage_move.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class SnippetRepositoryStorageMove < BasicRepositoryStorageMove + expose :snippet, using: Entities::BasicSnippet + end + end +end diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 3e1dd044c8d..167531fdaec 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -21,7 +21,7 @@ module API end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - route_setting :authentication, job_token_allowed: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true namespace ':id/packages/generic' do namespace ':package_name/*package_version/:file_name', requirements: GENERIC_PACKAGES_REQUIREMENTS do @@ -29,7 +29,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, job_token_allowed: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true params do requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true @@ -52,7 +52,7 @@ module API requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' end - route_setting :authentication, job_token_allowed: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true put do authorize_upload!(project) @@ -82,7 +82,7 @@ module API requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true end - route_setting :authentication, job_token_allowed: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true get do authorize_read_package!(project) diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 2bfd98a5b69..7425e1bd145 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -9,9 +9,7 @@ module API feature_category :boards - before do - authenticate! - end + before { authenticate! } helpers do def board_parent @@ -22,28 +20,40 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do segment ':id/boards' do + desc 'Get all group boards' do + detail 'This feature was introduced in 10.6' + success Entities::Board + end + params do + use :pagination + end + get '/' do + authorize!(:read_board, user_group) + present paginate(board_parent.boards.with_associations), with: Entities::Board + end + desc 'Find a group board' do detail 'This feature was introduced in 10.6' - success ::API::Entities::Board + success Entities::Board end get '/:board_id' do authorize!(:read_board, user_group) - present board, with: ::API::Entities::Board + present board, with: Entities::Board end - desc 'Get all group boards' do - detail 'This feature was introduced in 10.6' + desc 'Update a group board' do + detail 'This feature was introduced in 11.0' success Entities::Board end params do - use :pagination + use :update_params end - get '/' do - authorize!(:read_board, user_group) - present paginate(board_parent.boards.with_associations), with: Entities::Board + put '/:board_id' do + authorize!(:admin_board, board_parent) + + update_board end end diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index 31b28c3990f..d482f4d0585 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -31,12 +31,14 @@ module API desc: 'Return packages of a certain type' optional :package_name, type: String, desc: 'Return packages with this name' + optional :include_versionless, type: Boolean, + desc: 'Returns packages without a version' end get ':id/packages' do packages = Packages::GroupPackagesFinder.new( current_user, user_group, - declared(params).slice(:exclude_subgroups, :order_by, :sort, :package_type, :package_name) + declared(params).slice(:exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless) ).execute present paginate(packages), with: ::API::Entities::Package, user: current_user, group: true diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 6fe25471289..79af9c37378 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -220,6 +220,10 @@ module API user_project.builds.find(id.to_i) end + def find_job!(id) + user_project.processables.find(id.to_i) + end + def authenticate! unauthorized! unless current_user end @@ -275,6 +279,10 @@ module API authorize! :read_build_trace, build end + def authorize_read_job_artifacts!(build) + authorize! :read_job_artifacts, build + end + def authorize_destroy_artifacts! authorize! :destroy_artifacts, user_project end @@ -364,7 +372,7 @@ module API def forbidden!(reason = nil) message = ['403 Forbidden'] - message << " - #{reason}" if reason + message << "- #{reason}" if reason render_api_error!(message.join(' '), 403) end @@ -513,7 +521,7 @@ module API case headers['X-Sendfile-Type'] when 'X-Sendfile' header['X-Sendfile'] = path - body + body '' # to avoid an error from API::APIGuard::ResponseCoercerMiddleware else sendfile path end @@ -529,7 +537,7 @@ module API else header(*Gitlab::Workhorse.send_url(file.url)) status :ok - body "" + body '' # to avoid an error from API::APIGuard::ResponseCoercerMiddleware end end @@ -562,7 +570,7 @@ module API return unless Feature.enabled?(feature_flag, default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: values) rescue => error Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb new file mode 100644 index 00000000000..a6cfe930190 --- /dev/null +++ b/lib/api/helpers/authentication.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module API + module Helpers + module Authentication + extend ActiveSupport::Concern + + class_methods do + def authenticate_with(&block) + strategies = ::Gitlab::APIAuthentication::Builder.new.build(&block) + namespace_inheritable :authentication, strategies + end + end + + included do + helpers ::Gitlab::Utils::StrongMemoize + + helpers do + def token_from_namespace_inheritable + strong_memoize(:token_from_namespace_inheritable) do + strategies = namespace_inheritable(:authentication) + next unless strategies&.any? + + # Extract credentials from the request + found = strategies.to_h { |location, _| [location, ::Gitlab::APIAuthentication::TokenLocator.new(location).extract(current_request)] } + found.filter! { |location, raw| raw } + next unless found.any? + + # Specifying multiple credentials is an error + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_475984136 + bad_request!('Found more than one set of credentials') if found.size > 1 + + location, raw = found.first + find_token_from_raw_credentials(strategies[location], raw) + end + + rescue ::Gitlab::Auth::UnauthorizedError + # TODO: this should be rescued and converted by the exception handling middleware + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_475174516 + unauthorized! + end + + def access_token_from_namespace_inheritable + token = token_from_namespace_inheritable + token if token.is_a? PersonalAccessToken + end + + def user_from_namespace_inheritable + token = token_from_namespace_inheritable + return token if token.is_a? DeployToken + + token&.user + end + + private + + def find_token_from_raw_credentials(token_types, raw) + token_types.each do |token_type| + # Resolve a token from the raw credentials + token = ::Gitlab::APIAuthentication::TokenResolver.new(token_type).resolve(raw) + return token if token + end + + # If a request provides credentials via an allowed transport, the + # credentials must be valid. If we reach this point, the credentials + # must not be valid credentials of an allowed type. + raise ::Gitlab::Auth::UnauthorizedError + end + end + end + end + end +end diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb index 9b38eeb1e72..f8fe40f7135 100644 --- a/lib/api/helpers/merge_requests_helpers.rb +++ b/lib/api/helpers/merge_requests_helpers.rb @@ -21,6 +21,9 @@ module API coerce_with: Validations::Validators::CheckAssigneesCount.coerce, desc: 'Return merge requests which are assigned to the user with the given username' mutually_exclusive :assignee_id, :assignee_username + optional :reviewer_username, + type: String, + desc: 'Return merge requests which have the user as a reviewer with the given username' optional :labels, type: Array[String], @@ -32,6 +35,11 @@ module API params :merge_requests_base_params do use :merge_requests_negatable_params + optional :reviewer_id, + types: [Integer, String], + integer_none_any: true, + desc: 'Return merge requests which have the user as a reviewer with the given ID' + mutually_exclusive :reviewer_id, :reviewer_username optional :state, type: String, values: %w[opened closed locked merged all], @@ -72,6 +80,10 @@ module API optional :wip, type: String, values: %w[yes no], desc: 'Search merge requests for WIP in the title' optional :not, type: Hash, desc: 'Parameters to negate' do use :merge_requests_negatable_params + optional :reviewer_id, + types: Integer, + desc: 'Return merge requests which have the user as a reviewer with the given ID' + mutually_exclusive :reviewer_id, :reviewer_username end optional :deployed_before, diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb index 0784efc11d6..c32ce199dd6 100644 --- a/lib/api/helpers/packages/basic_auth_helpers.rb +++ b/lib/api/helpers/packages/basic_auth_helpers.rb @@ -12,6 +12,7 @@ module API end include Constants + include Gitlab::Utils::StrongMemoize def unauthorized_user_project @unauthorized_user_project ||= find_project(params[:id]) @@ -35,6 +36,18 @@ module API project end + def find_authorized_group! + strong_memoize(:authorized_group) do + group = find_group(params[:id]) + + unless group && can?(current_user, :read_group, group) + next unauthorized_or! { not_found! } + end + + group + end + end + def authorize!(action, subject = :global, reason = nil) return if can?(current_user, action, subject) diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 227aec224e5..48618e7d26d 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -3,8 +3,8 @@ module API module Helpers module Pagination - def paginate(*args) - Gitlab::Pagination::OffsetPagination.new(self).paginate(*args) + def paginate(*args, **kwargs) + Gitlab::Pagination::OffsetPagination.new(self).paginate(*args, **kwargs) end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index f5f45cf7351..cf2bcace33b 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -87,6 +87,7 @@ module API params :optional_update_params_ce do optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Skip older deployment jobs that are still pending' + optional :restrict_user_defined_variables, type: Boolean, desc: 'Restrict use of user-defined variables when triggering a pipeline' end params :optional_update_params_ee do @@ -99,7 +100,7 @@ module API params :optional_container_expiration_policy_params do optional :cadence, type: String, desc: 'Container expiration policy cadence for recurring job' - optional :keep_n, type: String, desc: 'Container expiration policy number of images to keep' + optional :keep_n, type: Integer, desc: 'Container expiration policy number of images to keep' optional :older_than, type: String, desc: 'Container expiration policy remove images older than value' optional :name_regex, type: String, desc: 'Container expiration policy regex for image removal' optional :name_regex_keep, type: String, desc: 'Container expiration policy regex for image retention' @@ -141,6 +142,7 @@ module API :repository_access_level, :request_access_enabled, :resolve_outdated_diff_discussions, + :restrict_user_defined_variables, :shared_runners_enabled, :snippets_access_level, :tag_list, diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 9d2fd9978d9..6101a8d307e 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -161,7 +161,6 @@ module API def self.services { - 'alerts' => [], 'asana' => [ { required: true, @@ -807,7 +806,6 @@ module API def self.service_classes [ - ::AlertsService, ::AsanaService, ::AssemblaService, ::BambooService, diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 332f2f1986f..12bb6e77c3e 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -16,6 +16,7 @@ module API user: -> { actor&.user }, project: -> { project }, caller_id: route.origin, + remote_ip: request.ip, feature_category: feature_category ) end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index be8147908e9..2ab1f97afe6 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -48,6 +48,24 @@ module API present_member_invitations invitations end + + desc 'Removes an invitation from a group or project.' + params do + requires :email, type: String, desc: 'The email address of the invitation' + end + delete ":id/invitations/:email", requirements: { email: /[^\/]+/ } do + source = find_source(source_type, params[:id]) + invite_email = params[:email] + authorize_admin_source!(source_type, source) + + invite = retrieve_member_invitations(source, invite_email).first + not_found! unless invite + + destroy_conditionally!(invite) do + ::Members::DestroyService.new(current_user, params).execute(invite) + unprocessable_entity! unless invite.destroyed? + end + end end end end diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 1faa28d6f07..28737f61f61 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -32,6 +32,7 @@ module API authorize_download_artifacts! latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) + authorize_read_job_artifacts!(latest_build) present_carrierwave_file!(latest_build.artifacts_file) end @@ -50,6 +51,7 @@ module API authorize_download_artifacts! build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) + authorize_read_job_artifacts!(build) path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) @@ -70,6 +72,7 @@ module API authorize_download_artifacts! build = find_build!(params[:job_id]) + authorize_read_job_artifacts!(build) present_carrierwave_file!(build.artifacts_file) end @@ -82,9 +85,11 @@ module API requires :artifact_path, type: String, desc: 'Artifact path' end get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do - authorize_read_builds! + authorize_download_artifacts! build = find_build!(params[:job_id]) + authorize_read_job_artifacts!(build) + not_found! unless build.artifacts? path = Gitlab::Ci::Build::Artifacts::Path diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 44751b3d76c..e14a4a5e680 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -138,25 +138,32 @@ module API present build, with: Entities::Ci::Job end - desc 'Trigger a actionable job (manual, delayed, etc)' do - success Entities::Ci::Job + desc 'Trigger an actionable job (manual, delayed, etc)' do + success Entities::Ci::JobBasic detail 'This feature was added in GitLab 8.11' end params do requires :job_id, type: Integer, desc: 'The ID of a Job' end + post ":id/jobs/:job_id/play" do authorize_read_builds! - build = find_build!(params[:job_id]) + job = find_job!(params[:job_id]) - authorize!(:update_build, build) - bad_request!("Unplayable Job") unless build.playable? + authorize!(:play_job, job) - build.play(current_user) + bad_request!("Unplayable Job") unless job.playable? + + job.play(current_user) status 200 - present build, with: Entities::Ci::Job + + if job.is_a?(::Ci::Build) + present job, with: Entities::Ci::Job + else + present job, with: Entities::Ci::Bridge + end end end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 58181adaa93..f1f34622187 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -12,14 +12,13 @@ module API end post '/lint' do result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute - error = result.errors.first status 200 - response = if error.blank? + response = if result.errors.empty? { status: 'valid', errors: [], warnings: result.warnings } else - { status: 'invalid', errors: [error], warnings: result.warnings } + { status: 'invalid', errors: result.errors, warnings: result.warnings } end response.tap do |response| diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 7b4e52d18e8..4a5b2ead163 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -220,9 +220,13 @@ module API file_name, format = extract_format(params[:file_name]) - package = ::Packages::Maven::FindOrCreatePackageService + result = ::Packages::Maven::FindOrCreatePackageService .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute + bad_request!(result.errors.first) if result.error? + + package = result.payload[:package] + case format when 'sha1' # After uploading a file, Maven tries to upload a sha1 and md5 version of it. diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb new file mode 100644 index 00000000000..e373f051b24 --- /dev/null +++ b/lib/api/nuget_group_packages.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# NuGet Package Manager Client API +# +# These API endpoints are not meant to be consumed directly by users. They are +# called by the NuGet package manager client when users run commands +# like `nuget install` or `nuget push`. +# +# This is the group level API. +module API + class NugetGroupPackages < ::API::Base + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + include ::API::Helpers::Authentication + + feature_category :package_registry + + default_format :json + + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end + + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + after_validation do + require_packages_enabled! + end + + helpers do + def project_or_group + find_authorized_group! + end + + def require_authenticated! + unauthorized! unless current_user + end + end + + params do + requires :id, type: String, desc: 'The ID of a group', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX + end + + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/-/packages/nuget' do + after_validation do + # This API can't be accessed anonymously + require_authenticated! + end + + include ::API::Concerns::Packages::NugetEndpoints + end + end + end +end diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index b2516cc91f8..2146f4d4b78 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -5,10 +5,13 @@ # These API endpoints are not meant to be consumed directly by users. They are # called by the NuGet package manager client when users run commands # like `nuget install` or `nuget push`. +# +# This is the project level API. module API class NugetProjectPackages < ::API::Base - helpers ::API::Helpers::PackagesManagerClientsHelpers + helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers + include ::API::Helpers::Authentication feature_category :package_registry @@ -16,25 +19,29 @@ module API default_format :json + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end + rescue_from ArgumentError do |e| render_api_error!(e.message, 400) end - before do + after_validation do require_packages_enabled! end + helpers do + def project_or_group + authorized_user_project + end + end + params do requires :id, type: String, desc: 'The ID of a project', regexp: ::API::Concerns::Packages::NugetEndpoints::POSITIVE_INTEGER_REGEX end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - authorized_user_project - end - namespace ':id/packages/nuget' do include ::API::Concerns::Packages::NugetEndpoints @@ -46,28 +53,20 @@ module API params do requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - put do - authorize_upload!(authorized_user_project) - bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) + authorize_upload!(project_or_group) + bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) file_params = params.merge( file: params[:package], file_name: PACKAGE_FILENAME ) - package = ::Packages::Nuget::CreatePackageService.new( - authorized_user_project, - current_user, - declared_params.merge(build: current_authenticated_job) - ).execute + package = ::Packages::Nuget::CreatePackageService.new(project_or_group, current_user, declared_params.merge(build: current_authenticated_job)) + .execute - package_file = ::Packages::CreatePackageFileService.new( - package, - file_params.merge(build: current_authenticated_job) - ).execute + package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) + .execute track_package_event('push_package', :nuget, category: 'API::NugetPackages') @@ -75,18 +74,15 @@ module API created! rescue ObjectStorage::RemoteStoreError => e - Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project_or_group.id }) forbidden! end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - put 'authorize' do authorize_workhorse!( - subject: authorized_user_project, + subject: project_or_group, has_length: false, - maximum_size: authorized_user_project.actual_limits.nuget_max_file_size + maximum_size: project_or_group.actual_limits.nuget_max_file_size ) end @@ -95,18 +91,15 @@ module API requires :package_name, type: String, desc: 'The NuGet package name', regexp: API::NO_SLASH_URL_PART_REGEX end namespace '/download/*package_name' do - before do - authorize_read_package!(authorized_user_project) + after_validation do + authorize_read_package!(project_or_group) end desc 'The NuGet Content Service - index request' do detail 'This feature was introduced in GitLab 12.8' end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - get 'index', format: :json do - present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages), + present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages(params[:package_name])), with: ::API::Entities::Nuget::PackagesVersions end @@ -117,12 +110,9 @@ module API requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX end - - route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true - get '*package_version/*package_filename', format: :nupkg do filename = "#{params[:package_filename]}.#{params[:format]}" - package_file = ::Packages::PackageFileFinder.new(find_package, filename, with_file_name_like: true) + package_file = ::Packages::PackageFileFinder.new(find_package(params[:package_name], params[:package_version]), filename, with_file_name_like: true) .execute not_found!('Package') unless package_file diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 56e94333433..32636662987 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -30,11 +30,13 @@ module API desc: 'Return packages of a certain type' optional :package_name, type: String, desc: 'Return packages with this name' + optional :include_versionless, type: Boolean, + desc: 'Returns packages without a version' end get ':id/packages' do packages = ::Packages::PackagesFinder.new( user_project, - declared_params.slice(:order_by, :sort, :package_type, :package_name) + declared_params.slice(:order_by, :sort, :package_type, :package_name, :include_versionless) ).execute present paginate(packages), with: ::API::Entities::Package, user: current_user diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index af5d96969ef..19244ed697f 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -4,7 +4,7 @@ module API class ProjectTemplates < ::API::Base include PaginationParams - TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze + TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls gitlab_ci_syntax_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze # The regex is needed to ensure a period (e.g. agpl-3.0) # isn't confused with a format type. We also need to allow encoded # values (e.g. C%2B%2B for C++), so allow % and + as well. @@ -16,7 +16,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' - requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' + requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|gitlab_ci_syntax_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of templates available to this project' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2012c348cd1..2d09ad01757 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -566,8 +566,8 @@ module API authorize_admin_project begin - ::Projects::HousekeepingService.new(user_project, :gc).execute - rescue ::Projects::HousekeepingService::LeaseTaken => error + ::Repositories::HousekeepingService.new(user_project, :gc).execute + rescue ::Repositories::HousekeepingService::LeaseTaken => error conflict!(error.message) end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index b3f09b431b0..f329a94adf2 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -91,6 +91,7 @@ module API optional :import_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, values: %w[github bitbucket bitbucket_server gitlab google_code fogbugz git gitlab_project gitea manifest phabricator], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :invisible_captcha_enabled, type: Boolean, desc: 'Enable Invisible Captcha spam detection during signup.' optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' optional :max_import_size, type: Integer, desc: 'Maximum import size in MB' diff --git a/lib/api/snippet_repository_storage_moves.rb b/lib/api/snippet_repository_storage_moves.rb new file mode 100644 index 00000000000..1a5b41eb1ec --- /dev/null +++ b/lib/api/snippet_repository_storage_moves.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module API + class SnippetRepositoryStorageMoves < ::API::Base + include PaginationParams + + before { authenticated_as_admin! } + + feature_category :gitaly + + resource :snippet_repository_storage_moves do + desc 'Get a list of all snippet repository storage moves' do + detail 'This feature was introduced in GitLab 13.8.' + success Entities::SnippetRepositoryStorageMove + end + params do + use :pagination + end + get do + storage_moves = SnippetRepositoryStorageMove.order_created_at_desc + + present paginate(storage_moves), with: Entities::SnippetRepositoryStorageMove, current_user: current_user + end + + desc 'Get a snippet repository storage move' do + detail 'This feature was introduced in GitLab 13.8.' + success Entities::SnippetRepositoryStorageMove + end + params do + requires :repository_storage_move_id, type: Integer, desc: 'The ID of a snippet repository storage move' + end + get ':repository_storage_move_id' do + storage_move = SnippetRepositoryStorageMove.find(params[:repository_storage_move_id]) + + present storage_move, with: Entities::SnippetRepositoryStorageMove, current_user: current_user + end + + desc 'Schedule bulk snippet repository storage moves' do + detail 'This feature was introduced in GitLab 13.8.' + end + params do + requires :source_storage_name, type: String, desc: 'The source storage shard', values: -> { Gitlab.config.repositories.storages.keys } + optional :destination_storage_name, type: String, desc: 'The destination storage shard', values: -> { Gitlab.config.repositories.storages.keys } + end + post do + ::Snippets::ScheduleBulkRepositoryShardMovesService.enqueue( + declared_params[:source_storage_name], + declared_params[:destination_storage_name] + ) + + accepted! + end + end + + params do + requires :id, type: String, desc: 'The ID of a snippet' + end + resource :snippets do + helpers do + def user_snippet + Snippet.find_by(id: params[:id]) # rubocop: disable CodeReuse/ActiveRecord + end + end + desc 'Get a list of all snippets repository storage moves' do + detail 'This feature was introduced in GitLab 13.8.' + success Entities::SnippetRepositoryStorageMove + end + params do + use :pagination + end + get ':id/repository_storage_moves' do + storage_moves = user_snippet.repository_storage_moves.order_created_at_desc + + present paginate(storage_moves), with: Entities::SnippetRepositoryStorageMove, current_user: current_user + end + + desc 'Get a snippet repository storage move' do + detail 'This feature was introduced in GitLab 13.8.' + success Entities::SnippetRepositoryStorageMove + end + params do + requires :repository_storage_move_id, type: Integer, desc: 'The ID of a snippet repository storage move' + end + get ':id/repository_storage_moves/:repository_storage_move_id' do + storage_move = user_snippet.repository_storage_moves.find(params[:repository_storage_move_id]) + + present storage_move, with: Entities::SnippetRepositoryStorageMove, current_user: current_user + end + + desc 'Schedule a snippet repository storage move' do + detail 'This feature was introduced in GitLab 13.8.' + success Entities::SnippetRepositoryStorageMove + end + params do + optional :destination_storage_name, type: String, desc: 'The destination storage shard' + end + post ':id/repository_storage_moves' do + storage_move = user_snippet.repository_storage_moves.build( + declared_params.merge(source_storage_name: user_snippet.repository_storage) + ) + + if storage_move.schedule + present storage_move, with: Entities::SnippetRepositoryStorageMove, current_user: current_user + else + render_validation_error!(storage_move) + end + end + end + end +end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index b7fb35eac03..bc1e427bcaa 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -13,6 +13,9 @@ module API gitlab_ci_ymls: { gitlab_version: 8.9 }, + gitlab_ci_syntax_ymls: { + gitlab_version: 13.8 + }, dockerfiles: { gitlab_version: 8.15 } diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index c664c0a4590..f6dfbcafbb6 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -14,6 +14,8 @@ module API before do authenticate! authorize! :read_terraform_state, user_project + + increment_unique_values('p_terraform_state_api_unique_users', current_user.id) end params do diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index cad2f52e951..c7d63f8d6ac 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -4,7 +4,7 @@ module API class UsageData < ::API::Base before { authenticate! } - feature_category :collection + feature_category :usage_ping namespace 'usage_data' do before do diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 3071f08e1de..31c923a219a 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -12,7 +12,9 @@ module API unauthorized! unless current_user { - merge_requests: current_user.assigned_open_merge_requests_count + merge_requests: current_user.assigned_open_merge_requests_count, # @deprecated + assigned_merge_requests: current_user.assigned_open_merge_requests_count, + review_requested_merge_requests: current_user.review_requested_open_merge_requests_count } end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 8b9b82877f7..cee09f60a2b 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -87,6 +87,7 @@ module API optional :created_before, type: DateTime, desc: 'Return users created before the specified time' optional :without_projects, type: Boolean, default: false, desc: 'Filters only users without projects' optional :exclude_internal, as: :non_internal, type: Boolean, default: false, desc: 'Filters only non internal users' + optional :admins, type: Boolean, default: false, desc: 'Filters only admin users' all_or_none_of :extern_uid, :provider use :sort_params @@ -745,8 +746,6 @@ module API optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the personal access token' end post feature_category: :authentication_and_authorization do - not_found! unless Feature.enabled?(:pat_creation_api_for_admin) - response = ::PersonalAccessTokens::CreateService.new( current_user: current_user, target_user: target_user, params: declared_params(include_missing: false) ).execute diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index da24d0e20ee..c67fe24d456 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -16,11 +16,15 @@ module Atlassian common = { project: project, update_sequence_id: update_sequence_id } dev_info = args.slice(:commits, :branches, :merge_requests) build_info = args.slice(:pipelines) + deploy_info = args.slice(:deployments) + ff_info = args.slice(:feature_flags) responses = [] responses << store_dev_info(**common, **dev_info) if dev_info.present? responses << store_build_info(**common, **build_info) if build_info.present? + responses << store_deploy_info(**common, **deploy_info) if deploy_info.present? + responses << store_ff_info(**common, **ff_info) if ff_info.present? raise ArgumentError, 'Invalid arguments' if responses.empty? responses.compact @@ -28,11 +32,47 @@ module Atlassian private + def store_ff_info(project:, feature_flags:, **opts) + return unless Feature.enabled?(:jira_sync_feature_flags, project) + + items = feature_flags.map { |flag| ::Atlassian::JiraConnect::Serializers::FeatureFlagEntity.represent(flag, opts) } + items.reject! { |item| item.issue_keys.empty? } + + return if items.empty? + + r = post('/rest/featureflags/0.1/bulk', { + flags: items, + properties: { projectId: "project-#{project.id}" } + }) + + handle_response(r, 'feature flags') do |data| + failed = data['failedFeatureFlags'] + if failed.present? + errors = failed.flat_map do |k, errs| + errs.map { |e| "#{k}: #{e['message']}" } + end + { 'errorMessages' => errors } + end + end + end + + def store_deploy_info(project:, deployments:, **opts) + return unless Feature.enabled?(:jira_sync_deployments, project) + + items = deployments.map { |d| ::Atlassian::JiraConnect::Serializers::DeploymentEntity.represent(d, opts) } + items.reject! { |d| d.issue_keys.empty? } + + return if items.empty? + + r = post('/rest/deployments/0.1/bulk', { deployments: items }) + handle_response(r, 'deployments') { |data| errors(data, 'rejectedDeployments') } + end + def store_build_info(project:, pipelines:, update_sequence_id: nil) return unless Feature.enabled?(:jira_sync_builds, project) builds = pipelines.map do |pipeline| - build = Serializers::BuildEntity.represent( + build = ::Atlassian::JiraConnect::Serializers::BuildEntity.represent( pipeline, update_sequence_id: update_sequence_id ) @@ -42,7 +82,8 @@ module Atlassian end.compact return if builds.empty? - post('/rest/builds/0.1/bulk', { builds: builds }) + r = post('/rest/builds/0.1/bulk', { builds: builds }) + handle_response(r, 'builds') { |data| errors(data, 'rejectedBuilds') } end def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil) @@ -75,6 +116,34 @@ module Atlassian { providerMetadata: { product: "GitLab #{Gitlab::VERSION}" } } end + def handle_response(response, name, &block) + data = response.parsed_response + + case response.code + when 200 then yield data + when 400 then { 'errorMessages' => data.map { |e| e['message'] } } + when 401 then { 'errorMessages' => ['Invalid JWT'] } + when 403 then { 'errorMessages' => ["App does not support #{name}"] } + when 413 then { 'errorMessages' => ['Data too large'] + data.map { |e| e['message'] } } + when 429 then { 'errorMessages' => ['Rate limit exceeded'] } + when 503 then { 'errorMessages' => ['Service unavailable'] } + else + { 'errorMessages' => ['Unknown error'], 'response' => data } + end + end + + def errors(data, key) + messages = if data[key].present? + data[key].flat_map do |rejection| + rejection['errors'].map { |e| e['message'] } + end + else + [] + end + + { 'errorMessages' => messages } + end + def user_notes_count(merge_requests) return unless merge_requests diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb index 3eb8b1f1978..8372d2a62da 100644 --- a/lib/atlassian/jira_connect/serializers/build_entity.rb +++ b/lib/atlassian/jira_connect/serializers/build_entity.rb @@ -25,8 +25,10 @@ module Atlassian # extract Jira issue keys from either the source branch/ref or the # merge request title. @issue_keys ||= begin - src = "#{pipeline.source_ref} #{pipeline.merge_request&.title}" - JiraIssueKeyExtractor.new(src).issue_keys + pipeline.all_merge_requests.flat_map do |mr| + src = "#{mr.source_branch} #{mr.title}" + JiraIssueKeyExtractor.new(src).issue_keys + end.uniq end end diff --git a/lib/atlassian/jira_connect/serializers/deployment_entity.rb b/lib/atlassian/jira_connect/serializers/deployment_entity.rb new file mode 100644 index 00000000000..9ef1666b61c --- /dev/null +++ b/lib/atlassian/jira_connect/serializers/deployment_entity.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Serializers + class DeploymentEntity < Grape::Entity + include Gitlab::Routing + + format_with(:iso8601, &:iso8601) + + expose :schema_version, as: :schemaVersion + expose :iid, as: :deploymentSequenceNumber + expose :update_sequence_id, as: :updateSequenceNumber + expose :display_name, as: :displayName + expose :description + expose :associations + expose :url + expose :label + expose :state + expose :updated_at, as: :lastUpdated, format_with: :iso8601 + expose :pipeline_entity, as: :pipeline + expose :environment_entity, as: :environment + + def issue_keys + return [] unless build&.pipeline.present? + + @issue_keys ||= BuildEntity.new(build.pipeline).issue_keys + end + + private + + delegate :project, :deployable, :environment, :iid, :ref, :short_sha, to: :object + alias_method :deployment, :object + alias_method :build, :deployable + + def associations + keys = issue_keys + + [{ associationType: :issueKeys, values: keys }] if keys.present? + end + + def display_name + "Deployment #{iid} (#{ref}@#{short_sha}) to #{environment.name}" + end + + def label + "#{project.full_path}-#{environment.name}-#{iid}-#{short_sha}" + end + + def description + "Deployment #{deployment.iid} of #{project.name} at #{short_sha} (#{build&.name}) to #{environment.name}" + end + + def url + # There is no controller action to show a single deployment, so we + # link to the build instead + project_job_url(project, build) if build + end + + def state + case deployment.status + when 'created' then 'pending' + when 'running' then 'in_progress' + when 'success' then 'successful' + when 'failed' then 'failed' + when 'canceled', 'skipped' then 'cancelled' + else + 'unknown' + end + end + + def schema_version + '1.0' + end + + def pipeline_entity + PipelineEntity.new(build.pipeline) if build&.pipeline.present? + end + + def environment_entity + EnvironmentEntity.new(environment) + end + + def update_sequence_id + options[:update_sequence_id] || Client.generate_update_sequence_id + end + end + end + end +end diff --git a/lib/atlassian/jira_connect/serializers/environment_entity.rb b/lib/atlassian/jira_connect/serializers/environment_entity.rb new file mode 100644 index 00000000000..f3699e4d0ee --- /dev/null +++ b/lib/atlassian/jira_connect/serializers/environment_entity.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Serializers + class EnvironmentEntity < Grape::Entity + format_with(:string, &:to_s) + + expose :id, format_with: :string + expose :display_name, as: :displayName + expose :type + + private + + alias_method :environment, :object + delegate :project, to: :object + + def display_name + "#{project.name}/#{environment.name}" + end + + def type + case environment.name + when /prod/i + 'production' + when /test/i + 'testing' + when /staging/i + 'staging' + when /(dev|review)/i + 'development' + else + 'unmapped' + end + end + end + end + end +end diff --git a/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb b/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb new file mode 100644 index 00000000000..e17c150aacb --- /dev/null +++ b/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Serializers + class FeatureFlagEntity < Grape::Entity + include Gitlab::Routing + + alias_method :flag, :object + + format_with(:string, &:to_s) + + expose :schema_version, as: :schemaVersion + expose :id, format_with: :string + expose :name, as: :key + expose :update_sequence_id, as: :updateSequenceId + expose :name, as: :displayName + expose :summary + expose :details + expose :issue_keys, as: :issueKeys + + def issue_keys + @issue_keys ||= JiraIssueKeyExtractor.new(flag.description).issue_keys + end + + def schema_version + '1.0' + end + + def update_sequence_id + options[:update_sequence_id] || Client.generate_update_sequence_id + end + + STRATEGY_NAMES = { + ::Operations::FeatureFlags::Strategy::STRATEGY_DEFAULT => 'All users', + ::Operations::FeatureFlags::Strategy::STRATEGY_GITLABUSERLIST => 'User List', + ::Operations::FeatureFlags::Strategy::STRATEGY_GRADUALROLLOUTUSERID => 'Percent of users', + ::Operations::FeatureFlags::Strategy::STRATEGY_FLEXIBLEROLLOUT => 'Percent rollout', + ::Operations::FeatureFlags::Strategy::STRATEGY_USERWITHID => 'User IDs' + }.freeze + + private + + # The summary does not map very well to our FeatureFlag model. + # + # We allow feature flags to have multiple strategies, depending + # on the environment. Jira expects a single rollout strategy. + # + # Also, we don't actually support showing a single flag, so we use the + # edit path as an interim solution. + def summary(strategies = flag.strategies) + { + url: project_url(flag.project) + "/-/feature_flags/#{flag.id}/edit", + lastUpdated: flag.updated_at.iso8601, + status: { + enabled: flag.active, + defaultValue: '', + rollout: { + percentage: strategies.map do |s| + s.parameters['rollout'] || s.parameters['percentage'] + end.compact.first&.to_f, + text: strategies.map { |s| STRATEGY_NAMES[s.name] }.compact.join(', ') + }.compact + } + } + end + + def details + envs = flag.strategies.flat_map do |s| + s.scopes.map do |es| + env_type = es.environment_scope.scan(/development|testing|staging|production/).first + [es.environment_scope, env_type, s] + end + end + + envs.map do |env_name, env_type, strat| + summary([strat]).merge(environment: { name: env_name, type: env_type }.compact) + end + end + end + end + end +end diff --git a/lib/atlassian/jira_connect/serializers/pipeline_entity.rb b/lib/atlassian/jira_connect/serializers/pipeline_entity.rb new file mode 100644 index 00000000000..e67cf1a7229 --- /dev/null +++ b/lib/atlassian/jira_connect/serializers/pipeline_entity.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Serializers + # Both this an BuildEntity represent a Ci::Pipeline + class PipelineEntity < Grape::Entity + include Gitlab::Routing + + format_with(:string, &:to_s) + + expose :id, format_with: :string + expose :display_name, as: :displayName + expose :url + + private + + alias_method :pipeline, :object + delegate :project, to: :object + + def display_name + "#{project.name} pipeline #{pipeline.iid}" + end + + def url + project_pipeline_url(project, pipeline) + end + end + end + end +end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 4248a86dc7c..d15114a72a3 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -40,31 +40,42 @@ module Backup end def restore + restore_project_repositories + restore_snippets + + restore_object_pools + end + + private + + def restore_project_repositories Project.find_each(batch_size: 1000) do |project| restore_repository(project, Gitlab::GlRepository::PROJECT) restore_repository(project, Gitlab::GlRepository::WIKI) restore_repository(project, Gitlab::GlRepository::DESIGN) end + end + def restore_snippets invalid_ids = Snippet.find_each(batch_size: 1000) .map { |snippet| restore_snippet_repository(snippet) } .compact cleanup_snippets_without_repositories(invalid_ids) - - restore_object_pools end - private - def check_valid_storages! - [ProjectRepository, SnippetRepository].each do |klass| + repository_storage_klasses.each do |klass| if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}" end end end + def repository_storage_klasses + [ProjectRepository, SnippetRepository] + end + def backup_repos_path @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') end @@ -103,12 +114,7 @@ module Backup end begin - case container - when Project - dump_project(container) - when Snippet - dump_snippet(container) - end + dump_container(container) rescue => e errors << e break @@ -130,6 +136,15 @@ module Backup end end + def dump_container(container) + case container + when Project + dump_project(container) + when Snippet + dump_snippet(container) + end + end + def dump_project(project) backup_repository(project, Gitlab::GlRepository::PROJECT) backup_repository(project, Gitlab::GlRepository::WIKI) @@ -308,3 +323,5 @@ module Backup end end end + +Backup::Repositories.prepend_if_ee('EE::Backup::Repositories') diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb index 11762c3bfb4..67f5baf4635 100644 --- a/lib/banzai/filter/ascii_doc_sanitization_filter.rb +++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb @@ -27,7 +27,7 @@ module Banzai TABLE_GRID_CLASSES = %w(grid-all grid-rows grid-cols grid-none).freeze TABLE_STRIPES_CLASSES = %w(stripes-all stripes-odd stripes-even stripes-hover stripes-none).freeze - ELEMENT_CLASSES_WHITELIST = { + ELEMENT_CLASSES_ALLOWLIST = { span: %w(big small underline overline line-through).freeze, div: ALIGNMENT_BUILTINS_CLASSES + ['admonitionblock'].freeze, td: ['icon'].freeze, @@ -38,35 +38,35 @@ module Banzai table: TABLE_FRAME_CLASSES + TABLE_GRID_CLASSES + TABLE_STRIPES_CLASSES }.freeze - def customize_whitelist(whitelist) + def customize_allowlist(allowlist) # Allow marks - whitelist[:elements].push('mark') + allowlist[:elements].push('mark') # Allow any classes in `span`, `i`, `div`, `td`, `ul`, `ol` and `a` elements # but then remove any unknown classes - whitelist[:attributes]['span'] = %w(class) - whitelist[:attributes]['div'].push('class') - whitelist[:attributes]['td'] = %w(class) - whitelist[:attributes]['i'] = %w(class) - whitelist[:attributes]['ul'] = %w(class) - whitelist[:attributes]['ol'] = %w(class) - whitelist[:attributes]['a'].push('class') - whitelist[:attributes]['table'] = %w(class) - whitelist[:transformers].push(self.class.remove_element_classes) + allowlist[:attributes]['span'] = %w(class) + allowlist[:attributes]['div'].push('class') + allowlist[:attributes]['td'] = %w(class) + allowlist[:attributes]['i'] = %w(class) + allowlist[:attributes]['ul'] = %w(class) + allowlist[:attributes]['ol'] = %w(class) + allowlist[:attributes]['a'].push('class') + allowlist[:attributes]['table'] = %w(class) + allowlist[:transformers].push(self.class.remove_element_classes) # Allow `id` in anchor and footnote elements - whitelist[:attributes]['a'].push('id') - whitelist[:attributes]['div'].push('id') + allowlist[:attributes]['a'].push('id') + allowlist[:attributes]['div'].push('id') # Allow `id` in heading elements for section anchors SECTION_HEADINGS.each do |header| - whitelist[:attributes][header] = %w(id) + allowlist[:attributes][header] = %w(id) end # Remove ids that are not explicitly allowed - whitelist[:transformers].push(self.class.remove_disallowed_ids) + allowlist[:transformers].push(self.class.remove_disallowed_ids) - whitelist + allowlist end class << self @@ -91,11 +91,11 @@ module Banzai lambda do |env| node = env[:node] - return unless (classes_whitelist = ELEMENT_CLASSES_WHITELIST[node.name.to_sym]) + return unless (classes_allowlist = ELEMENT_CLASSES_ALLOWLIST[node.name.to_sym]) return unless node.has_attribute?('class') classes = node['class'].strip.split(' ') - allowed_classes = (classes & classes_whitelist) + allowed_classes = (classes & classes_allowlist) if allowed_classes.empty? node.remove_attribute('class') else diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb index 8acd3917d81..55dc426edaf 100644 --- a/lib/banzai/filter/asset_proxy_filter.rb +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -15,7 +15,7 @@ module Banzai needs(:asset_proxy, :asset_proxy_secret_key) if asset_proxy_enabled? end - def asset_host_whitelisted?(host) + def asset_host_allowed?(host) context[:asset_proxy_domain_regexp] ? context[:asset_proxy_domain_regexp].match?(host) : false end @@ -44,21 +44,21 @@ module Banzai Gitlab.config.asset_proxy['enabled'] = application_settings.asset_proxy_enabled Gitlab.config.asset_proxy['url'] = application_settings.asset_proxy_url Gitlab.config.asset_proxy['secret_key'] = application_settings.asset_proxy_secret_key - Gitlab.config.asset_proxy['whitelist'] = determine_whitelist(application_settings) - Gitlab.config.asset_proxy['domain_regexp'] = compile_whitelist(Gitlab.config.asset_proxy.whitelist) + Gitlab.config.asset_proxy['allowlist'] = determine_allowlist(application_settings) + Gitlab.config.asset_proxy['domain_regexp'] = compile_allowlist(Gitlab.config.asset_proxy.allowlist) else Gitlab.config.asset_proxy['enabled'] = ::ApplicationSetting.defaults[:asset_proxy_enabled] end end - def self.compile_whitelist(domain_list) + def self.compile_allowlist(domain_list) return if domain_list.empty? escaped = domain_list.map { |domain| Regexp.escape(domain).gsub('\*', '.*?') } Regexp.new("^(#{escaped.join('|')})$", Regexp::IGNORECASE) end - def self.determine_whitelist(application_settings) + def self.determine_allowlist(application_settings) application_settings.asset_proxy_whitelist.presence || [Gitlab.config.gitlab.host] end end diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb index 4f9e8cffd11..c63453f94ca 100644 --- a/lib/banzai/filter/base_sanitization_filter.rb +++ b/lib/banzai/filter/base_sanitization_filter.rb @@ -16,42 +16,42 @@ module Banzai UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze - def whitelist - strong_memoize(:whitelist) do - whitelist = super.deep_dup + def allowlist + strong_memoize(:allowlist) do + allowlist = super.deep_dup # Allow span elements - whitelist[:elements].push('span') + allowlist[:elements].push('span') # Allow data-math-style attribute in order to support LaTeX formatting - whitelist[:attributes]['code'] = %w(data-math-style) - whitelist[:attributes]['pre'] = %w(data-math-style data-mermaid-style data-kroki-style) + allowlist[:attributes]['code'] = %w(data-math-style) + allowlist[:attributes]['pre'] = %w(data-math-style data-mermaid-style data-kroki-style) # Allow html5 details/summary elements - whitelist[:elements].push('details') - whitelist[:elements].push('summary') + allowlist[:elements].push('details') + allowlist[:elements].push('summary') # Allow abbr elements with title attribute - whitelist[:elements].push('abbr') - whitelist[:attributes]['abbr'] = %w(title) + allowlist[:elements].push('abbr') + allowlist[:attributes]['abbr'] = %w(title) # Disallow `name` attribute globally, allow on `a` - whitelist[:attributes][:all].delete('name') - whitelist[:attributes]['a'].push('name') + allowlist[:attributes][:all].delete('name') + allowlist[:attributes]['a'].push('name') # Allow any protocol in `a` elements # and then remove links with unsafe protocols - whitelist[:protocols].delete('a') - whitelist[:transformers].push(self.class.method(:remove_unsafe_links)) + allowlist[:protocols].delete('a') + allowlist[:transformers].push(self.class.method(:remove_unsafe_links)) # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(self.class.remove_rel) + allowlist[:transformers].push(self.class.remove_rel) - customize_whitelist(whitelist) + customize_allowlist(allowlist) end end - def customize_whitelist(whitelist) + def customize_allowlist(allowlist) raise NotImplementedError end diff --git a/lib/banzai/filter/broadcast_message_sanitization_filter.rb b/lib/banzai/filter/broadcast_message_sanitization_filter.rb index 042293170c8..183908d02a9 100644 --- a/lib/banzai/filter/broadcast_message_sanitization_filter.rb +++ b/lib/banzai/filter/broadcast_message_sanitization_filter.rb @@ -6,14 +6,14 @@ module Banzai # # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. class BroadcastMessageSanitizationFilter < Banzai::Filter::BaseSanitizationFilter - def customize_whitelist(whitelist) - whitelist[:elements].push('br') + def customize_allowlist(allowlist) + allowlist[:elements].push('br') - whitelist[:attributes]['a'].push('class', 'style') + allowlist[:attributes]['a'].push('class', 'style') - whitelist[:css] = { properties: %w(color border background padding margin text-decoration) } + allowlist[:css] = { properties: %w(color border background padding margin text-decoration) } - whitelist + allowlist end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index f57e57890f8..f6314040f28 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -9,26 +9,26 @@ module Banzai # Styles used by Markdown for table alignment TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze - def customize_whitelist(whitelist) - # Allow table alignment; we whitelist specific text-align values in a + def customize_allowlist(allowlist) + # Allow table alignment; we allow specific text-align values in a # transformer below - whitelist[:attributes]['th'] = %w(style) - whitelist[:attributes]['td'] = %w(style) - whitelist[:css] = { properties: ['text-align'] } + allowlist[:attributes]['th'] = %w(style) + allowlist[:attributes]['td'] = %w(style) + allowlist[:css] = { properties: ['text-align'] } # Allow the 'data-sourcepos' from CommonMark on all elements - whitelist[:attributes][:all].push('data-sourcepos') + allowlist[:attributes][:all].push('data-sourcepos') # Remove any `style` properties not required for table alignment - whitelist[:transformers].push(self.class.remove_unsafe_table_style) + allowlist[:transformers].push(self.class.remove_unsafe_table_style) # Allow `id` in a and li elements for footnotes # and remove any `id` properties not matching for footnotes - whitelist[:attributes]['a'].push('id') - whitelist[:attributes]['li'] = %w(id) - whitelist[:transformers].push(self.class.remove_non_footnote_ids) + allowlist[:attributes]['a'].push('id') + allowlist[:attributes]['li'] = %w(id) + allowlist[:transformers].push(self.class.remove_non_footnote_ids) - whitelist + allowlist end class << self diff --git a/lib/banzai/filter/truncate_source_filter.rb b/lib/banzai/filter/truncate_source_filter.rb new file mode 100644 index 00000000000..c903b83d868 --- /dev/null +++ b/lib/banzai/filter/truncate_source_filter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class TruncateSourceFilter < HTML::Pipeline::TextFilter + def call + return text unless context.key?(:limit) + + text.truncate_bytes(context[:limit]) + end + end + end +end diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index d5ff9b025cc..8f8ce1cbd41 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -3,14 +3,14 @@ module Banzai module Pipeline class DescriptionPipeline < FullPipeline - WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( + ALLOWLIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li) ) def self.transform_context(context) super(context).merge( # SanitizationFilter - whitelist: WHITELIST + allowlist: ALLOWLIST ) end end diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb index 1f7cb437fcd..eb6f35b0e2a 100644 --- a/lib/banzai/pipeline/pre_process_pipeline.rb +++ b/lib/banzai/pipeline/pre_process_pipeline.rb @@ -6,6 +6,7 @@ module Banzai def self.filters FilterArray[ Filter::NormalizeSourceFilter, + Filter::TruncateSourceFilter, Filter::FrontMatterFilter, Filter::BlockquoteFenceFilter, ] diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb index c0cef61d2b2..af274ee1299 100644 --- a/lib/bulk_imports/common/extractors/graphql_extractor.rb +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -11,14 +11,10 @@ module BulkImports def extract(context) client = graphql_client(context) - Enumerator.new do |yielder| - result = client.execute( - client.parse(query.to_s), - query.variables(context.entity) - ) - - yielder << result.original_hash.deep_dup - end + client.execute( + client.parse(query.to_s), + query.variables(context.entity) + ).original_hash.deep_dup end private diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index 82cb1ca03a2..6e1b86e9515 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -8,7 +8,6 @@ module BulkImports end def execute - entity.start! bulk_import = entity.bulk_import configuration = bulk_import.configuration @@ -18,9 +17,7 @@ module BulkImports configuration: configuration ) - BulkImports::Groups::Pipelines::GroupPipeline.new.run(context) - 'BulkImports::EE::Groups::Pipelines::EpicsPipeline'.constantize.new.run(context) if Gitlab.ee? - BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline.new.run(context) + pipelines.each { |pipeline| pipeline.new.run(context) } entity.finish! end @@ -28,6 +25,15 @@ module BulkImports private attr_reader :entity + + def pipelines + [ + BulkImports::Groups::Pipelines::GroupPipeline, + BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline + ] + end end end end + +BulkImports::Importers::GroupImporter.prepend_if_ee('EE::BulkImports::Importers::GroupImporter') diff --git a/lib/bulk_imports/importers/groups_importer.rb b/lib/bulk_imports/importers/groups_importer.rb deleted file mode 100644 index 8641577ff47..00000000000 --- a/lib/bulk_imports/importers/groups_importer.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Importers - class GroupsImporter - def initialize(bulk_import_id) - @bulk_import = BulkImport.find(bulk_import_id) - end - - def execute - bulk_import.start! unless bulk_import.started? - - if entities_to_import.empty? - bulk_import.finish! - else - entities_to_import.each do |entity| - BulkImports::Importers::GroupImporter.new(entity).execute - end - - # A new BulkImportWorker job is enqueued to either - # - Process the new BulkImports::Entity created for the subgroups - # - Or to mark the `bulk_import` as finished. - BulkImportWorker.perform_async(bulk_import.id) - end - end - - private - - attr_reader :bulk_import - - def entities_to_import - @entities_to_import ||= bulk_import.entities.with_status(:created) - end - end - end -end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index a44f8fc7193..06b81b5da14 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -10,16 +10,16 @@ module BulkImports private - def extractors - @extractors ||= self.class.extractors.map(&method(:instantiate)) + def extractor + @extractor ||= instantiate(self.class.get_extractor) end def transformers @transformers ||= self.class.transformers.map(&method(:instantiate)) end - def loaders - @loaders ||= self.class.loaders.map(&method(:instantiate)) + def loader + @loaders ||= instantiate(self.class.get_loader) end def after_run @@ -41,7 +41,7 @@ module BulkImports class_methods do def extractor(klass, options = nil) - add_attribute(:extractors, klass, options) + class_attributes[:extractor] = { klass: klass, options: options } end def transformer(klass, options = nil) @@ -49,23 +49,23 @@ module BulkImports end def loader(klass, options = nil) - add_attribute(:loaders, klass, options) + class_attributes[:loader] = { klass: klass, options: options } end def after_run(&block) class_attributes[:after_run] = block end - def extractors - class_attributes[:extractors] + def get_extractor + class_attributes[:extractor] end def transformers class_attributes[:transformers] end - def loaders - class_attributes[:loaders] + def get_loader + class_attributes[:loader] end def after_run_callback diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 88b96f0ab6e..11fb9722173 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -12,25 +12,15 @@ module BulkImports info(context, message: 'Pipeline started', pipeline_class: pipeline) - extractors.each do |extractor| - data = run_pipeline_step(:extractor, extractor.class.name, context) do - extractor.extract(context) + Array.wrap(extracted_data_from(context)).each do |entry| + transformers.each do |transformer| + entry = run_pipeline_step(:transformer, transformer.class.name, context) do + transformer.transform(context, entry) + end end - if data && data.respond_to?(:each) - data.each do |entry| - transformers.each do |transformer| - entry = run_pipeline_step(:transformer, transformer.class.name, context) do - transformer.transform(context, entry) - end - end - - loaders.each do |loader| - run_pipeline_step(:loader, loader.class.name, context) do - loader.load(context, entry) - end - end - end + run_pipeline_step(:loader, loader.class.name, context) do + loader.load(context, entry) end end @@ -55,6 +45,12 @@ module BulkImports mark_as_failed(context) if abort_on_failure? end + def extracted_data_from(context) + run_pipeline_step(:extractor, extractor.class.name, context) do + extractor.extract(context) + end + end + def mark_as_failed(context) warn(context, message: 'Pipeline failed', pipeline_class: pipeline) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 35f299c17e4..6f5acabe81f 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -22,6 +22,23 @@ module ContainerRegistry # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] + RETRY_EXCEPTIONS = [Faraday::Request::Retry::DEFAULT_EXCEPTIONS, Faraday::ConnectionFailed].flatten.freeze + RETRY_OPTIONS = { + max: 1, + interval: 5, + exceptions: RETRY_EXCEPTIONS + }.freeze + + ERROR_CALLBACK_OPTIONS = { + callback: -> (env, exception) do + Gitlab::ErrorTracking.log_exception( + exception, + class: name, + url: env[:url] + ) + end + }.freeze + def self.supports_tag_delete? registry_config = Gitlab.config.registry return false unless registry_config.enabled && registry_config.api_url.present? @@ -97,12 +114,12 @@ module ContainerRegistry end def upload_blob(name, content, digest) - upload = faraday.post("/v2/#{name}/blobs/uploads/") + upload = faraday(timeout_enabled: false).post("/v2/#{name}/blobs/uploads/") return upload unless upload.success? location = URI(upload.headers['location']) - faraday.put("#{location.path}?#{location.query}") do |req| + faraday(timeout_enabled: false).put("#{location.path}?#{location.query}") do |req| req.params['digest'] = digest req.headers['Content-Type'] = 'application/octet-stream' req.body = content @@ -137,7 +154,7 @@ module ContainerRegistry end def put_tag(name, reference, manifest) - response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req| + response = faraday(timeout_enabled: false).put("/v2/#{name}/manifests/#{reference}") do |req| req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE req.body = Gitlab::Json.pretty_generate(manifest) end @@ -158,6 +175,8 @@ module ContainerRegistry yield(conn) if block_given? + conn.request(:retry, RETRY_OPTIONS) + conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) conn.adapter :net_http end @@ -188,8 +207,8 @@ module ContainerRegistry faraday_redirect.get(uri) end - def faraday - @faraday ||= faraday_base do |conn| + def faraday(timeout_enabled: true) + @faraday ||= faraday_base(timeout_enabled: timeout_enabled) do |conn| initialize_connection(conn, @options, &method(:accept_manifest)) end end @@ -205,12 +224,22 @@ module ContainerRegistry def faraday_redirect @faraday_redirect ||= faraday_base do |conn| conn.request :json + + conn.request(:retry, RETRY_OPTIONS) + conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) conn.adapter :net_http end end - def faraday_base(&block) - Faraday.new(@base_uri, headers: { user_agent: "GitLab/#{Gitlab::VERSION}" }, &block) + def faraday_base(timeout_enabled: true, &block) + request_options = timeout_enabled ? Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS : nil + + Faraday.new( + @base_uri, + headers: { user_agent: "GitLab/#{Gitlab::VERSION}" }, + request: request_options, + &block + ) end def delete_if_exists(path) diff --git a/lib/declarative_enum.rb b/lib/declarative_enum.rb new file mode 100644 index 00000000000..f3c8c181c73 --- /dev/null +++ b/lib/declarative_enum.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# Extending this module will give you the ability of defining +# enum values in a declarative way. +# +# module DismissalReasons +# extend DeclarativeEnum +# +# key :dismissal_reason +# name 'DismissalReasonOfVulnerability' +# +# description <<~TEXT +# This enum holds the user selected dismissal reason +# when they are dismissing the vulnerabilities +# TEXT +# +# define do +# acceptable_risk value: 0, description: 'The vulnerability is known but is considered to be an acceptable business risk.' +# false_positive value: 1, description: 'An error in reporting the presence of a vulnerability in a system when the vulnerability is not present.' +# used_in_tests value: 2, description: 'The finding is not a vulnerability because it is part of a test or is test data.' +# end +# +# Then we can use this module to register enums for our Active Record models like so, +# +# class VulnerabilityFeedback +# declarative_enum DismissalReasons +# end +# +# Also we can use this module to create GraphQL Enum types like so, +# +# module Types +# module Vulnerabilities +# class DismissalReasonEnum < BaseEnum +# declarative_enum DismissalReasons +# end +# end +# end +# +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module DeclarativeEnum + # This `prepended` hook will merge the enum definition + # of the prepended module into the base module to be + # used by `prepend_if_ee` helper method. + def prepended(base) + base.definition.merge!(definition) + end + + def key(new_key = nil) + @key = new_key if new_key + + @key + end + + def name(new_name = nil) + @name = new_name if new_name + + @name + end + + def description(new_description = nil) + @description = new_description if new_description + + @description + end + + def define(&block) + raise LocalJumpError.new('No block given') unless block + + @definition = Builder.new(definition, block).build + end + + # We can use this method later to apply some sanity checks + # but for now, returning a Hash without any check is enough. + def definition + @definition.to_h + end + + class Builder + KeyCollisionError = Class.new(StandardError) + + def initialize(definition, block) + @definition = definition + @block = block + end + + def build + instance_exec(&@block) + + @definition + end + + private + + def method_missing(name, *arguments, value: nil, description: nil, &block) + key = name.downcase.to_sym + raise KeyCollisionError, "'#{key}' collides with an existing enum key!" if @definition[key] + + @definition[key] = { + value: value, + description: description + } + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index dc8f9d0c970..06cebab8f0a 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -16,6 +16,12 @@ module ExpandVariables end end + def possible_var_reference?(value) + return unless value + + %w[$ %].any? { |symbol| value.include?(symbol) } + end + private def replace_with(value, variables) diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 17dfe26bd82..5ad9af6ff7d 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -53,10 +53,12 @@ class Feature description: 'Short lived, used specifically to run A/B/n experiments.', optional: true, rollout_issue: true, - ee_only: true, + ee_only: false, default_enabled: false, example: <<-EOS experiment(:my_experiment, project: project, actor: current_user) { ...variant code... } + # or + Gitlab::Experimentation.in_experiment_group?(:my_experiment, subject: current_user) EOS } }.freeze diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 22aa680cbc1..43683ae174e 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -30,6 +30,10 @@ module Gitlab all.map { |stage| stage[:name] } end + def self.symbolized_stage_names + names.map(&:to_sym) + end + def self.params_for_issue_stage { name: 'issue', diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 39dc706dff5..27fc8bd9a1a 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -11,6 +11,7 @@ module Gitlab ENUM_MAPPING = { StageEvents::IssueCreated => 1, StageEvents::IssueFirstMentionedInCommit => 2, + StageEvents::IssueDeployedToProduction => 3, StageEvents::MergeRequestCreated => 100, StageEvents::MergeRequestFirstDeployedToProduction => 101, StageEvents::MergeRequestLastBuildFinished => 102, @@ -18,8 +19,7 @@ module Gitlab StageEvents::MergeRequestMerged => 104, StageEvents::CodeStageStart => 1_000, StageEvents::IssueStageEnd => 1_001, - StageEvents::PlanStageStart => 1_002, - StageEvents::ProductionStageEnd => 1_003 + StageEvents::PlanStageStart => 1_002 }.freeze EVENTS = ENUM_MAPPING.keys.freeze @@ -27,8 +27,7 @@ module Gitlab INTERNAL_EVENTS = [ StageEvents::CodeStageStart, StageEvents::IssueStageEnd, - StageEvents::PlanStageStart, - StageEvents::ProductionStageEnd + StageEvents::PlanStageStart ].freeze # Defines which start_event and end_event pairs are allowed @@ -41,7 +40,7 @@ module Gitlab ], StageEvents::IssueCreated => [ StageEvents::IssueStageEnd, - StageEvents::ProductionStageEnd + StageEvents::IssueDeployedToProduction ], StageEvents::MergeRequestCreated => [ StageEvents::MergeRequestMerged diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb index b778364a917..3e93e60e686 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb @@ -4,13 +4,13 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class ProductionStageEnd < StageEvent + class IssueDeployedToProduction < StageEvent def self.name _("Issue first deployed to production") end def self.identifier - :production_stage_end + :issue_deployed_to_production end def object_type diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index 292048dcad9..e367d33d743 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics class UniqueVisits def track_visit(visitor_id, target_id, time = Time.zone.now) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, target_id, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(target_id, values: visitor_id, time: time) end # Returns number of unique visitors for given targets in given time frame diff --git a/lib/gitlab/api_authentication/builder.rb b/lib/gitlab/api_authentication/builder.rb new file mode 100644 index 00000000000..717c664826a --- /dev/null +++ b/lib/gitlab/api_authentication/builder.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Authentication Strategies Builder +# +# AuthBuilder and its child classes, TokenType and SentThrough, support +# declaring allowed authentication strategies with patterns like +# `accept.token_type(:job_token).sent_through(:http_basic)`. +module Gitlab + module APIAuthentication + class Builder + def build + strategies = Hash.new([]) + yield ::Gitlab::APIAuthentication::TokenTypeBuilder.new(strategies) + strategies + end + end + end +end diff --git a/lib/gitlab/api_authentication/sent_through_builder.rb b/lib/gitlab/api_authentication/sent_through_builder.rb new file mode 100644 index 00000000000..f66e5960019 --- /dev/null +++ b/lib/gitlab/api_authentication/sent_through_builder.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# See Gitlab::APIAuthentication::Builder +module Gitlab + module APIAuthentication + class SentThroughBuilder + def initialize(strategies, resolvers) + @strategies = strategies + @resolvers = resolvers + end + + def sent_through(*locators) + locators.each do |locator| + @strategies[locator] |= @resolvers + end + end + end + end +end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb new file mode 100644 index 00000000000..32a98908e5b --- /dev/null +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module APIAuthentication + class TokenLocator + UsernameAndPassword = Struct.new(:username, :password) + + include ActiveModel::Validations + include ActionController::HttpAuthentication::Basic + + attr_reader :location + + validates :location, inclusion: { in: %i[http_basic_auth] } + + def initialize(location) + @location = location + validate! + end + + def extract(request) + case @location + when :http_basic_auth + extract_from_http_basic_auth request + end + end + + private + + def extract_from_http_basic_auth(request) + username, password = user_name_and_password(request) + return unless username.present? && password.present? + + UsernameAndPassword.new(username, password) + end + end + end +end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb new file mode 100644 index 00000000000..5b30777b6ec --- /dev/null +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module APIAuthentication + class TokenResolver + include ActiveModel::Validations + + attr_reader :token_type + + validates :token_type, inclusion: { in: %i[personal_access_token job_token deploy_token] } + + def initialize(token_type) + @token_type = token_type + validate! + end + + # Existing behavior is known to be inconsistent across authentication + # methods with regards to whether to silently ignore present but invalid + # credentials or to raise an error/respond with 401. + # + # If a token can be located from the provided credentials, but the token + # or credentials are in some way invalid, this implementation opts to + # raise an error. + # + # For example, if the raw credentials include a username and password, and + # a token is resolved from the password, but the username does not match + # the token, an error will be raised. + # + # See https://gitlab.com/gitlab-org/gitlab/-/issues/246569 + + def resolve(raw) + case @token_type + when :personal_access_token + resolve_personal_access_token raw + + when :job_token + resolve_job_token raw + + when :deploy_token + resolve_deploy_token raw + end + end + + private + + def resolve_personal_access_token(raw) + # Check if the password is a personal access token + pat = ::PersonalAccessToken.find_by_token(raw.password) + return unless pat + + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_435907856 + raise ::Gitlab::Auth::UnauthorizedError unless pat.user.username == raw.username + + pat + end + + def resolve_job_token(raw) + # Only look for a job if the username is correct + return if ::Gitlab::Auth::CI_JOB_USER != raw.username + + job = ::Ci::AuthJobFinder.new(token: raw.password).execute + + # Actively reject credentials with the username `gitlab-ci-token` if + # the password is not a valid job token. This replicates existing + # behavior of #find_user_from_job_token. + raise ::Gitlab::Auth::UnauthorizedError unless job + + job + end + + def resolve_deploy_token(raw) + # Check if the password is a deploy token + token = ::DeployToken.active.find_by_token(raw.password) + return unless token + + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #deploy_token_from_request. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_474826205 + raise ::Gitlab::Auth::UnauthorizedError unless token.username == raw.username + + token + end + end + end +end diff --git a/lib/gitlab/api_authentication/token_type_builder.rb b/lib/gitlab/api_authentication/token_type_builder.rb new file mode 100644 index 00000000000..4a57cdc2742 --- /dev/null +++ b/lib/gitlab/api_authentication/token_type_builder.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See Gitlab::Auth::AuthBuilder +module Gitlab + module APIAuthentication + class TokenTypeBuilder + def initialize(strategies) + @strategies = strategies + end + + def token_types(*resolvers) + ::Gitlab::APIAuthentication::SentThroughBuilder.new(@strategies, resolvers) + end + + alias_method :token_type, :token_types + end + end +end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 84fe3d1c959..cefe983848c 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -12,6 +12,7 @@ module Gitlab Attribute.new(:namespace, Namespace), Attribute.new(:user, User), Attribute.new(:caller_id, String), + Attribute.new(:remote_ip, String), Attribute.new(:related_class, String), Attribute.new(:feature_category, String) ].freeze @@ -45,6 +46,7 @@ module Gitlab hash[:project] = -> { project_path } if set_values.include?(:project) hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:caller_id] = caller_id if set_values.include?(:caller_id) + hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) hash[:related_class] = related_class if set_values.include?(:related_class) hash[:feature_category] = feature_category if set_values.include?(:feature_category) end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index caa881eeeab..4c6254c9e69 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -92,10 +92,10 @@ module Gitlab # We only allow Private Access Tokens with `api` scope to be used by web # requests on RSS feeds or ICS files for backwards compatibility. # It is also used by GraphQL/API requests. - def find_user_from_web_access_token(request_format) + def find_user_from_web_access_token(request_format, scopes: [:api]) return unless access_token && valid_web_access_format?(request_format) - validate_access_token!(scopes: [:api]) + validate_access_token!(scopes: scopes) ::PersonalAccessTokens::LastUsedService.new(access_token).execute @@ -194,11 +194,15 @@ module Gitlab def access_token strong_memoize(:access_token) do - # The token can be a PAT or an OAuth (doorkeeper) token - # It is also possible that a PAT is encapsulated in a `Bearer` OAuth token - # (e.g. NPM client registry auth), this case will be properly handled - # by find_personal_access_token - find_oauth_access_token || find_personal_access_token + if try(:namespace_inheritable, :authentication) + access_token_from_namespace_inheritable + else + # The token can be a PAT or an OAuth (doorkeeper) token + # It is also possible that a PAT is encapsulated in a `Bearer` OAuth token + # (e.g. NPM client registry auth), this case will be properly handled + # by find_personal_access_token + find_oauth_access_token || find_personal_access_token + end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index f5931a1d5eb..97e4f921228 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -28,7 +28,7 @@ module Gitlab end def self.servers - Gitlab.config.ldap['servers']&.values || [] + Gitlab.config.ldap.servers&.values || [] end def self.available_servers @@ -42,9 +42,18 @@ module Gitlab end def self.providers - servers.map { |server| server['provider_name'] } + provider_names_from_servers(servers) end + def self.available_providers + provider_names_from_servers(available_servers) + end + + def self.provider_names_from_servers(servers) + servers&.map { |server| server['provider_name'] } || [] + end + private_class_method :provider_names_from_servers + def self.valid_provider?(provider) providers.include?(provider) end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index d28ee54cfbc..504265a83ef 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -30,7 +30,7 @@ module Gitlab end def find_sessionless_user(request_format) - find_user_from_web_access_token(request_format) || + find_user_from_web_access_token(request_format, scopes: [:api, :read_api]) || find_user_from_feed_token(request_format) || find_user_from_static_object_token(request_format) || find_user_from_basic_auth_job || diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb new file mode 100644 index 00000000000..0a8c203421b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill expire_at for a range of Ci::JobArtifact + class BackfillArtifactExpiryDate + include Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 1_000 + DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze + OLD_ARTIFACT_AGE = 15.months + OLD_ARTIFACT_EXPIRY_OFFSET = 3.months + RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year + + # Ci::JobArtifact model + class Ci::JobArtifact < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'ci_job_artifacts' + + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) } + scope :without_expiry_date, -> { where(expire_at: nil) } + scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) } + scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) } + end + + def perform(start_id, end_id) + Ci::JobArtifact.between(start_id, end_id) + .without_expiry_date.before_default_expiration_switch + .each_batch(of: BATCH_SIZE) do |batch| + batch.old.update_all(expire_at: old_artifact_expiry_date) + batch.recent.update_all(expire_at: recent_artifact_expiry_date) + end + end + + private + + def offset_date + strong_memoize(:offset_date) do + current_date = Time.current + target_date = Time.zone.local(current_date.year, current_date.month, 22, 0, 0, 0) + + current_date.day < 22 ? target_date : target_date.next_month + end + end + + def old_artifact_expiry_date + offset_date + OLD_ARTIFACT_EXPIRY_OFFSET + end + + def recent_artifact_expiry_date + offset_date + RECENT_ARTIFACT_EXPIRY_OFFSET + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb new file mode 100644 index 00000000000..16c0de39a3b --- /dev/null +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration that extends CopyColumn to update the value of a + # column using the value of another column in the same table. + # + # - The {start_id, end_id} arguments are at the start so that it can be used + # with `queue_background_migration_jobs_by_range_at_intervals` + # - Provides support for background job tracking through the use of + # Gitlab::Database::BackgroundMigrationJob + # - Uses sub-batching so that we can keep each update's execution time at + # low 100s ms, while being able to update more records per 2 minutes + # that we allow background migration jobs to be scheduled one after the other + # - We skip the NULL checks as they may result in not using an index scan + # - The table that is migrated does _not_ need `id` as the primary key + # We use the provided primary_key column to perform the update. + class CopyColumnUsingBackgroundMigrationJob + include Gitlab::Database::DynamicModelHelpers + + PAUSE_SECONDS = 0.1 + + # start_id - The start ID of the range of rows to update. + # end_id - The end ID of the range of rows to update. + # table - The name of the table that contains the columns. + # primary_key - The primary key column of the table. + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + # sub_batch_size - We don't want updates to take more than ~100ms + # This allows us to run multiple smaller batches during + # the minimum 2.minute interval that we can schedule jobs + def perform(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) + quoted_copy_from = connection.quote_column_name(copy_from) + quoted_copy_to = connection.quote_column_name(copy_to) + + parent_batch_relation = relation_scoped_to_range(table, primary_key, start_id, end_id) + + parent_batch_relation.each_batch(column: primary_key, of: sub_batch_size) do |sub_batch| + sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + + sleep(PAUSE_SECONDS) + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) + end + + private + + def connection + ActiveRecord::Base.connection + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table).where(source_key_column => start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb new file mode 100644 index 00000000000..52b09e07fd5 --- /dev/null +++ b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates the `finding_uuid` attribute for + # the existing `vulnerability_feedback` records. + class PopulateFindingUuidForVulnerabilityFeedback + REPORT_TYPES = { + sast: 0, + dependency_scanning: 1, + container_scanning: 2, + dast: 3, + secret_detection: 4, + coverage_fuzzing: 5, + api_fuzzing: 6 + }.freeze + + class VulnerabilityFeedback < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerability_feedback' + + enum category: REPORT_TYPES + + scope :in_range, -> (start, stop) { where(id: start..stop) } + scope :without_uuid, -> { where(finding_uuid: nil) } + + def self.load_vulnerability_findings + all.to_a.tap { |collection| collection.each(&:vulnerability_finding) } + end + + def set_finding_uuid + return unless vulnerability_finding.present? && vulnerability_finding.primary_identifier.present? + + update_column(:finding_uuid, calculated_uuid) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + end + + def vulnerability_finding + BatchLoader.for(finding_key).batch(replace_methods: false) do |finding_keys, loader| + project_ids = finding_keys.map { |key| key[:project_id] } + categories = finding_keys.map { |key| key[:category] } + fingerprints = finding_keys.map { |key| key[:project_fingerprint] } + + findings = Finding.with_primary_identifier.where( + project_id: project_ids.uniq, + report_type: categories.uniq, + project_fingerprint: fingerprints.uniq + ).to_a + + finding_keys.each do |finding_key| + loader.call( + finding_key, + findings.find { |f| finding_key == f.finding_key } + ) + end + end + end + + private + + def calculated_uuid + Gitlab::UUID.v5(uuid_components) + end + + def uuid_components + [ + category, + vulnerability_finding.primary_identifier.fingerprint, + vulnerability_finding.location_fingerprint, + project_id + ].join('-') + end + + def finding_key + { + project_id: project_id, + category: category, + project_fingerprint: project_fingerprint + } + end + end + + class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation + include ShaAttribute + + self.table_name = 'vulnerability_occurrences' + + sha_attribute :project_fingerprint + sha_attribute :location_fingerprint + + belongs_to :primary_identifier, class_name: 'Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback::Identifier' + + enum report_type: REPORT_TYPES + + scope :with_primary_identifier, -> { includes(:primary_identifier) } + + def finding_key + { + project_id: project_id, + category: report_type, + project_fingerprint: project_fingerprint + } + end + end + + class Identifier < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'vulnerability_identifiers' + end + + def perform(*range) + feedback = VulnerabilityFeedback.without_uuid.in_range(*range).load_vulnerability_findings + feedback.each(&:set_finding_uuid) + + log_info(feedback.count) + end + + def log_info(feedback_count) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: '`finding_uuid` attributes has been set', + count: feedback_count + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_duplicate_services.rb b/lib/gitlab/background_migration/remove_duplicate_services.rb new file mode 100644 index 00000000000..59fb9143a72 --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_services.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Remove duplicated service records with the same project and type. + # These were created in the past for unknown reasons, and should be blocked + # now by the uniqueness validation in the Service model. + class RemoveDuplicateServices + # See app/models/service + class Service < ActiveRecord::Base + include EachBatch + + self.table_name = 'services' + self.inheritance_column = :_type_disabled + + scope :project_ids_with_duplicates, -> do + select(:project_id) + .distinct + .where.not(project_id: nil) + .group(:project_id, :type) + .having('count(*) > 1') + end + + scope :types_with_duplicates, -> (project_ids) do + select(:project_id, :type) + .where(project_id: project_ids) + .group(:project_id, :type) + .having('count(*) > 1') + end + end + + def perform(*project_ids) + types_with_duplicates = Service.types_with_duplicates(project_ids).pluck(:project_id, :type) + + types_with_duplicates.each do |project_id, type| + remove_duplicates(project_id, type) + end + end + + private + + def remove_duplicates(project_id, type) + scope = Service.where(project_id: project_id, type: type) + + # Build a subquery to determine which service record is actually in use, + # by querying for it without specifying an order. + # + # This should match the record returned by `Project#find_service`, + # and the `has_one` service associations on `Project`. + correct_service = scope.select(:id).limit(1) + + # Delete all other services with the same `project_id` and `type` + duplicate_services = scope.where.not(id: correct_service) + duplicate_services.delete_all + end + end + end +end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index c0b228dee59..b146fea66b9 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -6,37 +6,20 @@ module Gitlab include Gitlab::Utils::StrongMemoize LOG_MESSAGES = { - validate_file_paths: "Validating diffs' file paths...", - diff_content_check: "Validating diff contents..." + validate_file_paths: "Validating diffs' file paths..." }.freeze def validate! return if deletion? - return unless should_run_diff_validations? + return unless should_run_validations? return if commits.empty? - file_paths = [] - - if ::Feature.enabled?(:diff_check_with_paths_changed_rpc, project, default_enabled: true) - paths = project.repository.find_changed_paths(commits.map(&:sha)) - paths.each do |path| - file_paths.concat([path.path]) - - validate_diff(path) - end - else - process_commits do |commit| - validate_once(commit) do - commit.raw_deltas.each do |diff| - file_paths.concat([diff.new_path, diff.old_path].compact) - - validate_diff(diff) - end - end - end + paths = project.repository.find_changed_paths(commits.map(&:sha)) + paths.each do |path| + validate_path(path) end - validate_file_paths(file_paths.uniq) + validate_file_paths(paths.map(&:path).uniq) end private @@ -47,43 +30,30 @@ module Gitlab end end - def should_run_diff_validations? - validations_for_diff.present? || path_validations.present? + def should_run_validations? + validations_for_path.present? || file_paths_validations.present? end - def validate_diff(diff) - validations_for_diff.each do |validation| - if error = validation.call(diff) + def validate_path(path) + validations_for_path.each do |validation| + if error = validation.call(path) raise ::Gitlab::GitAccess::ForbiddenError, error end end end # Method overwritten in EE to inject custom validations - def validations_for_diff + def validations_for_path [] end - def path_validations + def file_paths_validations validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] end - def process_commits - logger.log_timed(LOG_MESSAGES[:diff_content_check]) do - # n+1: https://gitlab.com/gitlab-org/gitlab/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - logger.check_timeout_reached - - yield(commit) - end - end - end - end - def validate_file_paths(file_paths) logger.log_timed(LOG_MESSAGES[__method__]) do - path_validations.each do |validation| + file_paths_validations.each do |validation| if error = validation.call(file_paths) raise ::Gitlab::GitAccess::ForbiddenError, error end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 071a8ef830f..8ed4dc61920 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -70,6 +70,10 @@ module Gitlab @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end + def included_templates + @context.expandset.filter_map { |i| i[:template] } + end + private def expand_config(config) @@ -98,7 +102,8 @@ module Gitlab project: project, sha: sha || project&.repository&.root_ref_sha, user: user, - parent_pipeline: parent_pipeline) + parent_pipeline: parent_pipeline, + variables: project&.predefined_variables&.to_runner_variables) end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 206dbaea272..6118ff49928 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -27,6 +27,7 @@ module Gitlab with_options allow_nil: true do validates :name, type: String + validates :public, boolean: true validates :untracked, boolean: true validates :paths, array_of_strings: true validates :paths, array_of_strings: { diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index cf6c2961ee7..e0adb1b19c2 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -7,14 +7,15 @@ module Gitlab class Context TimeoutError = Class.new(StandardError) - attr_reader :project, :sha, :user, :parent_pipeline + attr_reader :project, :sha, :user, :parent_pipeline, :variables attr_reader :expandset, :execution_deadline - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline + @variables = variables @expandset = Set.new @execution_deadline = 0 diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index e74f5b33de7..fdb3e1b00f9 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -41,7 +41,8 @@ module Gitlab project: context.project, sha: context.sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index be479741784..114d493381c 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -72,7 +72,8 @@ module Gitlab project: project, sha: sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 90692eafc3f..4d91cfd4c57 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -47,14 +48,14 @@ module Gitlab # convert location if String to canonical form def normalize_location(location) if location.is_a?(String) - normalize_location_string(location) + expanded_location = expand_variables(location) + normalize_location_string(expanded_location) else location.deep_symbolize_keys end end def expand_project_files(location) - return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true) return location unless location[:project] Array.wrap(location[:file]).map do |file| @@ -96,6 +97,33 @@ module Gitlab matching.first end + + def expand_variables(data) + return data unless ::Feature.enabled?(:variables_in_include_section_ci) + + if data.is_a?(String) + expand(data) + else + transform(data) + end + end + + def transform(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand(value.to_s) } + when String + expand(values) + else + values + end + end + end + + def expand(data) + ExpandVariables.expand(data, context.variables) + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index af1df933b36..7956cf14203 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -56,23 +56,19 @@ module Gitlab end def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: false) - end - - def self.seed_block_run_before_workflow_rules_enabled?(project) - ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true) + ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) end def self.ci_pipeline_editor_page_enabled?(project) - ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false) + ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes) + ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) end def self.rules_variables_enabled?(project) - ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false) + ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index fb795152abe..364e67db02b 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -18,9 +18,10 @@ module Gitlab end end - def initialize(project:, current_user:) + def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user + @sha = sha || project.repository.commit.sha end def validate(content, dry_run: false) @@ -51,7 +52,7 @@ module Gitlab content, project: @project, user: @current_user, - sha: @project.repository.commit.sha + sha: @sha ).execute Result.new( @@ -99,7 +100,8 @@ module Gitlab except: job[:except], environment: job[:environment], when: job[:when], - allow_failure: job[:allow_failure] + allow_failure: job[:allow_failure], + needs: job.dig(:needs_attributes) } end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 57f73c265b2..985639982aa 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -15,8 +15,8 @@ module Gitlab } end - def self.fabricate!(file_type) - parsers.fetch(file_type.to_sym).new + def self.fabricate!(file_type, *args) + parsers.fetch(file_type.to_sym).new(*args) rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index 1edcbac2f25..eb3adf713d4 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -36,7 +36,7 @@ module Gitlab end def parse_node(key, value, coverage_report, context) - if key == 'sources' && value['source'].present? + if key == 'sources' && value && value['source'].present? parse_sources(value['source'], context) elsif key == 'package' Array.wrap(value).each do |item| diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 9662209f88e..f0548284001 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -5,6 +5,9 @@ module Gitlab module Pipeline module Chain class Build < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + def perform! @pipeline.assign_attributes( source: @command.source, @@ -20,12 +23,34 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - variables_attributes: Array(@command.variables_attributes) + locked: @command.project.latest_pipeline_locked, + variables_attributes: variables_attributes ) end def break? - false + @pipeline.errors.any? + end + + private + + def variables_attributes + variables = Array(@command.variables_attributes) + + # We allow parent pipelines to pass variables to child pipelines since + # these variables are coming from internal configurations. We will check + # permissions to :set_pipeline_variables when those are injected upstream, + # to the parent pipeline. + # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) + # the variables are provided from the outside and those should be guarded. + return variables if @command.creates_child_pipeline? + + if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) + error("Insufficient permissions to set pipeline variables") + variables = [] + end + + variables end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d05be54267c..815fe6bac6d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -79,6 +79,10 @@ module Gitlab bridge&.parent_pipeline end + def creates_child_pipeline? + bridge&.triggers_child_pipeline? + end + def metrics @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 083f0bec1df..7b537125b9b 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -19,13 +19,6 @@ module Gitlab # Build to prevent erroring out on ambiguous refs. pipeline.protected = @command.protected_ref? - unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - end - ## # Gather all runtime build/stage errors # diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb index f8e62949bea..67424635603 100644 --- a/lib/gitlab/ci/pipeline/chain/seed_block.rb +++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb @@ -9,8 +9,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - ## # Populate pipeline with block argument of CreatePipelineService#execute. # @@ -20,8 +18,6 @@ module Gitlab end def break? - return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - pipeline.errors.any? end end diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb new file mode 100644 index 00000000000..c1a7b4ed453 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class TemplateUsage < Chain::Base + def perform! + included_templates.each do |template| + track_event(template) + end + end + + def break? + false + end + + private + + def track_event(template) + Gitlab::UsageDataCounters::CiTemplateUniqueCounter + .track_unique_project_event(project_id: pipeline.project_id, template: template) + end + + def included_templates + command.yaml_processor_result.included_templates + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 8f1e690c081..e68d9020a21 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -19,7 +19,7 @@ module Gitlab end unless allowed_to_write_ref? - error("Insufficient permissions for protected ref '#{command.ref}'") + error("You do not have sufficient permission to run a pipeline on '#{command.ref}'. Please select a different branch or contact your administrator for assistance. <a href=https://docs.gitlab.com/ee/ci/pipelines/#pipeline-security-on-protected-branches>Learn more</a>".html_safe) end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 2271915a72b..fe3c2bca551 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -134,7 +134,7 @@ module Gitlab stage.seeds_names.include?(need[:name]) end - "#{name}: needs '#{need[:name]}'" unless result + "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result end.compact end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index beceac5423a..c024e794ad5 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -12,8 +12,6 @@ module Gitlab end def load! - return unless Feature.enabled?(:test_failure_history, project) - recent_failures_count.each do |key_hash, count| failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) end diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb index ee785856fdd..37e2b7320e2 100644 --- a/lib/gitlab/ci/status/group/factory.rb +++ b/lib/gitlab/ci/status/group/factory.rb @@ -8,6 +8,10 @@ module Gitlab def self.common_helpers Status::Group::Common end + + def self.extended_statuses + [[Status::SuccessWarning]] + end end end end diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml new file mode 100644 index 00000000000..7182b96594d --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml @@ -0,0 +1,52 @@ +# +# You can use artifacts to pass data to jobs in later stages. +# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job might build an important file, and pass it to later jobs." + - echo "This is the content of the important file" > important-file.txt + artifacts: + paths: + - important-file.txt + +test-job-with-artifacts: + stage: test + script: + - echo "This job uses the artifact from the job in the earlier stage." + - cat important-file.txt + - echo "It creates another file, and adds it to the artifacts." + - echo "This is a second important file" > important-file2.txt + artifacts: + paths: + - important-file2.txt + +test-job-with-no-artifacts: + stage: test + dependencies: [] # Use to skip downloading any artifacts + script: + - echo "This job does not get the artifacts from other jobs." + - cat important-file.txt || exit 0 + +deploy-job-with-all-artifacts: + stage: deploy + script: + - echo "By default, jobs download all available artifacts." + - cat important-file.txt + - cat important-file2.txt + +deploy-job-with-1-artifact: + stage: deploy + dependencies: + - build-job # Download artifacts from only this job + script: + - echo "You can configure a job to download artifacts from only certain jobs." + - cat important-file.txt + - cat important-file2.txt || exit 0 diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml new file mode 100644 index 00000000000..382bac09ed7 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml @@ -0,0 +1,36 @@ +# +# You can define common tasks and run them before or after the main scripts in jobs. +# For more information, see: +# - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script +# - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script +# + +stages: + - test + +default: + before_script: + - echo "This script runs before the main script in every job, unless the job overrides it." + - echo "It may set up common dependencies, for example." + after_script: + - echo "This script runs after the main script in every job, unless the job overrides it." + - echo "It may do some common final clean up tasks" + +job-standard: + stage: test + script: + - echo "This job uses both of the globally defined before and after scripts." + +job-override-before: + stage: test + before_script: + - echo "Use a different before_script in this job." + script: + - echo "This job uses its own before_script, and the global after_script." + +job-override-after: + stage: test + after_script: + - echo "Use a different after_script in this job." + script: + - echo "This job uses its own after_script, and the global before_script." diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml new file mode 100644 index 00000000000..5f27def74c9 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml @@ -0,0 +1,53 @@ +# +# A manual job is a type of job that is not executed automatically and must be explicitly started by a user. +# To make a job manual, add when: manual to its configuration. +# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job is not a manual job" + +manual-build: + stage: build + script: + - echo "This manual job passes after you trigger it." + when: manual + +manual-build-allowed-to-fail: + stage: build + script: + - echo "This manual job fails after you trigger it." + - echo "It is allowed to fail, so the pipeline does not fail. + when: manual + allow_failure: true # Default behavior + +test-job: + stage: test + script: + - echo "This is a normal test job" + - echo "It runs when the when the build stage completes." + - echo "It does not need to wait for the manual jobs in the build stage to run." + +manual-test-not-allowed-to-fail: + stage: test + script: + - echo "This manual job fails after you trigger it." + - echo "It is NOT allowed to fail, so the pipeline is marked as failed + - echo "when this job completes." + - exit 1 + when: manual + allow_failure: false # Optional behavior + +deploy-job: + stage: deploy + script: + - echo "This is a normal deploy job" + - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed, + - echo "this job does not run". diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml new file mode 100644 index 00000000000..aced628aacb --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml @@ -0,0 +1,33 @@ +# +# A pipeline is composed of independent jobs that run scripts, grouped into stages. +# Stages run in sequential order, but jobs within stages run in parallel. +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job runs in the build stage, which runs first." + +test-job1: + stage: test + script: + - echo "This job runs in the test stage." + - echo "It only starts when the job in the build stage completes successfully." + +test-job2: + stage: test + script: + - echo "This job also runs in the test stage." + - echo "This job can run at the same time as test-job2." + +deploy-job: + stage: deploy + script: + - echo "This job runs in the deploy stage." + - echo "It only runs when both jobs in the test stage complete successfully" diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml new file mode 100644 index 00000000000..2b8cf7bab44 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml @@ -0,0 +1,47 @@ +# +# Variables can be used to for more dynamic behavior in jobs and scripts. +# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html +# + +stages: + - test + +variables: + VAR1: "Variable 1 defined globally" + +use-a-variable: + stage: test + script: + - echo "You can use variables in jobs." + - echo "The content of 'VAR1' is = $VAR1" + +override-a-variable: + stage: test + variables: + VAR1: "Variable 1 was overriden in in the job." + script: + - echo "You can override global variables in jobs." + - echo "The content of 'VAR1' is = $VAR1" + +define-a-new-variable: + stage: test + variables: + VAR2: "Variable 2 is new and defined in the job only." + script: + - echo "You can mix global variables with variables defined in jobs." + - echo "The content of 'VAR1' is = $VAR1" + - echo "The content of 'VAR2' is = $VAR2" + +incorrect-variable-usage: + stage: test + script: + - echo "You can't use variables only defined in other jobs." + - echo "The content of 'VAR2' is = $VAR2" + +predefined-variables: + stage: test + script: + - echo "Some variables are predefined by GitLab CI/CD, for example:" + - echo "The commit author's username is $GITLAB_USER_LOGIN" + - echo "The commit branch is $CI_COMMIT_BRANCH" + - echo "The project path is $CI_PROJECT_PATH" diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml new file mode 100644 index 00000000000..c06ef83c180 --- /dev/null +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -0,0 +1,84 @@ +# This template is on early stage of development. +# Use it with caution. For usage instruction please read +# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md + +include: + # workflow rules to prevent duplicate detached pipelines + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + # auto devops build + - template: 'Jobs/Build.gitlab-ci.yml' + +stages: + - build + - test + - provision + - deploy + - destroy + +variables: + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_COMMIT_REF_SLUG} + TF_VAR_ENVIRONMENT_NAME: ${CI_PROJECT_PATH_SLUG}_${CI_PROJECT_ID}_${CI_COMMIT_REF_SLUG} + TF_VAR_SERVICE_DESK_EMAIL: incoming+${CI_PROJECT_PATH_SLUG}-${CI_PROJECT_ID}-issue-@incoming.gitlab.com + TF_VAR_SHORT_ENVIRONMENT_NAME: ${CI_PROJECT_ID}-${CI_COMMIT_REF_SLUG} + TF_VAR_SMTP_FROM: ${SMTP_FROM} + +cache: + paths: + - .terraform + +.needs_aws_vars: + rules: + - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION' + when: on_success + - when: never + +terraform_apply: + stage: provision + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + extends: .needs_aws_vars + resource_group: terraform + before_script: + - cp /*.tf . + - cp /deploy.sh . + script: + - gitlab-terraform init + - gitlab-terraform plan + - gitlab-terraform plan-json + - gitlab-terraform apply + +deploy: + stage: deploy + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + extends: .needs_aws_vars + resource_group: deploy + before_script: + - cp /*.tf . + - cp /deploy.sh . + - cp /conf.nginx . + script: + - ./deploy.sh + artifacts: + reports: + dotenv: deploy.env + environment: + name: $CI_COMMIT_REF_SLUG + url: $DYNAMIC_ENVIRONMENT_URL + on_stop: terraform_destroy + +terraform_destroy: + variables: + GIT_STRATEGY: none + stage: destroy + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + before_script: + - cp /*.tf . + - cp /deploy.sh . + script: + - gitlab-terraform destroy -auto-approve + environment: + name: $CI_COMMIT_REF_SLUG + action: stop + rules: + - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION && $CI_COMMIT_REF_PROTECTED == "false"' + when: manual + - when: never diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml new file mode 100644 index 00000000000..504ece611ca --- /dev/null +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -0,0 +1,29 @@ +code_quality: + stage: test + image: "cirrusci/flutter:1.22.5" + before_script: + - pub global activate dart_code_metrics + - export PATH="$PATH":"$HOME/.pub-cache/bin" + script: + - metrics lib -r codeclimate > gl-code-quality-report.json + artifacts: + reports: + codequality: gl-code-quality-report.json + +test: + stage: test + image: "cirrusci/flutter:1.22.5" + before_script: + - pub global activate junitreport + - export PATH="$PATH":"$HOME/.pub-cache/bin" + script: + - flutter test --machine --coverage | tojunit -o report.xml + - lcov --summary coverage/lcov.info + - genhtml coverage/lcov.info --output=coverage + coverage: '/lines\.*: \d+\.\d+\%/' + artifacts: + name: coverage + paths: + - $CI_PROJECT_DIR/coverage + reports: + junit: report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 2ae9730ec1a..501d8737acd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 23dfeda31cc..192b1509fdc 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.36.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml new file mode 100644 index 00000000000..fc1acd09714 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -0,0 +1,43 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + DAST_VERSION: 1 + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + +dast: + stage: dast + image: + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - if [ -z "$DAST_WEBSITE$DAST_API_SPECIFICATION" ]; then echo "Either DAST_WEBSITE or DAST_API_SPECIFICATION must be set. See https://docs.gitlab.com/ee/user/application_security/dast/#configuration for more details." && exit 1; fi + - /analyze + artifacts: + reports: + dast: gl-dast-report.json + rules: + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && $DAST_WEBSITE == null && + $DAST_API_SPECIFICATION == null + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_COMMIT_BRANCH && + $DAST_WEBSITE + - if: $CI_COMMIT_BRANCH && + $DAST_API_SPECIFICATION diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index f4ee8ebd47e..56c6fbd96bc 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -10,6 +10,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" + SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SCAN_KUBERNETES_MANIFESTS: "false" @@ -44,6 +45,8 @@ bandit-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /bandit/ exists: @@ -58,6 +61,8 @@ brakeman-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: @@ -72,6 +77,8 @@ eslint-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /eslint/ exists: @@ -90,6 +97,8 @@ flawfinder-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ exists: @@ -105,6 +114,8 @@ kubesec-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -118,6 +129,8 @@ gosec-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /gosec/ exists: @@ -136,6 +149,8 @@ mobsf-android-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' @@ -155,6 +170,8 @@ mobsf-ios-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' @@ -170,6 +187,8 @@ nodejs-scan-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: @@ -184,6 +203,8 @@ phpcs-security-audit-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ exists: @@ -198,6 +219,8 @@ pmd-apex-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: @@ -212,6 +235,8 @@ security-code-scan-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ exists: @@ -227,6 +252,8 @@ sobelow-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: @@ -239,6 +266,8 @@ spotbugs-sast: variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 8ca1d2e08ba..d2a6fa06dd8 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -37,6 +37,7 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: + - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 377c72e8031..7e2828d010f 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -17,6 +17,7 @@ variables: cache: paths: - .terraform + - .terraform.lock.hcl before_script: - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 910e711f046..c2db0fc44f1 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -19,6 +19,7 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ + - ${TF_ROOT}/.terraform.lock.hcl .init: &init stage: init diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb new file mode 100644 index 00000000000..6abc6a5644f --- /dev/null +++ b/lib/gitlab/ci/variables/collection/sorted.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Collection + class Sorted + include TSort + include Gitlab::Utils::StrongMemoize + + def initialize(variables) + @variables = variables + end + + def valid? + errors.nil? + end + + # errors sorts an array of variables, ignoring unknown variable references, + # and returning an error string if a circular variable reference is found + def errors + return if Feature.disabled?(:variable_inside_variable) + + strong_memoize(:errors) do + # Check for cyclic dependencies and build error message in that case + errors = each_strongly_connected_component.filter_map do |component| + component.map { |v| v[:key] }.inspect if component.size > 1 + end + + "circular variable reference detected: #{errors.join(', ')}" if errors.any? + end + end + + # sort sorts an array of variables, ignoring unknown variable references. + # If a circular variable reference is found, the original array is returned + def sort + return @variables if Feature.disabled?(:variable_inside_variable) + return @variables if errors + + tsort + end + + private + + def tsort_each_node(&block) + @variables.each(&block) + end + + def tsort_each_child(variable, &block) + each_variable_reference(variable[:value], &block) + end + + def input_vars + strong_memoize(:input_vars) do + @variables.index_by { |env| env.fetch(:key) } + end + end + + def walk_references(value) + return unless ExpandVariables.possible_var_reference?(value) + + value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref| + yield(input_vars, var_ref.first) + end + end + + def each_variable_reference(value) + walk_references(value) do |vars_hash, ref_var_name| + variable = vars_hash.dig(ref_var_name) + yield variable if variable + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index ee55eb8b22a..dc4951f76bb 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -10,12 +10,6 @@ module Gitlab class YamlProcessor ValidationError = Class.new(StandardError) - def self.validation_message(content, opts = {}) - result = new(content, opts).execute - - result.errors.first - end - def initialize(config_content, opts = {}) @config_content = config_content @opts = opts diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index cd7d781a574..86749cda9c7 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -53,6 +53,10 @@ module Gitlab @stages ||= @ci_config.stages end + def included_templates + @included_templates ||= @ci_config.included_templates + end + def build_attributes(name) job = jobs.fetch(name.to_sym, {}) diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb new file mode 100644 index 00000000000..de9a17a453f --- /dev/null +++ b/lib/gitlab/composer/version_index.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Composer + class VersionIndex + include API::Helpers::RelatedResourcesHelpers + + def initialize(packages) + @packages = packages + end + + def as_json(_options = nil) + { 'packages' => { @packages.first.name => package_versions_map } } + end + + def sha + Digest::SHA256.hexdigest(to_json) + end + + private + + def package_versions_map + @packages.each_with_object({}) do |package, map| + map[package.version] = package_metadata(package) + end + end + + def package_metadata(package) + json = package.composer_metadatum.composer_json + + json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version) + end + + def package_dist(package) + sha = package.composer_metadatum.target_sha + archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true) + + { + 'type' => 'zip', + 'url' => expose_url(archive_api_path) + "?sha=#{sha}", + 'reference' => sha, + 'shasum' => '' + } + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 4d7590a8e38..fbf021345ca 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -9,9 +9,13 @@ module Gitlab CONTEXT_LINES = 3 + CONFLICT_MARKER_OUR = 'conflict_marker_our' + CONFLICT_MARKER_THEIR = 'conflict_marker_their' + CONFLICT_MARKER_SEPARATOR = 'conflict_marker' + CONFLICT_TYPES = { - "old" => "conflict_marker_their", - "new" => "conflict_marker_our" + "old" => "conflict_their", + "new" => "conflict_our" }.freeze attr_reader :merge_request @@ -59,18 +63,25 @@ module Gitlab if section[:conflict] lines = [] - initial_type = nil + lines << create_separator_line(section[:lines].first, CONFLICT_MARKER_OUR) + + current_type = section[:lines].first.type section[:lines].each do |line| - if line.type != initial_type - lines << create_separator_line(line) - initial_type = line.type + if line.type != current_type # insert a separator between our changes and theirs + lines << create_separator_line(line, CONFLICT_MARKER_SEPARATOR) + current_type = line.type end line.type = CONFLICT_TYPES[line.type] + + # Swap the positions around due to conflicts/diffs display inconsistency + # https://gitlab.com/gitlab-org/gitlab/-/issues/291989 + line.old_pos, line.new_pos = line.new_pos, line.old_pos + lines << line end - lines << create_separator_line(lines.last) + lines << create_separator_line(lines.last, CONFLICT_MARKER_THEIR) lines else @@ -156,8 +167,8 @@ module Gitlab Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) end - def create_separator_line(line) - Gitlab::Diff::Line.new('', 'conflict_marker', line.index, nil, nil) + def create_separator_line(line, type) + Gitlab::Diff::Line.new('', type, line.index, nil, nil) end # Any line beginning with a letter, an underscore, or a dollar can be used in a diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb deleted file mode 100644 index 6c6dd90e450..00000000000 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class BaseEventFetcher - include BaseQuery - include GroupProjectsProvider - - attr_reader :projections, :query, :stage, :options - - MAX_EVENTS = 50 - - def initialize(stage:, options:) - @stage = stage - @options = options - end - - def fetch - update_author! - - event_result.map do |event| - serialize(event) if has_permission?(event['id']) - end.compact - end - - def order - @order || default_order - end - - private - - def update_author! - return unless event_result.any? && event_result.first['author_id'] - - Updater.update!(event_result, from: 'author_id', to: 'author', klass: User) - end - - def event_result - @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a - end - - def events_query - diff_fn = subtract_datetimes_diff(base_query, options[:start_time_attrs], options[:end_time_attrs]) - - base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS) - end - - def default_order - [options[:start_time_attrs]].flatten.first - end - - def serialize(_event) - raise NotImplementedError.new("Expected #{self.name} to implement serialize(event)") - end - - def has_permission?(id) - allowed_ids.nil? || allowed_ids.include?(id.to_i) - end - - def allowed_ids - @allowed_ids ||= allowed_ids_finder_class - .new(options[:current_user], allowed_ids_source) - .execute.where(id: event_result_ids).pluck(:id) - end - - def event_result_ids - event_result.map { |event| event['id'] } - end - - def allowed_ids_source - group ? { group_id: group.id, include_subgroups: true } : { project_id: project.id } - end - - def serialization_context - {} - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb deleted file mode 100644 index 6aedbf64f26..00000000000 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module BaseQuery - include MetricsTables - include Gitlab::Database::Median - include Gitlab::Database::DateTime - - private - - def base_query - @base_query ||= stage_query(projects.map(&:id)) - end - - def stage_query(project_ids) - query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) - .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) - .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) - .project(issue_table[:project_id].as("project_id")) - .project(projects_table[:path].as("project_path")) - .project(routes_table[:path].as("namespace_path")) - - query = limit_query(query, project_ids) - query = limit_query_by_date_range(query) - - # Load merge_requests - - query = load_merge_requests(query) - - query - end - - def limit_query(query, project_ids) - query.where(issue_table[:project_id].in(project_ids)) - .where(routes_table[:source_type].eq('Namespace')) - end - - def limit_query_by_date_range(query) - query = query.where(issue_table[:created_at].gteq(options[:from])) - query = query.where(issue_table[:created_at].lteq(options[:to])) if options[:to] - query - end - - def load_merge_requests(query) - query.join(mr_table, Arel::Nodes::OuterJoin) - .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) - .join(mr_metrics_table) - .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb deleted file mode 100644 index 06f0cbed147..00000000000 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class BaseStage - include BaseQuery - include GroupProjectsProvider - - attr_reader :options - - def initialize(options:) - @options = options - end - - def events - event_fetcher.fetch - end - - def as_json(serializer: AnalyticsStageSerializer) - serializer.new.represent(self) - end - - def title - raise NotImplementedError.new("Expected #{self.name} to implement title") - end - - def project_median - return if project.nil? - - BatchLoader.for(project.id).batch(key: name) do |project_ids, loader| - if project_ids.one? - loader.call(project.id, median_query(project_ids)) - else - begin - median_datetimes(cte_table, interval_query(project_ids), name, :project_id)&.each do |project_id, median| - loader.call(project_id, median) - end - rescue NotSupportedError - {} - end - end - end - end - - def group_median - median_query(projects.map(&:id)) - end - - def median_query(project_ids) - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # value stream analytics stage. - - median_datetime(cte_table, interval_query(project_ids), name) - end - - def name - raise NotImplementedError.new("Expected #{self.name} to implement name") - end - - def cte_table - Arel::Table.new("cte_table_for_#{name}") - end - - def interval_query(project_ids) - Arel::Nodes::As.new(cte_table, - subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) - end - - private - - def event_fetcher - @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(stage: name, - options: event_options) - end - - def event_options - options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/builds_event_helper.rb b/lib/gitlab/cycle_analytics/builds_event_helper.rb deleted file mode 100644 index c39d41578e9..00000000000 --- a/lib/gitlab/cycle_analytics/builds_event_helper.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module BuildsEventHelper - def initialize(...) - @projections = [build_table[:id]] - @order = build_table[:created_at] - - super(...) - end - - def fetch - Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) - - super - end - - def events_query - base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - - super - end - - private - - def allowed_ids - nil - end - - def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb deleted file mode 100644 index 790bf32c6c7..00000000000 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class CodeEventFetcher < BaseEventFetcher - include CodeHelper - - def initialize(...) - @projections = [mr_table[:title], - mr_table[:iid], - mr_table[:id], - mr_table[:created_at], - mr_table[:state_id], - mr_table[:author_id]] - @order = mr_table[:created_at] - - super(...) - end - - private - - def serialize(event) - AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - MergeRequestsFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/code_helper.rb b/lib/gitlab/cycle_analytics/code_helper.rb deleted file mode 100644 index 8f28bdd2502..00000000000 --- a/lib/gitlab/cycle_analytics/code_helper.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module CodeHelper - def stage_query(project_ids) - super(project_ids).where(mr_table[:created_at].gteq(issue_metrics_table[:first_mentioned_in_commit_at])) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb deleted file mode 100644 index 89a6430221c..00000000000 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class CodeStage < BaseStage - include CodeHelper - - def start_time_attrs - @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_table[:created_at] - end - - def name - :code - end - - def title - s_('CycleAnalyticsStage|Code') - end - - def legend - _("Related Merge Requests") - end - - def description - _("Time until first merge request") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb deleted file mode 100644 index 04f4b4f053f..00000000000 --- a/lib/gitlab/cycle_analytics/event_fetcher.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module EventFetcher - def self.[](stage_name) - CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher", false) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb deleted file mode 100644 index fd04ec090b3..00000000000 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class IssueEventFetcher < BaseEventFetcher - include IssueHelper - - def initialize(...) - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id]] - - super(...) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - IssuesFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb deleted file mode 100644 index f6f85b84ed8..00000000000 --- a/lib/gitlab/cycle_analytics/issue_helper.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module IssueHelper - def stage_query(project_ids) - query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) - .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) - .project(issue_table[:project_id].as("project_id")) - .project(projects_table[:path].as("project_path")) - .project(routes_table[:path].as("namespace_path")) - - query = limit_query(query, project_ids) - limit_query_by_date_range(query) - end - - def limit_query(query, project_ids) - query.where(issue_table[:project_id].in(project_ids)) - .where(routes_table[:source_type].eq('Namespace')) - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb deleted file mode 100644 index 738cb3eba03..00000000000 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class IssueStage < BaseStage - include IssueHelper - - def start_time_attrs - @start_time_attrs ||= issue_table[:created_at] - end - - def end_time_attrs - @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at]] - end - - def name - :issue - end - - def title - s_('CycleAnalyticsStage|Issue') - end - - def legend - _("Related Issues") - end - - def description - _("Time before an issue gets scheduled") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index 0e094fabb01..9164c8b1bff 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -23,7 +23,7 @@ module Gitlab end def get - ::CycleAnalytics::LevelBase::STAGES.each do |stage| + Gitlab::Analytics::CycleAnalytics::DefaultStages.symbolized_stage_names.each do |stage| @stage_permission_hash[stage] = authorized_stage?(stage) end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb deleted file mode 100644 index 4d98d589e46..00000000000 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class PlanEventFetcher < BaseEventFetcher - include PlanHelper - - def initialize(...) - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id]] - - super(...) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - IssuesFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb deleted file mode 100644 index af4bf6ed3eb..00000000000 --- a/lib/gitlab/cycle_analytics/plan_helper.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module PlanHelper - def stage_query(project_ids) - query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) - .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) - .project(issue_table[:project_id].as("project_id")) - .project(projects_table[:path].as("project_path")) - .project(routes_table[:path].as("namespace_path")) - .where(issue_table[:project_id].in(project_ids)) - .where(routes_table[:source_type].eq('Namespace')) - query = limit_query(query) - - limit_query_by_date_range(query) - end - - def limit_query(query) - query.where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb deleted file mode 100644 index 0b27d114f52..00000000000 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class PlanStage < BaseStage - include PlanHelper - - def start_time_attrs - @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at]] - end - - def end_time_attrs - @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] - end - - def name - :plan - end - - def title - s_('CycleAnalyticsStage|Plan') - end - - def legend - _("Related Issues") - end - - def description - _("Time before an issue starts implementation") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb deleted file mode 100644 index 5fa286bd3df..00000000000 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ProductionEventFetcher < BaseEventFetcher - include ProductionHelper - - def initialize(...) - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id], - routes_table[:path]] - - super(...) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - IssuesFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb deleted file mode 100644 index 778757a9ede..00000000000 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module ProductionHelper - def stage_query(project_ids) - super(project_ids) - .where(mr_metrics_table[:first_deployed_to_production_at] - .gteq(options[:from])) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb deleted file mode 100644 index 0b7d160c7de..00000000000 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ReviewEventFetcher < BaseEventFetcher - include ReviewHelper - - def initialize(...) - @projections = [mr_table[:title], - mr_table[:iid], - mr_table[:id], - mr_table[:created_at], - mr_table[:state_id], - mr_table[:author_id]] - - super(...) - end - - private - - def serialize(event) - AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - MergeRequestsFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/review_helper.rb b/lib/gitlab/cycle_analytics/review_helper.rb deleted file mode 100644 index c53249652b5..00000000000 --- a/lib/gitlab/cycle_analytics/review_helper.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module ReviewHelper - def stage_query(project_ids) - super(project_ids).where(mr_metrics_table[:merged_at].not_eq(nil)) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb deleted file mode 100644 index e9df8cd5a05..00000000000 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ReviewStage < BaseStage - include ReviewHelper - - def start_time_attrs - @start_time_attrs ||= mr_table[:created_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:merged_at] - end - - def name - :review - end - - def title - s_('CycleAnalyticsStage|Review') - end - - def legend - _("Related Merged Requests") - end - - def description - _("Time between merge request creation and merge/close") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb deleted file mode 100644 index 5cfd9ea4730..00000000000 --- a/lib/gitlab/cycle_analytics/stage.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module Stage - def self.[](stage_name) - CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage", false) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb deleted file mode 100644 index 1454a1a33eb..00000000000 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class StagingEventFetcher < BaseEventFetcher - include ProductionHelper - include BuildsEventHelper - end - end -end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb deleted file mode 100644 index e03627c6cd1..00000000000 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class StagingStage < BaseStage - include ProductionHelper - - def start_time_attrs - @start_time_attrs ||= mr_metrics_table[:merged_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] - end - - def name - :staging - end - - def title - s_('CycleAnalyticsStage|Staging') - end - - def legend - _("Related Deployed Jobs") - end - - def description - _("From merge request merge until deploy to production") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb deleted file mode 100644 index 2fa44b1b364..00000000000 --- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class TestEventFetcher < BaseEventFetcher - include TestHelper - include BuildsEventHelper - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_helper.rb b/lib/gitlab/cycle_analytics/test_helper.rb deleted file mode 100644 index d9124d62c7c..00000000000 --- a/lib/gitlab/cycle_analytics/test_helper.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module TestHelper - def stage_query(project_ids) - if branch - super(project_ids).where(build_table[:ref].eq(branch)) - else - super(project_ids) - end - end - - private - - def branch - @branch ||= options[:branch] - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb deleted file mode 100644 index 4787a906c07..00000000000 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class TestStage < BaseStage - include TestHelper - - def start_time_attrs - @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at] - end - - def name - :test - end - - def title - s_('CycleAnalyticsStage|Test') - end - - def legend - _("Related Jobs") - end - - def description - _("Total test time for all commits/merges") - end - end - end -end diff --git a/lib/gitlab/danger/base_linter.rb b/lib/gitlab/danger/base_linter.rb index df2e9e745aa..898434724bd 100644 --- a/lib/gitlab/danger/base_linter.rb +++ b/lib/gitlab/danger/base_linter.rb @@ -1,11 +1,12 @@ # frozen_string_literal: true +require_relative 'title_linting' + module Gitlab module Danger class BaseLinter MIN_SUBJECT_WORDS_COUNT = 3 MAX_LINE_LENGTH = 72 - WIP_PREFIX = 'WIP: ' attr_reader :commit, :problems @@ -58,7 +59,7 @@ module Gitlab private def subject - message_parts[0].delete_prefix(WIP_PREFIX) + TitleLinting.remove_draft_flag(message_parts[0]) end def subject_too_short? diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb index 92af6849b2f..4b85775ed98 100644 --- a/lib/gitlab/danger/changelog.rb +++ b/lib/gitlab/danger/changelog.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative 'title_linting' + module Gitlab module Danger module Changelog @@ -75,7 +77,7 @@ module Gitlab end def sanitized_mr_title - helper.sanitize_mr_title(gitlab.mr_json["title"]) + TitleLinting.sanitize_mr_title(gitlab.mr_json["title"]) end def categories_need_changelog? diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 7e2e0fb0acb..e23f5900433 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -1,9 +1,15 @@ # frozen_string_literal: true -require_relative 'base_linter' - emoji_checker_path = File.expand_path('emoji_checker', __dir__) -defined?(Rails) ? require_dependency(emoji_checker_path) : require_relative(emoji_checker_path) +base_linter_path = File.expand_path('base_linter', __dir__) + +if defined?(Rails) + require_dependency(base_linter_path) + require_dependency(emoji_checker_path) +else + require_relative(base_linter_path) + require_relative(emoji_checker_path) +end module Gitlab module Danger diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d22f28ff7f2..09e013e24b8 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true require_relative 'teammate' +require_relative 'title_linting' module Gitlab module Danger module Helper RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze # Returns a list of all files that have been added, modified or renamed. # `git.modified_files` might contain paths that already have been renamed, @@ -128,7 +128,7 @@ module Gitlab }.freeze # First-match win, so be sure to put more specific regex at the top... CATEGORIES = { - [%r{usage_data\.rb}, %r{^(\+|-).*(count|distinct_count)\(.*\)(.*)$}] => [:database, :backend], + [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend], %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs, %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, @@ -216,14 +216,10 @@ module Gitlab usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end - def sanitize_mr_title(title) - title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') - end - def draft_mr? return false unless gitlab_helper - DRAFT_REGEX.match?(gitlab_helper.mr_json['title']) + TitleLinting.has_draft_flag?(gitlab_helper.mr_json['title']) end def security_mr? diff --git a/lib/gitlab/danger/merge_request_linter.rb b/lib/gitlab/danger/merge_request_linter.rb index d401d332aa7..ed354bfc68d 100644 --- a/lib/gitlab/danger/merge_request_linter.rb +++ b/lib/gitlab/danger/merge_request_linter.rb @@ -1,6 +1,12 @@ # frozen_string_literal: true -require_relative 'base_linter' +base_linter_path = File.expand_path('base_linter', __dir__) + +if defined?(Rails) + require_dependency(base_linter_path) +else + require_relative(base_linter_path) +end module Gitlab module Danger diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index 328083f7002..21feda2cf20 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -2,6 +2,8 @@ require_relative 'teammate' require_relative 'request_helper' unless defined?(Gitlab::Danger::RequestHelper) +require_relative 'weightage/reviewers' +require_relative 'weightage/maintainers' module Gitlab module Danger @@ -151,20 +153,14 @@ module Gitlab %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) end - hungry_reviewers = reviewers.select { |member| member.hungry } - hungry_traintainers = traintainers.select { |member| member.hungry } - - # TODO: take CODEOWNERS into account? - # https://gitlab.com/gitlab-org/gitlab/issues/26723 random = new_random(mr_source_branch) - # Make hungry traintainers have 4x the chance to be picked as a reviewer - # Make traintainers have 3x the chance to be picked as a reviewer - # Make hungry reviewers have 2x the chance to be picked as a reviewer - weighted_reviewers = reviewers + hungry_reviewers + traintainers + traintainers + traintainers + hungry_traintainers + weighted_reviewers = Weightage::Reviewers.new(reviewers, traintainers).execute + weighted_maintainers = Weightage::Maintainers.new(maintainers).execute + reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) - maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) + maintainer = spin_for_person(weighted_maintainers, random: random, timezone_experiment: timezone_experiment) Spin.new(category, reviewer, maintainer, false, timezone_experiment) end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 4481977db15..911b84d93ec 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,7 +3,7 @@ module Gitlab module Danger class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :reduced_capacity, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) @@ -15,6 +15,7 @@ module Gitlab @projects = options['projects'] @available = options['available'] @hungry = options['hungry'] + @reduced_capacity = options['reduced_capacity'] @tz_offset_hours = options['tz_offset_hours'] end @@ -94,6 +95,7 @@ module Gitlab when :engineering_productivity return false unless role[/Engineering Productivity/] return true if kind == :reviewer + return true if capabilities(project).include?("#{kind} engineering_productivity") capabilities(project).include?("#{kind} backend") else diff --git a/lib/gitlab/danger/title_linting.rb b/lib/gitlab/danger/title_linting.rb new file mode 100644 index 00000000000..db1ccaaf9a9 --- /dev/null +++ b/lib/gitlab/danger/title_linting.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + module TitleLinting + DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze + + module_function + + def sanitize_mr_title(title) + remove_draft_flag(title).gsub(/`/, '\\\`') + end + + def remove_draft_flag(title) + title.gsub(DRAFT_REGEX, '') + end + + def has_draft_flag?(title) + DRAFT_REGEX.match?(title) + end + end + end +end diff --git a/lib/gitlab/danger/weightage.rb b/lib/gitlab/danger/weightage.rb new file mode 100644 index 00000000000..67fade27573 --- /dev/null +++ b/lib/gitlab/danger/weightage.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + module Weightage + CAPACITY_MULTIPLIER = 2 # change this number to change what it means to be a reduced capacity reviewer 1/this number + BASE_REVIEWER_WEIGHT = 1 + end + end +end diff --git a/lib/gitlab/danger/weightage/maintainers.rb b/lib/gitlab/danger/weightage/maintainers.rb new file mode 100644 index 00000000000..cc0eb370e7a --- /dev/null +++ b/lib/gitlab/danger/weightage/maintainers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require_relative '../weightage' + +module Gitlab + module Danger + module Weightage + class Maintainers + def initialize(maintainers) + @maintainers = maintainers + end + + def execute + maintainers.each_with_object([]) do |maintainer, weighted_maintainers| + add_weighted_reviewer(weighted_maintainers, maintainer, BASE_REVIEWER_WEIGHT) + end + end + + private + + attr_reader :maintainers + + def add_weighted_reviewer(reviewers, reviewer, weight) + if reviewer.reduced_capacity + reviewers.fill(reviewer, reviewers.size, weight) + else + reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) + end + end + end + end + end +end diff --git a/lib/gitlab/danger/weightage/reviewers.rb b/lib/gitlab/danger/weightage/reviewers.rb new file mode 100644 index 00000000000..c8019be716e --- /dev/null +++ b/lib/gitlab/danger/weightage/reviewers.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require_relative '../weightage' + +module Gitlab + module Danger + module Weightage + # Weights after (current multiplier of 2) + # + # +------------------------------+--------------------------------+ + # | reviewer type | weight(times in reviewer pool) | + # +------------------------------+--------------------------------+ + # | reduced capacity reviewer | 1 | + # | reviewer | 2 | + # | hungry reviewer | 4 | + # | reduced capacity traintainer | 3 | + # | traintainer | 6 | + # | hungry traintainer | 8 | + # +------------------------------+--------------------------------+ + # + class Reviewers + DEFAULT_REVIEWER_WEIGHT = CAPACITY_MULTIPLIER * BASE_REVIEWER_WEIGHT + TRAINTAINER_WEIGHT = 3 + + def initialize(reviewers, traintainers) + @reviewers = reviewers + @traintainers = traintainers + end + + def execute + # TODO: take CODEOWNERS into account? + # https://gitlab.com/gitlab-org/gitlab/issues/26723 + + weighted_reviewers + weighted_traintainers + end + + private + + attr_reader :reviewers, :traintainers + + def weighted_reviewers + reviewers.each_with_object([]) do |reviewer, total_reviewers| + add_weighted_reviewer(total_reviewers, reviewer, BASE_REVIEWER_WEIGHT) + end + end + + def weighted_traintainers + traintainers.each_with_object([]) do |reviewer, total_traintainers| + add_weighted_reviewer(total_traintainers, reviewer, TRAINTAINER_WEIGHT) + end + end + + def add_weighted_reviewer(reviewers, reviewer, weight) + if reviewer.reduced_capacity + reviewers.fill(reviewer, reviewers.size, weight) + elsif reviewer.hungry + reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER + DEFAULT_REVIEWER_WEIGHT) + else + reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) + end + end + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb deleted file mode 100644 index 603b125d8b4..00000000000 --- a/lib/gitlab/database/median.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true - -# https://www.periscopedata.com/blog/medians-in-sql.html -module Gitlab - module Database - module Median - NotSupportedError = Class.new(StandardError) - - def median_datetime(arel_table, query_so_far, column_sym) - extract_median(execute_queries(arel_table, query_so_far, column_sym)).presence - end - - def median_datetimes(arel_table, query_so_far, column_sym, partition_column) - extract_medians(execute_queries(arel_table, query_so_far, column_sym, partition_column)).presence - end - - def extract_median(results) - result = results.compact.first - - result = result.first.presence - - result['median']&.to_f if result - end - - def extract_medians(results) - median_values = results.compact.first.values - - median_values.each_with_object({}) do |(id, median), hash| - hash[id.to_i] = median&.to_f - end - end - - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) - # Create a CTE with the column we're operating on, row number (after sorting by the column - # we're operating on), and count of the table we're operating on (duplicated across) all rows - # of the CTE. For example, if we're looking to find the median of the `projects.star_count` - # column, the CTE might look like this: - # - # star_count | row_id | ct - # ------------+--------+---- - # 5 | 1 | 3 - # 9 | 2 | 3 - # 15 | 3 | 3 - # - # If a partition column is used we will do the same operation but for separate partitions, - # when that happens the CTE might look like this: - # - # project_id | star_count | row_id | ct - # ------------+------------+--------+---- - # 1 | 5 | 1 | 2 - # 1 | 9 | 2 | 2 - # 2 | 10 | 1 | 3 - # 2 | 15 | 2 | 3 - # 2 | 20 | 3 | 3 - cte_table = Arel::Table.new("ordered_records") - - cte = Arel::Nodes::As.new( - cte_table, - arel_table.project(*rank_rows(arel_table, column_sym, partition_column)). - # Disallow negative values - where(arel_table[column_sym].gteq(zero_interval))) - - # From the CTE, select either the middle row or the middle two rows (this is accomplished - # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the - # selected rows, and this is the median value. - result = - cte_table - .project(*median_projections(cte_table, column_sym, partition_column)) - .where( - Arel::Nodes::Between.new( - cte_table[:row_id], - Arel::Nodes::And.new( - [(cte_table[:ct] / Arel.sql('2.0')), - (cte_table[:ct] / Arel.sql('2.0') + 1)] - ) - ) - ) - .with(query_so_far, cte) - - result.group(cte_table[partition_column]).order(cte_table[partition_column]) if partition_column - - result.to_sql - end - - private - - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - - Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } - end - - def average(args, as) - Arel::Nodes::NamedFunction.new("AVG", args, as) - end - - def rank_rows(arel_table, column_sym, partition_column) - column_row = arel_table[column_sym].as(column_sym.to_s) - - if partition_column - partition_row = arel_table[partition_column] - row_id = - Arel::Nodes::Over.new( - Arel::Nodes::NamedFunction.new('rank', []), - Arel::Nodes::Window.new.partition(arel_table[partition_column]) - .order(arel_table[column_sym]) - ).as('row_id') - - count = arel_table.from.from(arel_table.alias) - .project('COUNT(*)') - .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) - .as('ct') - - [partition_row, column_row, row_id, count] - else - row_id = - Arel::Nodes::Over.new( - Arel::Nodes::NamedFunction.new('row_number', []), - Arel::Nodes::Window.new.order(arel_table[column_sym]) - ).as('row_id') - - count = arel_table.where(arel_table[column_sym].gteq(zero_interval)).project("COUNT(1)").as('ct') - - [column_row, row_id, count] - end - end - - def median_projections(table, column_sym, partition_column) - projections = [] - projections << table[partition_column] if partition_column - projections << average([extract_epoch(table[column_sym])], "median") - projections - end - - def extract_epoch(arel_attribute) - Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) - end - - def extract_diff_epoch(diff) - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) - end - - # Need to cast '0' to an INTERVAL before we can check if the interval is positive - def zero_interval - Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) - end - end - end -end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 164fce5a5a3..6b169a504f3 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -70,6 +70,61 @@ module Gitlab end end + # + # Creates a new table, optionally allowing the caller to add check constraints to the table. + # Aside from that addition, this method should behave identically to Rails' `create_table` method. + # + # Example: + # + # create_table_with_constraints :some_table do |t| + # t.integer :thing, null: false + # t.text :other_thing + # + # t.check_constraint :thing_is_not_null, 'thing IS NOT NULL' + # t.text_limit :other_thing, 255 + # end + # + # See Rails' `create_table` for more info on the available arguments. + def create_table_with_constraints(table_name, **options, &block) + helper_context = self + check_constraints = [] + + with_lock_retries do + create_table(table_name, **options) do |t| + t.define_singleton_method(:check_constraint) do |name, definition| + helper_context.send(:validate_check_constraint_name!, name) # rubocop:disable GitlabSecurity/PublicSend + + check_constraints << { name: name, definition: definition } + end + + t.define_singleton_method(:text_limit) do |column_name, limit, name: nil| + # rubocop:disable GitlabSecurity/PublicSend + name = helper_context.send(:text_limit_name, table_name, column_name, name: name) + helper_context.send(:validate_check_constraint_name!, name) + # rubocop:enable GitlabSecurity/PublicSend + + column_name = helper_context.quote_column_name(column_name) + definition = "char_length(#{column_name}) <= #{limit}" + + check_constraints << { name: name, definition: definition } + end + + t.instance_eval(&block) unless block.nil? + end + + next if check_constraints.empty? + + constraint_clauses = check_constraints.map do |constraint| + "ADD CONSTRAINT #{quote_table_name(constraint[:name])} CHECK (#{constraint[:definition]})" + end + + execute(<<~SQL) + ALTER TABLE #{quote_table_name(table_name)} + #{constraint_clauses.join(",\n")} + SQL + end + end + # Creates a new index, concurrently # # Example: @@ -858,6 +913,120 @@ module Gitlab end end + # Initializes the conversion of an integer column to bigint + # + # It can be used for converting both a Primary Key and any Foreign Keys + # that may reference it or any other integer column that we may want to + # upgrade (e.g. columns that store IDs, but are not set as FKs). + # + # - For primary keys and Foreign Keys (or other columns) defined as NOT NULL, + # the new bigint column is added with a hardcoded NOT NULL DEFAULT 0 + # which allows us to skip a very costly verification step once we + # are ready to switch it. + # This is crucial for Primary Key conversions, because setting a column + # as the PK converts even check constraints to NOT NULL constraints + # and forces an inline re-verification of the whole table. + # - It backfills the new column with the values of the existing primary key + # by scheduling background jobs. + # - It tracks the scheduled background jobs through the use of + # Gitlab::Database::BackgroundMigrationJob + # which allows a more thorough check that all jobs succeeded in the + # cleanup migration and is way faster for very large tables. + # - It sets up a trigger to keep the two columns in sync + # - It does not schedule a cleanup job: we have to do that with followup + # post deployment migrations in the next release. + # + # This needs to be done manually by using the + # `cleanup_initialize_conversion_of_integer_to_bigint` + # (not yet implemented - check #288005) + # + # table - The name of the database table containing the column + # column - The name of the column that we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + # batch_size - The number of rows to schedule in a single background migration + # sub_batch_size - The smaller batches that will be used by each scheduled job + # to update the table. Useful to keep each update at ~100ms while executing + # more updates per interval (2.minutes) + # Note that each execution of a sub-batch adds a constant 100ms sleep + # time in between the updates, which must be taken into account + # while calculating the batch, sub_batch and interval values. + # interval - The time interval between every background migration + # + # example: + # Assume that we have figured out that updating 200 records of the events + # table takes ~100ms on average. + # We can set the sub_batch_size to 200, leave the interval to the default + # and set the batch_size to 50_000 which will require + # ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space + # between the scheduled jobs + def initialize_conversion_of_integer_to_bigint( + table, + column, + primary_key: :id, + batch_size: 20_000, + sub_batch_size: 1000, + interval: 2.minutes + ) + + if transaction_open? + raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction' + end + + unless table_exists?(table) + raise "Table #{table} does not exist" + end + + unless column_exists?(table, primary_key) + raise "Column #{primary_key} does not exist on #{table}" + end + + unless column_exists?(table, column) + raise "Column #{column} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + old_column = column_for(table, column) + tmp_column = "#{column}_convert_to_bigint" + + with_lock_retries do + if (column.to_s == primary_key.to_s) || !old_column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) + else + add_column(table, tmp_column, :bigint, default: old_column.default) + end + + install_rename_triggers(table, column, tmp_column) + end + + source_model = Class.new(ActiveRecord::Base) do + include EachBatch + + self.table_name = table + self.inheritance_column = :_type_disabled + end + + queue_background_migration_jobs_by_range_at_intervals( + source_model, + 'CopyColumnUsingBackgroundMigrationJob', + interval, + batch_size: batch_size, + other_job_arguments: [table, primary_key, column, tmp_column, sub_batch_size], + track_jobs: true, + primary_column_name: primary_key + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob') + end + end + # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc @@ -996,9 +1165,9 @@ module Gitlab Arel::Nodes::SqlLiteral.new(replace.to_sql) end - def remove_foreign_key_if_exists(*args) - if foreign_key_exists?(*args) - remove_foreign_key(*args) + def remove_foreign_key_if_exists(...) + if foreign_key_exists?(...) + remove_foreign_key(...) end end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 36073844765..12dcf68da2f 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -100,6 +100,7 @@ module Gitlab end final_delay = 0 + batch_counter = 0 model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck(Arel.sql("MIN(#{primary_column_name}), MAX(#{primary_column_name})")).first @@ -112,8 +113,17 @@ module Gitlab track_in_database(job_class_name, full_job_arguments) if track_jobs migrate_in(final_delay, job_class_name, full_job_arguments) + + batch_counter += 1 end + duration = initial_delay + delay_interval * batch_counter + say <<~SAY + Scheduled #{batch_counter} #{job_class_name} jobs with a maximum of #{batch_size} records per batch and an interval of #{delay_interval} seconds. + + The migration is expected to take at least #{duration} seconds. Expect all jobs to have completed after #{Time.zone.now + duration}." + SAY + final_delay end diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index f367292f4b0..0bc1343acca 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -32,7 +32,7 @@ module Gitlab return end - partitioned_table.postgres_partitions.each do |partition| + partitioned_table.postgres_partitions.order(:name).each do |partition| partition_index_name = generated_index_name(partition.identifier, options[:name]) partition_options = options.merge(name: partition_index_name) diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 33faa2ef1b0..62dfaeeaae3 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -16,9 +16,9 @@ module Gitlab # Grouped relations are NOT supported yet. # # @example Usage - # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).estimate_distinct_count + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).execute # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.with_active_services.service_desk_enabled.where(time_period)) - # .estimate_distinct_count( + # .execute( # batch_size: 1_000, # start: ::Project.with_active_services.service_desk_enabled.where(time_period).minimum(:id), # finish: ::Project.with_active_services.service_desk_enabled.where(time_period).maximum(:id) @@ -30,7 +30,6 @@ module Gitlab # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. class BatchDistinctCounter ERROR_RATE = 4.9 # max encountered empirical error rate, used in tests - FALLBACK = -1 MIN_REQUIRED_BATCH_SIZE = 750 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep MAX_DATA_VOLUME = 4_000_000_000 @@ -38,8 +37,10 @@ module Gitlab # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 DEFAULT_BATCH_SIZE = 10_000 + ZERO_OFFSET = 1 + BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2) BIT_31_MASK = "B'0#{'1' * 31}'" - BIT_9_MASK = "B'#{'0' * 23}#{'1' * 9}'" + BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'" # @example source_query # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits # FROM %{relation} @@ -48,73 +49,58 @@ module Gitlab # AND %{column} IS NOT NULL BUCKETED_DATA_SQL = <<~SQL WITH hashed_attributes AS (%{source_query}) - SELECT (attr_hash_32_bits & #{BIT_9_MASK})::int AS bucket_num, + SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash FROM hashed_attributes GROUP BY 1 SQL - TOTAL_BUCKETS_NUMBER = 512 + WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid) def initialize(relation, column = nil) @relation = relation @column = column || relation.primary_key end - def unwanted_configuration?(finish, batch_size, start) - batch_size <= MIN_REQUIRED_BATCH_SIZE || - (finish - start) >= MAX_DATA_VOLUME || - start > finish - end - - def estimate_distinct_count(batch_size: nil, start: nil, finish: nil) + # Executes counter that iterates over database source and return Gitlab::Database::PostgresHll::Buckets + # that can be used to estimation of number of uniq elements in analysed set + # + # @param batch_size maximal number of rows that will be analysed by single database query + # @param start initial pkey range + # @param finish final pkey range + # @return [Gitlab::Database::PostgresHll::Buckets] HyperLogLog data structure instance that can estimate number of unique elements + def execute(batch_size: nil, start: nil, finish: nil) raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? batch_size ||= DEFAULT_BATCH_SIZE - start = actual_start(start) finish = actual_finish(finish) - raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) + raise WRONG_CONFIGURATION_ERROR if unwanted_configuration?(start, finish, batch_size) batch_start = start - hll_blob = {} + hll_buckets = Buckets.new while batch_start <= finish begin - hll_blob.merge!(hll_blob_for_batch(batch_start, batch_start + batch_size)) {|_key, old, new| new > old ? new : old } + hll_buckets.merge_hash!(hll_buckets_for_batch(batch_start, batch_start + batch_size)) batch_start += batch_size end sleep(SLEEP_TIME_IN_SECONDS) end - estimate_cardinality(hll_blob) + hll_buckets end private - # arbitrary values that are present in #estimate_cardinality - # are sourced from https://www.sisense.com/blog/hyperloglog-in-pure-sql/ - # article, they are not representing any entity and serves as tune value - # for the whole equation - def estimate_cardinality(hll_blob) - num_zero_buckets = TOTAL_BUCKETS_NUMBER - hll_blob.size - - num_uniques = ( - ((TOTAL_BUCKETS_NUMBER**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER))) / - (num_zero_buckets + hll_blob.values.sum { |bucket_hash| 2**(-1 * bucket_hash)} ) - ).to_i - - if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS_NUMBER - ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER)) * (TOTAL_BUCKETS_NUMBER * - Math.log2(TOTAL_BUCKETS_NUMBER.to_f / num_zero_buckets))) - else - num_uniques - end + def unwanted_configuration?(start, finish, batch_size) + batch_size <= MIN_REQUIRED_BATCH_SIZE || + (finish - start) >= MAX_DATA_VOLUME || + start > finish || start < 0 || finish < 0 end - def hll_blob_for_batch(start, finish) + def hll_buckets_for_batch(start, finish) @relation .connection .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb new file mode 100644 index 00000000000..429e823379f --- /dev/null +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PostgresHll + # Bucket class represent data structure build with HyperLogLog algorithm + # that models data distribution in analysed set. This representation than can be used + # for following purposes + # 1. Estimating number of unique elements that this structure represents + # 2. Merging with other Buckets structure to later estimate number of unique elements in sum of two + # represented data sets + # 3. Serializing Buckets structure to json format, that can be stored in various persistence layers + # + # @example Usage + # ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1).estimated_distinct_count + # ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1).merge_hash!(141 => 1, 56 => 5).estimated_distinct_count + # ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1).to_json + + # @note HyperLogLog is an PROBABILISTIC algorithm that ESTIMATES distinct count of given attribute value for supplied relation + # Like all probabilistic algorithm is has ERROR RATE margin, that can affect values, + # for given implementation no higher value was reported (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45673#accuracy-estimation) than 5.3% + # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. + class Buckets + TOTAL_BUCKETS = 512 + + def initialize(buckets = {}) + @buckets = buckets + end + + # Based on HyperLogLog structure estimates number of unique elements in analysed set. + # + # @return [Float] Estimate number of unique elements + def estimated_distinct_count + @estimated_distinct_count ||= estimate_cardinality + end + + # Updates instance underlying HyperLogLog structure by merging it with other HyperLogLog structure + # + # @param other_buckets_hash hash with HyperLogLog structure representation + def merge_hash!(other_buckets_hash) + buckets.merge!(other_buckets_hash) {|_key, old, new| new > old ? new : old } + end + + # Serialize instance underlying HyperLogLog structure to JSON format, that can be stored in various persistence layers + # + # @return [String] HyperLogLog data structure serialized to JSON + def to_json(_ = nil) + buckets.to_json + end + + private + + attr_accessor :buckets + + # arbitrary values that are present in #estimate_cardinality + # are sourced from https://www.sisense.com/blog/hyperloglog-in-pure-sql/ + # article, they are not representing any entity and serves as tune value + # for the whole equation + def estimate_cardinality + num_zero_buckets = TOTAL_BUCKETS - buckets.size + + num_uniques = ( + ((TOTAL_BUCKETS**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS))) / + (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash)} ) + ).to_i + + if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS + ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS)) * (TOTAL_BUCKETS * + Math.log2(TOTAL_BUCKETS.to_f / num_zero_buckets))) + else + num_uniques + end + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 832f7438cf9..0cfad690283 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -8,9 +8,9 @@ module Gitlab # candidate_indexes: Array of Gitlab::Database::PostgresIndex def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) - indexes = IndexSelection.new(candidate_indexes).take(how_many) - - Coordinator.new(indexes).perform + IndexSelection.new(candidate_indexes).take(how_many).each do |index| + Coordinator.new(index).perform + end end def self.candidate_indexes diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 0957f43e166..7a7d17ca196 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -12,26 +12,44 @@ module Gitlab # statement timeouts). TIMEOUT_PER_ACTION = 1.day - attr_reader :indexes + attr_reader :index, :notifier - def initialize(indexes) - @indexes = indexes + def initialize(index, notifier = GrafanaNotifier.new) + @index = index + @notifier = notifier end def perform - indexes.each do |index| - # This obtains a global lease such that there's - # only one live reindexing process at a time. - try_obtain_lease do - ReindexAction.keep_track_of(index) do - ConcurrentReindex.new(index).perform - end + # This obtains a global lease such that there's + # only one live reindexing process at a time. + try_obtain_lease do + action = ReindexAction.create_for(index) + + with_notifications(action) do + perform_for(index, action) end end end private + def with_notifications(action) + notifier.notify_start(action) + yield + ensure + notifier.notify_end(action) + end + + def perform_for(index, action) + ConcurrentReindex.new(index).perform + rescue + action.state = :failed + + raise + ensure + action.finish + end + def lease_timeout TIMEOUT_PER_ACTION end diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb new file mode 100644 index 00000000000..b1e5ecb9ade --- /dev/null +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + # This can be used to send annotations for reindexing to a Grafana API + class GrafanaNotifier + def initialize(api_key = ENV['GITLAB_GRAFANA_API_KEY'], api_url = ENV['GITLAB_GRAFANA_API_URL'], additional_tag = ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env) + @api_key = api_key + @api_url = api_url + @additional_tag = additional_tag + end + + def notify_start(action) + return unless enabled? + + payload = base_payload(action).merge( + text: "Started reindexing of #{action.index.name} on #{action.index.tablename}" + ) + + annotate(payload) + end + + def notify_end(action) + return unless enabled? + + payload = base_payload(action).merge( + text: "Finished reindexing of #{action.index.name} on #{action.index.tablename} (#{action.state})", + timeEnd: (action.action_end.utc.to_f * 1000).to_i, + isRegion: true + ) + + annotate(payload) + end + + private + + def base_payload(action) + { + time: (action.action_start.utc.to_f * 1000).to_i, + tags: ['reindex', @additional_tag, action.index.tablename, action.index.name].compact + } + end + + def annotate(payload) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer #{@api_key}" + } + + success = Gitlab::HTTP.post("#{@api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true).success? + + log_error("Response code #{response.code}") unless success + + success + rescue => err + log_error(err) + + false + end + + def log_error(err) + Gitlab::AppLogger.warn("Unable to notify Grafana from #{self.class}: #{err}") + end + + def enabled? + !(@api_url.blank? || @api_key.blank?) + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb index 8c59cffe5fb..7e58201889f 100644 --- a/lib/gitlab/database/reindexing/reindex_action.rb +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -14,27 +14,23 @@ module Gitlab scope :recent, -> { where(state: :finished).where('action_end > ?', Time.zone.now - RECENT_THRESHOLD) } - def self.keep_track_of(index, &block) - action = create!( + def self.create_for(index) + create!( index_identifier: index.identifier, action_start: Time.zone.now, ondisk_size_bytes_start: index.ondisk_size_bytes, bloat_estimate_bytes_start: index.bloat_size ) + end - yield - - action.state = :finished - rescue - action.state = :failed - raise - ensure + def finish index.reload # rubocop:disable Cop/ActiveRecordAssociationReload - action.action_end = Time.zone.now - action.ondisk_size_bytes_end = index.ondisk_size_bytes + self.state = :finished unless failed? + self.action_end = Time.zone.now + self.ondisk_size_bytes_end = index.ondisk_size_bytes - action.save! + save! end end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index b1093b2fca4..d1ada8c723e 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -75,7 +75,7 @@ module Gitlab if response # In the add_prometheus_manual_configuration method, the Prometheus - # listen_address config is saved as an api_url in the PrometheusService + # server_address config is saved as an api_url in the PrometheusService # model. There are validates hooks in the PrometheusService model that # check if the project associated with the PrometheusService is the # self_monitoring project. It checks @@ -105,7 +105,7 @@ module Gitlab def add_prometheus_manual_configuration(result) return success(result) unless prometheus_enabled? - return success(result) unless prometheus_listen_address.present? + return success(result) unless prometheus_server_address.present? service = result[:project].find_or_initialize_service('prometheus') @@ -132,8 +132,8 @@ module Gitlab ::Gitlab::Prometheus::Internal.prometheus_enabled? end - def prometheus_listen_address - ::Gitlab::Prometheus::Internal.listen_address + def prometheus_server_address + ::Gitlab::Prometheus::Internal.server_address end def docs_path @@ -152,13 +152,13 @@ module Gitlab } end - def internal_prometheus_listen_address_uri + def internal_prometheus_server_address_uri ::Gitlab::Prometheus::Internal.uri end def prometheus_service_attributes { - api_url: internal_prometheus_listen_address_uri, + api_url: internal_prometheus_server_address_uri, manual_configuration: true, active: true } diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index af9140215f0..98ed2400d82 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,9 +8,9 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :old_pos, :new_pos + attr_reader :line_code attr_writer :rich_text - attr_accessor :text, :index, :type + attr_accessor :text, :index, :type, :old_pos, :new_pos def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index e43f301c280..74c33c46598 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -19,6 +19,7 @@ module Gitlab :height, :x, :y, + :line_range, :position_type, to: :formatter # A position can belong to a text line or to an image coordinate @@ -167,6 +168,12 @@ module Gitlab end end + def multiline? + return unless on_text? && line_range + + line_range['start'] != line_range['end'] + end + private def find_diff_file(repository) diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index 1b8421d34f3..e71ea154355 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -11,6 +11,7 @@ module Gitlab [ CreateNoteHandler, CreateIssueHandler, + CreateNoteOnIssuableHandler, UnsubscribeHandler, CreateMergeRequestHandler, ServiceDeskHandler diff --git a/lib/gitlab/email/handler/create_note_on_issuable_handler.rb b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb new file mode 100644 index 00000000000..aed3647744a --- /dev/null +++ b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'gitlab/email/handler/base_handler' + +# Handles comment creation emails when sent/forwarded by an authorized +# user. Attachments are allowed. Quoted material is _not_ stripped, just like +# create issue emails +# Supports these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-issue-34@incoming.gitlab.com +module Gitlab + module Email + module Handler + class CreateNoteOnIssuableHandler < BaseHandler + include ReplyProcessing + + attr_reader :issuable_iid + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue-(?<issuable_iid>\d+)\z/.freeze + + def initialize(mail, mail_key) + super(mail, mail_key) + + if (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + @issuable_iid = matched[:issuable_iid]&.to_i + end + end + + def can_handle? + incoming_email_token && project_id && issuable_iid + end + + def execute + raise ProjectNotFound unless project + + validate_permission!(:create_note) + + raise NoteableNotFoundError unless noteable + raise EmptyEmailError if message_including_reply.blank? + + verify_record!( + record: create_note, + invalid_exception: InvalidNoteError, + record_name: 'comment') + end + + def metrics_event + :receive_email_create_note_issuable + end + + def noteable + return unless issuable_iid + + @noteable ||= project&.issues&.find_by_iid(issuable_iid) + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def author + @author ||= User.find_by(incoming_email_token: incoming_email_token) + end + # rubocop: enable CodeReuse/ActiveRecord + + def create_note + Notes::CreateService.new(project, author, note_params).execute + end + + def note_params + { + noteable_type: noteable.class.to_s, + noteable_id: noteable.id, + note: message_including_reply + } + end + end + end + end +end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 0bbe3980f67..f66e8a8794f 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -68,7 +68,7 @@ module Gitlab end def valid_project_key?(project, slug) - project.present? && slug == project.full_path_slug && Feature.enabled?(:service_desk_custom_address, project, default_enabled: true) + project.present? && slug == project.full_path_slug end def create_issue! diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index a5ace2be773..1a8e5aaf07a 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -111,8 +111,8 @@ module Gitlab private def before_send(event, hint) - event = add_context_from_exception_type(event, hint) - event = custom_fingerprinting(event, hint) + inject_context_for_exception(event, hint[:exception]) + custom_fingerprinting(event, hint[:exception]) event end @@ -123,7 +123,6 @@ module Gitlab end extra = sanitize_request_parameters(extra) - inject_sql_query_into_extra(exception, extra) if sentry && Raven.configuration.server Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -150,12 +149,6 @@ module Gitlab filter.filter(parameters) end - def inject_sql_query_into_extra(exception, extra) - return unless exception.is_a?(ActiveRecord::StatementInvalid) - - extra[:sql] = PgQuery.normalize(exception.sql.to_s) - end - def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled @@ -183,31 +176,21 @@ module Gitlab {} end - # Debugging for https://gitlab.com/gitlab-org/gitlab-foss/issues/57727 - def add_context_from_exception_type(event, hint) - if ActiveModel::MissingAttributeError === hint[:exception] - columns_hash = ActiveRecord::Base - .connection - .schema_cache - .instance_variable_get(:@columns_hash) - .transform_values { |v| v.map(&:first) } - - event.extra.merge!(columns_hash) - end - - event - end - # Group common, mostly non-actionable exceptions by type and message, # rather than cause - def custom_fingerprinting(event, hint) - ex = hint[:exception] - + def custom_fingerprinting(event, ex) return event unless CUSTOM_FINGERPRINTING.include?(ex.class.name) event.fingerprint = [ex.class.name, ex.message] + end - event + def inject_context_for_exception(event, ex) + case ex + when ActiveRecord::StatementInvalid + event.extra[:sql] = PgQuery.normalize(ex.sql.to_s) + else + inject_context_for_exception(event, ex.cause) if ex.cause.present? + end end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 94523813662..196203211ed 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -87,6 +87,24 @@ module Gitlab }, invite_members_empty_project_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyProjectVersionA' + }, + trial_during_signup: { + tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' + }, + ci_syntax_templates: { + tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates' + }, + pipelines_empty_state: { + tracking_category: 'Growth::Activation::Experiment::PipelinesEmptyState' + }, + invite_members_new_dropdown: { + tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' + }, + show_trial_status_in_sidebar: { + tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar' + }, + trial_onboarding_issues: { + tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' } }.freeze diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index c85d3f4eee6..e43f3c8c007 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -15,7 +15,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group + helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :tracking_label end def set_experimentation_subject_id_cookie @@ -130,7 +130,10 @@ module Gitlab end def forced_enabled?(experiment_key) - params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + return true if params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + return false if cookies[:force_experiment].blank? + + cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } end def tracking_label(subject) diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index e594c3bedeb..36cd673a38f 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -3,17 +3,21 @@ module Gitlab module Experimentation class Experiment + FEATURE_FLAG_SUFFIX = "_experiment_percentage" + attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index def initialize(key, **params) @key = key @tracking_category = params[:tracking_category] @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] - - @experiment_percentage = Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet end def active? + # TODO: just touch a feature flag + # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? + Feature.enabled?(feature_flag_name, type: :experiment, default_enabled: :yaml) + ::Gitlab.dev_env_or_com? && experiment_percentage > 0 end @@ -25,7 +29,17 @@ module Gitlab private - attr_reader :experiment_percentage + def experiment_percentage + feature_flag.percentage_of_time_value + end + + def feature_flag + Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet + end + + def feature_flag_name + :"#{key}#{FEATURE_FLAG_SUFFIX}" + end end end end diff --git a/lib/gitlab/faraday.rb b/lib/gitlab/faraday.rb new file mode 100644 index 00000000000..f92392ec1a9 --- /dev/null +++ b/lib/gitlab/faraday.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module Faraday + ::Faraday::Request.register_middleware(gitlab_error_callback: -> { ::Gitlab::Faraday::ErrorCallback }) + end +end diff --git a/lib/gitlab/faraday/error_callback.rb b/lib/gitlab/faraday/error_callback.rb new file mode 100644 index 00000000000..f99be5b4d04 --- /dev/null +++ b/lib/gitlab/faraday/error_callback.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Faraday + # Simple Faraday Middleware that catches any error risen during the request and run the configured callback. + # (https://lostisland.github.io/faraday/middleware/) + # + # By default, a no op callback is setup. + # + # Note that the error is not swallowed: it will be rerisen again. In that regard, this callback acts more + # like an error spy than anything else. + # + # The callback has access to the request `env` and the exception instance. For more details, see + # https://lostisland.github.io/faraday/middleware/custom + # + # Faraday.new do |conn| + # conn.request( + # :error_callback, + # callback: -> (env, exception) { Rails.logger.debug("Error #{exception.class.name} when trying to contact #{env[:url]}" ) } + # ) + # conn.adapter(:net_http) + # end + class ErrorCallback < ::Faraday::Middleware + def initialize(app, options = nil) + super(app) + @options = ::Gitlab::Faraday::ErrorCallback::Options.from(options) # rubocop: disable CodeReuse/ActiveRecord + end + + def call(env) + @app.call(env) + rescue => e + @options.callback&.call(env, e) + + raise + end + + class Options < ::Faraday::Options.new(:callback) + def callback + self[:callback] + end + end + end + end +end diff --git a/lib/gitlab/git/changed_path.rb b/lib/gitlab/git/changed_path.rb new file mode 100644 index 00000000000..033779466f6 --- /dev/null +++ b/lib/gitlab/git/changed_path.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class ChangedPath + attr_reader :status, :path + + def initialize(status:, path:) + @status = status + @path = path + end + + def new_file? + status == :ADDED + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 8df4bc3de05..19462e6cb02 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -13,7 +13,7 @@ module Gitlab def self.default_limits(project: nil) if Feature.enabled?(:increased_diff_limits, project) - { max_files: 200, max_lines: 7500 } + { max_files: 300, max_lines: 10000 } else { max_files: 100, max_lines: 5000 } end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f6601379202..e316d52ac05 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -801,7 +801,8 @@ module Gitlab # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # prune - should we use --prune flag? - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + # check_tags_changed - should we ask gitaly to calculate whether any tags changed? + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) wrapped_gitaly_errors do gitaly_repository_client.fetch_remote( remote, @@ -809,6 +810,7 @@ module Gitlab forced: forced, no_tags: no_tags, prune: prune, + check_tags_changed: check_tags_changed, timeout: GITLAB_PROJECTS_TIMEOUT ) end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb index 475a9d4d1b9..efe39fa852c 100644 --- a/lib/gitlab/git/wiki_page_version.rb +++ b/lib/gitlab/git/wiki_page_version.rb @@ -10,7 +10,12 @@ module Gitlab @format = format end - delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + delegate :message, :sha, :id, :author_name, :author_email, :authored_date, to: :commit + + def author_url + user = ::User.find_by_any_email(author_email) + user.nil? ? "mailto:#{author_email}" : Gitlab::UrlBuilder.build(user) + end end end end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 854bf6e9c9e..88a75f72840 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -30,7 +30,10 @@ module Gitlab def check(cmd, changes) check_snippet_accessibility! - super + super.tap do |_| + # Ensure HEAD points to the default branch in case it is not master + snippet.change_head_to_default_branch + end end override :download_ability @@ -56,7 +59,7 @@ module Gitlab # TODO: Investigate if expanding actor/authentication types are needed. # https://gitlab.com/gitlab-org/gitlab/issues/202190 if actor && !allowed_actor? - raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] + raise ForbiddenError, error_message(:authentication_mechanism) end super @@ -68,14 +71,18 @@ module Gitlab override :check_push_access! def check_push_access! - raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user + raise ForbiddenError, error_message(:update_snippet) unless user + + if snippet&.repository_read_only? + raise ForbiddenError, error_message(:read_only) + end check_change_access! end def check_snippet_accessibility! if snippet.blank? - raise NotFoundError, ERROR_MESSAGES[:snippet_not_found] + raise NotFoundError, error_message(:snippet_not_found) end end @@ -91,14 +98,14 @@ module Gitlab passed = guest_can_download_code? || user_can_download_code? unless passed - raise ForbiddenError, ERROR_MESSAGES[:read_snippet] + raise ForbiddenError, error_message(:read_snippet) end end override :check_change_access! def check_change_access! unless user_can_push? - raise ForbiddenError, ERROR_MESSAGES[:update_snippet] + raise ForbiddenError, error_message(:update_snippet) end check_size_before_push! diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e1324530412..31734abe77f 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -215,12 +215,16 @@ module Gitlab 'client_name' => CLIENT_NAME } + context_data = Labkit::Context.current&.to_h + feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id metadata['gitaly-session-id'] = session_id + metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) + metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) metadata.merge!(Feature::Gitaly.server_feature_flags) deadline_info = request_deadline(timeout) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 599bce176c9..ea940150941 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -225,7 +225,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map do |path| - OpenStruct.new( + Gitlab::Git::ChangedPath.new( status: path.status, path: EncodingHelper.encode!(path.path) ) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index e41a406ebd3..bd450249355 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -70,10 +70,11 @@ module Gitlab end.join end - def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, - no_tags: no_tags, timeout: timeout, no_prune: !prune + no_tags: no_tags, timeout: timeout, no_prune: !prune, + check_tags_changed: check_tags_changed ) if ssh_auth&.ssh_mirror_url? diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 7ae91912b8a..1401c92a44e 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -56,7 +56,7 @@ module Gitlab # The initial fetch can bring in lots of loose refs and objects. # Running a `git gc` will make importing pull requests faster. - Projects::HousekeepingService.new(project, :gc).execute + Repositories::HousekeepingService.new(project, :gc).execute true rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e diff --git a/lib/gitlab/gitpod.rb b/lib/gitlab/gitpod.rb deleted file mode 100644 index e35fb8fed02..00000000000 --- a/lib/gitlab/gitpod.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class Gitpod - class << self - def feature_available? - # The gitpod_bundle feature could be conditionally applied, so check if `!off?` - !feature.off? || feature_enabled? - end - - def feature_enabled?(actor = nil) - Feature.enabled?(:gitpod, actor, default_enabled: true) - end - - def feature_and_settings_enabled?(actor = nil) - feature_enabled?(actor) && Gitlab::CurrentSettings.gitpod_enabled - end - - private - - def feature - Feature.get(:gitpod) # rubocop:disable Gitlab/AvoidFeatureGet - end - end - end -end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 362da8ea53e..0ba535b500e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -4,7 +4,6 @@ module Gitlab module GonHelper - include StartupCssHelper include WebpackHelper def add_gon_variables @@ -48,9 +47,7 @@ module Gitlab push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, default_enabled: true) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - - # Startup CSS feature is a special one as it can be enabled by means of cookies and params - gon.push({ features: { 'startupCss' => use_startup_css? } }, true) + push_frontend_feature_flag(:gl_tooltips, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/batch_key.rb b/lib/gitlab/graphql/batch_key.rb new file mode 100644 index 00000000000..51203af5a43 --- /dev/null +++ b/lib/gitlab/graphql/batch_key.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class BatchKey + attr_reader :object + delegate :hash, to: :object + + def initialize(object, lookahead = nil, object_name: nil) + @object = object + @lookahead = lookahead + @object_name = object_name + end + + def requires?(path) + return false unless @lookahead + return false unless path.present? + + field = path.pop + + path + .reduce(@lookahead) { |q, f| q.selection(f) } + .selects?(field) + end + + def eql?(other) + other.is_a?(self.class) && object == other.object + end + alias_method :==, :eql? + + def method_missing(method_name, *args, **kwargs) + return @object if method_name.to_sym == @object_name + return @object.public_send(method_name) if args.empty? && kwargs.empty? # rubocop: disable GitlabSecurity/PublicSend + + super + end + end + end +end diff --git a/lib/gitlab/graphql/lazy.rb b/lib/gitlab/graphql/lazy.rb index 54013cf4790..3563504226c 100644 --- a/lib/gitlab/graphql/lazy.rb +++ b/lib/gitlab/graphql/lazy.rb @@ -17,6 +17,14 @@ module Gitlab self.class.new { yield force } end + def catch(error_class = StandardError, &block) + self.class.new do + force + rescue error_class => e + yield e + end + end + # Force evaluation of a (possibly) lazy value def self.force(value) case value diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 2ad8d2f7ab7..f95c91c5706 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -67,9 +67,14 @@ module Gitlab # next page true elsif first - # If we count the number of requested items plus one (`limit_value + 1`), - # then if we get `limit_value + 1` then we know there is a next page - relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + case sliced_nodes + when Array + sliced_nodes.size > limit_value + else + # If we count the number of requested items plus one (`limit_value + 1`), + # then if we get `limit_value + 1` then we know there is a next page + relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + end else false end @@ -157,8 +162,8 @@ module Gitlab list = OrderInfo.build_order_list(items) - if loaded?(items) - @order_list = list.presence || [items.primary_key] + if loaded?(items) && !before.present? && !after.present? + @order_list = list.presence || [OrderInfo.new(items.primary_key)] # already sorted, or trivially sorted next items if list.present? || items.size <= 1 @@ -194,7 +199,7 @@ module Gitlab ordering = { 'id' => node[:id].to_s } order_list.each do |field| - field_name = field.attribute_name + field_name = field.try(:attribute_name) || field field_value = node[field_name] ordering[field_name] = if field_value.is_a?(Time) field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb index 331981ce723..29169449843 100644 --- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb +++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb @@ -40,7 +40,10 @@ module Gitlab # "issues"."id" > 500 # def conditions - attr_values = order_list.map { |field| decoded_cursor[field.attribute_name] } + attr_values = order_list.map do |field| + name = field.try(:attribute_name) || field + decoded_cursor[name] + end if order_list.count == 1 && attr_values.first.nil? raise Gitlab::Graphql::Errors::ArgumentError.new('Before/after cursor invalid: `nil` was provided as only sortable value') diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb new file mode 100644 index 00000000000..de971743490 --- /dev/null +++ b/lib/gitlab/graphql/queries.rb @@ -0,0 +1,286 @@ +# frozen_string_literal: true + +require 'find' + +module Gitlab + module Graphql + module Queries + IMPORT_RE = /^#\s*import "(?<path>[^"]+)"$/m.freeze + EE_ELSE_CE = /^ee_else_ce/.freeze + HOME_RE = /^~/.freeze + HOME_EE = %r{^ee/}.freeze + DOTS_RE = %r{^(\.\./)+}.freeze + DOT_RE = %r{^\./}.freeze + IMPLICIT_ROOT = %r{^app/}.freeze + CONN_DIRECTIVE = /@connection\(key: "\w+"\)/.freeze + + class WrappedError + delegate :message, to: :@error + + def initialize(error) + @error = error + end + + def path + [] + end + end + + class FileNotFound + def initialize(file) + @file = file + end + + def message + "File not found: #{@file}" + end + + def path + [] + end + end + + # We need to re-write queries to remove all @client fields. Ideally we + # would do that as a source-to-source transformation of the AST, but doing it using a + # printer is much simpler. + class ClientFieldRedactor < GraphQL::Language::Printer + attr_reader :fields_printed, :skipped_arguments, :printed_arguments, :used_fragments + + def initialize(skips = true) + @skips = skips + @fields_printed = 0 + @in_operation = false + @skipped_arguments = [].to_set + @printed_arguments = [].to_set + @used_fragments = [].to_set + @skipped_fragments = [].to_set + @used_fragments = [].to_set + end + + def print_variable_identifier(variable_identifier) + @printed_arguments << variable_identifier.name + super + end + + def print_fragment_spread(fragment_spread, indent: "") + @used_fragments << fragment_spread.name + super + end + + def print_operation_definition(op, indent: "") + @in_operation = true + out = +"#{indent}#{op.operation_type}" + out << " #{op.name}" if op.name + + # Do these first, so that we detect any skipped arguments + dirs = print_directives(op.directives) + sels = print_selections(op.selections, indent: indent) + + # remove variable definitions only used in skipped (client) fields + vars = op.variables.reject do |v| + @skipped_arguments.include?(v.name) && !@printed_arguments.include?(v.name) + end + + if vars.any? + out << "(#{vars.map { |v| print_variable_definition(v) }.join(", ")})" + end + + out + dirs + sels + ensure + @in_operation = false + end + + def print_field(field, indent: '') + if skips? && field.directives.any? { |d| d.name == 'client' } + skipped = self.class.new(false) + + skipped.print_node(field) + @skipped_fragments |= skipped.used_fragments + @skipped_arguments |= skipped.printed_arguments + + return '' + end + + ret = super + + @fields_printed += 1 if @in_operation && ret != '' + + ret + end + + def print_fragment_definition(fragment_def, indent: "") + if skips? && @skipped_fragments.include?(fragment_def.name) && !@used_fragments.include?(fragment_def.name) + return '' + end + + super + end + + def skips? + @skips + end + end + + class Definition + attr_reader :file, :imports + + def initialize(path, fragments) + @file = path + @fragments = fragments + @imports = [] + @errors = [] + @ee_else_ce = [] + end + + def text(mode: :ce) + qs = [query] + all_imports(mode: mode).uniq.sort.map { |p| fragment(p).query } + t = qs.join("\n\n").gsub(/\n\n+/, "\n\n") + + return t unless /@client/.match?(t) + + doc = ::GraphQL.parse(t) + printer = ClientFieldRedactor.new + redacted = doc.dup.to_query_string(printer: printer) + + return redacted if printer.fields_printed > 0 + end + + def query + return @query if defined?(@query) + + # CONN_DIRECTIVEs are purely client-side constructs + @query = File.read(file).gsub(CONN_DIRECTIVE, '').gsub(IMPORT_RE) do + path = $~[:path] + + if EE_ELSE_CE.match?(path) + @ee_else_ce << path.gsub(EE_ELSE_CE, '') + else + @imports << fragment_path(path) + end + + '' + end + rescue Errno::ENOENT + @errors << FileNotFound.new(file) + @query = nil + end + + def all_imports(mode: :ce) + return [] if query.nil? + + home = mode == :ee ? @fragments.home_ee : @fragments.home + eithers = @ee_else_ce.map { |p| home + p } + + (imports + eithers).flat_map { |p| [p] + @fragments.get(p).all_imports(mode: mode) } + end + + def all_errors + return @errors.to_set if query.nil? + + paths = imports + @ee_else_ce.flat_map { |p| [@fragments.home + p, @fragments.home_ee + p] } + + paths.map { |p| fragment(p).all_errors }.reduce(@errors.to_set) { |a, b| a | b } + end + + def validate(schema) + return [:client_query, []] if query.present? && text.nil? + + errs = all_errors.presence || schema.validate(text) + if @ee_else_ce.present? + errs += schema.validate(text(mode: :ee)) + end + + [:validated, errs] + rescue ::GraphQL::ParseError => e + [:validated, [WrappedError.new(e)]] + end + + private + + def fragment(path) + @fragments.get(path) + end + + def fragment_path(import_path) + frag_path = import_path.gsub(HOME_RE, @fragments.home) + frag_path = frag_path.gsub(HOME_EE, @fragments.home_ee + '/') + frag_path = frag_path.gsub(DOT_RE) do + Pathname.new(file).parent.to_s + '/' + end + frag_path = frag_path.gsub(DOTS_RE) do |dots| + rel_dir(dots.split('/').count) + end + frag_path = frag_path.gsub(IMPLICIT_ROOT) do + (Rails.root / 'app').to_s + '/' + end + + frag_path + end + + def rel_dir(n_steps_up) + path = Pathname.new(file).parent + while n_steps_up > 0 + path = path.parent + n_steps_up -= 1 + end + + path.to_s + '/' + end + end + + class Fragments + def initialize(root, dir = 'app/assets/javascripts') + @root = root + @store = {} + @dir = dir + end + + def home + @home ||= (@root / @dir).to_s + end + + def home_ee + @home_ee ||= (@root / 'ee' / @dir).to_s + end + + def get(frag_path) + @store[frag_path] ||= Definition.new(frag_path, self) + end + end + + def self.find(root) + definitions = [] + + ::Find.find(root.to_s) do |path| + definitions << Definition.new(path, fragments) if query?(path) + end + + definitions + rescue Errno::ENOENT + [] # root does not exist + end + + def self.fragments + @fragments ||= Fragments.new(Rails.root) + end + + def self.all + ['.', 'ee'].flat_map do |prefix| + find(Rails.root / prefix / 'app/assets/javascripts') + end + end + + def self.known_failure?(path) + @known_failures ||= YAML.safe_load(File.read(Rails.root.join('config', 'known_invalid_graphql_queries.yml'))) + + @known_failures.fetch('filenames', []).any? { |known_failure| path.to_s.ends_with?(known_failure) } + end + + def self.query?(path) + path.ends_with?('.graphql') && + !path.ends_with?('.fragment.graphql') && + !path.ends_with?('typedefs.graphql') + end + end + end +end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 7965f165683..d3468569e5e 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -65,6 +65,7 @@ module Gitlab def self.projects_list(relation_name, relation) listing(relation_name, relation.with_route) do |project| $stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red) + $stdout.puts " #{project.repository.disk_path}" end end @@ -92,6 +93,37 @@ module Gitlab end end # rubocop: enable CodeReuse/ActiveRecord + + def self.prune(relation_name, relation, dry_run: true, root: nil) + root ||= '../repositories' + + known_paths = Set.new + listing(relation_name, relation) { |p| known_paths << "#{root}/#{p.repository.disk_path}" } + + marked_for_deletion = Set.new(Dir["#{root}/@hashed/*/*/*"]) + marked_for_deletion.reject! do |path| + base = path.gsub(/\.(\w+\.)?git$/, '') + known_paths.include?(base) + end + + if marked_for_deletion.empty? + $stdout.puts "No orphaned directories found. Nothing to do!" + else + n = marked_for_deletion.size + $stdout.puts "Found #{n} orphaned #{'directory'.pluralize(n)}" + $stdout.puts "Dry run. (Run again with FORCE=1 to delete). We would have deleted:" if dry_run + end + + marked_for_deletion.each do |p| + p = Pathname.new(p) + if dry_run + $stdout.puts " - #{p}" + else + $stdout.puts "Removing #{p}" + p.rmtree + end + end + end end end end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index c09d8170d17..f0b08bb6b6a 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -4,7 +4,7 @@ module Gitlab module Jira # Gitlab JIRA HTTP client to be used with jira-ruby gem, this subclasses JIRA::HTTPClient. # Uses Gitlab::HTTP to make requests to JIRA REST API. - # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.7.0/lib/jira/http_client.rb + # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/master/lib/jira/http_client.rb class HttpClient < JIRA::HttpClient extend ::Gitlab::Utils::Override @@ -43,6 +43,8 @@ module Gitlab result end + private + def auth_params return {} unless @options[:username] && @options[:password] @@ -54,8 +56,6 @@ module Gitlab } end - private - def get_cookies cookie_array = @cookies.values.map { |cookie| "#{cookie.name}=#{cookie.value[0]}" } cookie_array += Array(@options[:additional_cookies]) if @options.key?(:additional_cookies) diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb index 9043932bbe5..f77b3e8de99 100644 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -12,7 +12,7 @@ module Gitlab # We are modeling existing kubernetes resource and don't have # control over amount of parameters. # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil) + def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil) @name = name @description = description @namespace = namespace @@ -22,6 +22,7 @@ module Gitlab @resource_version = resource_version @ingress = ingress @egress = egress + @annotations = annotations end # rubocop:enable Metrics/ParameterLists @@ -37,6 +38,7 @@ module Gitlab name: metadata[:name], description: policy[:description], namespace: metadata[:namespace], + annotations: metadata[:annotations], resource_version: metadata[:resourceVersion], labels: metadata[:labels], selector: spec[:endpointSelector], @@ -57,6 +59,7 @@ module Gitlab name: metadata[:name], description: resource[:description], namespace: metadata[:namespace], + annotations: metadata[:annotations]&.to_h, resource_version: metadata[:resourceVersion], labels: metadata[:labels]&.to_h, creation_timestamp: metadata[:creationTimestamp], @@ -80,7 +83,7 @@ module Gitlab private - attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress + attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations def selector @selector ||= {} @@ -90,6 +93,7 @@ module Gitlab meta = { name: name, namespace: namespace } meta[:labels] = labels if labels meta[:resourceVersion] = resource_version if resource_version + meta[:annotations] = annotations if annotations meta end diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb index e8fde28b44d..f3ac19e210a 100644 --- a/lib/gitlab/kubernetes/kubectl_cmd.rb +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -17,7 +17,7 @@ module Gitlab def delete_crds_from_group(group) api_resources_args = %w(-o name --api-group).push(group) - api_resources(*api_resources_args) + " | xargs " + delete('--ignore-not-found', 'crd') + PodCmd.retry_command(api_resources(*api_resources_args) + " | xargs -r " + delete('--ignore-not-found', 'crd')) end def api_resources(*args) diff --git a/lib/gitlab/kubernetes/pod_cmd.rb b/lib/gitlab/kubernetes/pod_cmd.rb new file mode 100644 index 00000000000..e4c25424e69 --- /dev/null +++ b/lib/gitlab/kubernetes/pod_cmd.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + # Miscellaneous commands that run in the helm-install-image pod, tuned to + # the idiosynchrasies of the default shell of helm-install-image + module PodCmd + class << self + def retry_command(command, times: 3) + "for i in $(seq 1 #{times.to_i}); do #{command} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)" + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb index 9f4979fa673..043d2ae84cc 100644 --- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb +++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb @@ -4,9 +4,9 @@ module Gitlab module Metrics module Samplers class ActionCableSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - def initialize(interval = SAMPLING_INTERVAL_SECONDS, action_cable: ::ActionCable.server) + def initialize(interval = nil, action_cable: ::ActionCable.server) super(interval) @action_cable = action_cable end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 39a49187e45..7f9055fed5d 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -9,7 +9,9 @@ module Gitlab attr_reader :interval # interval - The sampling interval in seconds. - def initialize(interval = self.class::SAMPLING_INTERVAL_SECONDS) + def initialize(interval = nil) + interval ||= ENV[interval_env_key]&.to_i + interval ||= self.class::DEFAULT_SAMPLING_INTERVAL_SECONDS interval_half = interval.to_f / 2 @interval = interval @@ -50,6 +52,14 @@ module Gitlab attr_reader :running + def sampler_class + self.class.name.demodulize + end + + def interval_env_key + "#{sampler_class.underscore.upcase}_INTERVAL_SECONDS" + end + def start_working @running = true diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 9ee4b0960c5..60ae22df607 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics module Samplers class DatabaseSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 METRIC_PREFIX = 'gitlab_database_connection_pool_' diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index d295beb59f1..848a55e59ff 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics module Samplers class PumaSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 def metrics @metrics ||= init_metrics diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index dac9fbd1247..76175b465e4 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,7 +6,7 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 60 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze def initialize(*) diff --git a/lib/gitlab/metrics/samplers/threads_sampler.rb b/lib/gitlab/metrics/samplers/threads_sampler.rb index 05acef7ce0c..a460594fb59 100644 --- a/lib/gitlab/metrics/samplers/threads_sampler.rb +++ b/lib/gitlab/metrics/samplers/threads_sampler.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics module Samplers class ThreadsSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 KNOWN_PUMA_THREAD_NAMES = ['puma worker check pipe', 'puma server', 'puma threadpool reaper', 'puma threadpool trimmer', 'puma worker check pipe', 'puma stat payload'].freeze diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index d7935d65e12..2fa324f3fea 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -4,6 +4,8 @@ module Gitlab module Metrics module Samplers class UnicornSampler < BaseSampler + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 + def metrics @metrics ||= init_metrics end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 43005303dec..9bbcd1e056c 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -17,6 +17,20 @@ module Gitlab RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze + def self.summary + proportional_mem = memory_usage_uss_pss + { + version: RUBY_DESCRIPTION, + gc_stat: GC.stat, + memory_rss: memory_usage_rss, + memory_uss: proportional_mem[:uss], + memory_pss: proportional_mem[:pss], + time_cputime: cpu_time, + time_realtime: real_time, + time_monotonic: monotonic_time + } + end + # Returns the current process' RSS (resident set size) in bytes. def self.memory_usage_rss sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index a6d8a778e05..79f1abe820f 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -41,7 +41,7 @@ module Gitlab end def with_open_files - @rewritten_fields.each do |field, tmp_path| + @rewritten_fields.keys.each do |field| raise "invalid field: #{field.inspect}" unless valid_field_name?(field) parsed_field = Rack::Utils.parse_nested_query(field) @@ -51,10 +51,10 @@ module Gitlab if value.nil? # we have a top level param, eg. field = 'foo' and not 'foo[bar]' raise "invalid field: #{field.inspect}" if field != key - value = open_file(@request.params, key, tmp_path.presence) + value = open_file(extract_upload_params_from(@request.params, with_prefix: key)) @open_files << value else - value = decorate_params_value(value, @request.params[key], tmp_path.presence) + value = decorate_params_value(value, @request.params[key]) end update_param(key, value) @@ -67,12 +67,12 @@ module Gitlab end # This function calls itself recursively - def decorate_params_value(path_hash, value_hash, path_override = nil) - unless path_hash.is_a?(Hash) && path_hash.count == 1 - raise "invalid path: #{path_hash.inspect}" + def decorate_params_value(hash_path, value_hash) + unless hash_path.is_a?(Hash) && hash_path.count == 1 + raise "invalid path: #{hash_path.inspect}" end - path_key, path_value = path_hash.first + path_key, path_value = hash_path.first unless value_hash.is_a?(Hash) && value_hash[path_key] raise "invalid value hash: #{value_hash.inspect}" @@ -80,19 +80,19 @@ module Gitlab case path_value when nil - value_hash[path_key] = open_file(value_hash.dig(path_key), '', path_override) + value_hash[path_key] = open_file(extract_upload_params_from(value_hash[path_key])) @open_files << value_hash[path_key] value_hash when Hash - decorate_params_value(path_value, value_hash[path_key], path_override) + decorate_params_value(path_value, value_hash[path_key]) value_hash else raise "unexpected path value: #{path_value.inspect}" end end - def open_file(params, key, path_override = nil) - ::UploadedFile.from_params(params, key, allowed_paths, path_override) + def open_file(params) + ::UploadedFile.from_params(params, allowed_paths) end # update_params ensures that both rails controllers and rack middleware can find @@ -111,6 +111,20 @@ module Gitlab private + def extract_upload_params_from(params, with_prefix: '') + param_key = "#{with_prefix}#{JWT_PARAM_SUFFIX}" + jwt_token = params[param_key] + raise "Empty JWT param: #{param_key}" if jwt_token.blank? + + payload = Gitlab::Workhorse.decode_jwt(jwt_token).first + raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) + + upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) + raise "Empty params for: #{param_key}" if upload_params.empty? + + upload_params + end + def valid_field_name?(name) # length validation return false if name.size >= REWRITTEN_FIELD_NAME_MAX_LENGTH @@ -149,82 +163,6 @@ module Gitlab end end - # TODO this class is meant to replace Handler when the feature flag - # upload_middleware_jwt_params_handler is removed - # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps - class HandlerForJWTParams < Handler - def with_open_files - @rewritten_fields.keys.each do |field| - raise "invalid field: #{field.inspect}" unless valid_field_name?(field) - - parsed_field = Rack::Utils.parse_nested_query(field) - raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 - - key, value = parsed_field.first - if value.nil? # we have a top level param, eg. field = 'foo' and not 'foo[bar]' - raise "invalid field: #{field.inspect}" if field != key - - value = open_file(extract_upload_params_from(@request.params, with_prefix: key)) - @open_files << value - else - value = decorate_params_value(value, @request.params[key]) - end - - update_param(key, value) - end - - yield - ensure - @open_files.compact - .each(&:close) - end - - # This function calls itself recursively - def decorate_params_value(hash_path, value_hash) - unless hash_path.is_a?(Hash) && hash_path.count == 1 - raise "invalid path: #{hash_path.inspect}" - end - - path_key, path_value = hash_path.first - - unless value_hash.is_a?(Hash) && value_hash[path_key] - raise "invalid value hash: #{value_hash.inspect}" - end - - case path_value - when nil - value_hash[path_key] = open_file(extract_upload_params_from(value_hash[path_key])) - @open_files << value_hash[path_key] - value_hash - when Hash - decorate_params_value(path_value, value_hash[path_key]) - value_hash - else - raise "unexpected path value: #{path_value.inspect}" - end - end - - def open_file(params) - ::UploadedFile.from_params_without_field(params, allowed_paths) - end - - private - - def extract_upload_params_from(params, with_prefix: '') - param_key = "#{with_prefix}#{JWT_PARAM_SUFFIX}" - jwt_token = params[param_key] - raise "Empty JWT param: #{param_key}" if jwt_token.blank? - - payload = Gitlab::Workhorse.decode_jwt(jwt_token).first - raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) - - upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) - raise "Empty params for: #{param_key}" if upload_params.empty? - - upload_params - end - end - def initialize(app) @app = app end @@ -235,22 +173,12 @@ module Gitlab message = ::Gitlab::Workhorse.decode_jwt(encoded_message)[0] - handler_class.new(env, message).with_open_files do + ::Gitlab::Middleware::Multipart::Handler.new(env, message).with_open_files do @app.call(env) end rescue UploadedFile::InvalidPathError => e [400, { 'Content-Type' => 'text/plain' }, e.message] end - - private - - def handler_class - if Feature.enabled?(:upload_middleware_jwt_params_handler, default_enabled: true) - ::Gitlab::Middleware::Multipart::HandlerForJWTParams - else - ::Gitlab::Middleware::Multipart::Handler - end - end end end end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index bf8d4b202b6..133d777fc32 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -15,23 +15,39 @@ module Gitlab # schedules a job which parses peek profile data and adds them # to a structured log + # rubocop:disable Gitlab/ModuleWithInstanceVariables def enqueue_stats_job(request_id) return unless gather_stats? - @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) return unless uuid = Gitlab::ExclusiveLease.new( GitlabPerformanceBarStatsWorker::LEASE_KEY, timeout: GitlabPerformanceBarStatsWorker::LEASE_TIMEOUT ).try_obtain - GitlabPerformanceBarStatsWorker.perform_in(GitlabPerformanceBarStatsWorker::WORKER_DELAY, uuid) + # stats key should be periodically processed and deleted by + # GitlabPerformanceBarStatsWorker but if it doesn't happen for + # some reason, we set expiration for the stats key to avoid + # keeping millions of request ids which would be already expired + # anyway + # rubocop:disable Gitlab/ModuleWithInstanceVariables + @client.expire( + GitlabPerformanceBarStatsWorker::STATS_KEY, + GitlabPerformanceBarStatsWorker::STATS_KEY_EXPIRE + ) + + GitlabPerformanceBarStatsWorker.perform_in( + GitlabPerformanceBarStatsWorker::WORKER_DELAY, + uuid + ) end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def gather_stats? return unless Feature.enabled?(:performance_bar_stats) - Gitlab.com? || !Rails.env.production? + Gitlab.com? || Gitlab.staging? || !Rails.env.production? end end end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 6ba36fadfa3..56eeea6e746 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -63,7 +63,8 @@ module Gitlab ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'), ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), - ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management') + ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), + ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux') ].freeze end diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb index c2f4035821e..fe06b97add6 100644 --- a/lib/gitlab/prometheus/internal.rb +++ b/lib/gitlab/prometheus/internal.rb @@ -4,43 +4,39 @@ module Gitlab module Prometheus class Internal def self.uri - return if listen_address.blank? + return if server_address.blank? - if listen_address.starts_with?('0.0.0.0:') + if server_address.starts_with?('0.0.0.0:') # 0.0.0.0:9090 - port = ':' + listen_address.split(':').second + port = ':' + server_address.split(':').second 'http://localhost' + port - elsif listen_address.starts_with?(':') + elsif server_address.starts_with?(':') # :9090 - 'http://localhost' + listen_address + 'http://localhost' + server_address - elsif listen_address.starts_with?('http') + elsif server_address.starts_with?('http') # https://localhost:9090 - listen_address + server_address else # localhost:9090 - 'http://' + listen_address + 'http://' + server_address end end def self.server_address - uri&.strip&.sub(/^http[s]?:\/\//, '') - end - - def self.listen_address - Gitlab.config.prometheus.listen_address.to_s if Gitlab.config.prometheus + Gitlab.config.prometheus.server_address.to_s if Gitlab.config.prometheus rescue Settingslogic::MissingSetting - Gitlab::AppLogger.error('Prometheus listen_address is not present in config/gitlab.yml') + Gitlab::AppLogger.error('Prometheus server_address is not present in config/gitlab.yml') nil end def self.prometheus_enabled? - Gitlab.config.prometheus.enable if Gitlab.config.prometheus + Gitlab.config.prometheus.enabled if Gitlab.config.prometheus rescue Settingslogic::MissingSetting - Gitlab::AppLogger.error('prometheus.enable is not present in config/gitlab.yml') + Gitlab::AppLogger.error('prometheus.enabled is not present in config/gitlab.yml') false end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 1822b0c8bd5..c162ee545c6 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -170,7 +170,8 @@ module Gitlab end types Issue condition do - !quick_action_target.confidential? && + quick_action_target.issue_type_supports?(:confidentiality) && + !quick_action_target.confidential? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :confidential do diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 6607c73a5c3..4934c12a339 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -26,7 +26,7 @@ module Gitlab end types Issue, MergeRequest condition do - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + quick_action_target.supports_assignee? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |assignee_param| extract_users(assignee_param) diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 1986b7a1789..b56fd8278a1 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -9,53 +9,72 @@ module Gitlab included do # MergeRequest only quick actions definitions desc do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) - _("Merge automatically (%{strategy})") % { strategy: preferred_strategy.humanize } - else - _("Merge immediately") - end + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Merge automatically (%{strategy})") % { strategy: preferred_strategy.humanize } else - _('Merge (when the pipeline succeeds)') + _("Merge immediately") end end explanation do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) - _("Schedules to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } - else - _('Merges this merge request immediately.') - end + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Schedules to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } else - _('Merges this merge request when the pipeline succeeds.') + _('Merges this merge request immediately.') end end execution_message do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) - _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } - else - _('Merged this merge request.') - end + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } else - _('Scheduled to merge this merge request when the pipeline succeeds.') + _('Merged this merge request.') end end types MergeRequest condition do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - quick_action_target.persisted? && - merge_orchestration_service.can_merge?(quick_action_target) - else - last_diff_sha = params && params[:merge_request_diff_head_sha] - quick_action_target.persisted? && - quick_action_target.mergeable_with_quick_action?(current_user, autocomplete_precheck: !last_diff_sha, last_diff_sha: last_diff_sha) - end + quick_action_target.persisted? && + merge_orchestration_service.can_merge?(quick_action_target) end command :merge do @updates[:merge] = params[:merge_request_diff_head_sha] end + types MergeRequest + desc do + _('Rebase source branch') + end + explanation do + _('Rebase source branch on the target branch.') + end + condition do + merge_request = quick_action_target + + next false unless merge_request.open? + next false unless merge_request.source_branch_exists? + + access_check = ::Gitlab::UserAccess + .new(current_user, container: merge_request.source_project) + + access_check.can_push_to_branch?(merge_request.source_branch) + end + command :rebase do + if quick_action_target.cannot_be_merged? + @execution_message[:rebase] = _('This merge request cannot be rebased while there are conflicts.') + next + end + + if quick_action_target.rebase_in_progress? + @execution_message[:rebase] = _('A rebase is already in progress.') + next + end + + # This will be used to avoid simultaneous "/merge" and "/rebase" actions + @updates[:rebase] = true + + branch = quick_action_target.source_branch + + @execution_message[:rebase] = _('Scheduled a rebase of branch %{branch}.') % { branch: branch } + end + desc 'Toggle the Draft status' explanation do noun = quick_action_target.to_ability_name.humanize(capitalize: false) @@ -135,6 +154,112 @@ module Gitlab @execution_message[:approve] = _('Approved the current merge request.') end + + desc do + if quick_action_target.allows_multiple_reviewers? + _('Assign reviewer(s)') + else + _('Assign reviewer') + end + end + explanation do |users| + reviewers = reviewers_to_add(users) + _('Assigns %{reviewer_users_sentence} as %{reviewer_text}.') % { reviewer_users_sentence: reviewer_users_sentence(users), + reviewer_text: 'reviewer'.pluralize(reviewers.size) } + end + execution_message do |users = nil| + reviewers = reviewers_to_add(users) + if reviewers.blank? + _("Failed to assign a reviewer because no user was found.") + else + _('Assigned %{reviewer_users_sentence} as %{reviewer_text}.') % { reviewer_users_sentence: reviewer_users_sentence(users), + reviewer_text: 'reviewer'.pluralize(reviewers.size) } + end + end + params do + quick_action_target.allows_multiple_reviewers? ? '@user1 @user2' : '@user' + end + types MergeRequest + condition do + Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |reviewer_param| + extract_users(reviewer_param) + end + command :assign_reviewer, :reviewer do |users| + next if users.empty? + + if quick_action_target.allows_multiple_reviewers? + @updates[:reviewer_ids] ||= quick_action_target.reviewers.map(&:id) + @updates[:reviewer_ids] |= users.map(&:id) + else + @updates[:reviewer_ids] = [users.first.id] + end + end + + desc do + if quick_action_target.allows_multiple_reviewers? + _('Remove all or specific reviewer(s)') + else + _('Remove reviewer') + end + end + explanation do |users = nil| + reviewers = reviewers_for_removal(users) + _("Removes %{reviewer_text} %{reviewer_references}.") % + { reviewer_text: 'reviewer'.pluralize(reviewers.size), reviewer_references: reviewers.map(&:to_reference).to_sentence } + end + execution_message do |users = nil| + reviewers = reviewers_for_removal(users) + _("Removed %{reviewer_text} %{reviewer_references}.") % + { reviewer_text: 'reviewer'.pluralize(reviewers.size), reviewer_references: reviewers.map(&:to_reference).to_sentence } + end + params do + quick_action_target.allows_multiple_reviewers? ? '@user1 @user2' : '' + end + types MergeRequest + condition do + quick_action_target.persisted? && + Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && + quick_action_target.reviewers.any? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |unassign_reviewer_param| + # When multiple users are assigned, all will be unassigned if multiple reviewers are no longer allowed + extract_users(unassign_reviewer_param) if quick_action_target.allows_multiple_reviewers? + end + command :unassign_reviewer, :remove_reviewer do |users = nil| + if quick_action_target.allows_multiple_reviewers? && users&.any? + @updates[:reviewer_ids] ||= quick_action_target.reviewers.map(&:id) + @updates[:reviewer_ids] -= users.map(&:id) + else + @updates[:reviewer_ids] = [] + end + end + end + + def reviewer_users_sentence(users) + reviewers_to_add(users).map(&:to_reference).to_sentence + end + + def reviewers_for_removal(users) + reviewers = quick_action_target.reviewers + if users.present? && quick_action_target.allows_multiple_reviewers? + users + else + reviewers + end + end + + def reviewers_to_add(users) + return if users.blank? + + if quick_action_target.allows_multiple_reviewers? + users + else + [users.first] + end end def merge_orchestration_service diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 7c336153e32..2a94fb91880 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -10,14 +10,70 @@ module Gitlab def self.configure(rack_attack) # This adds some methods used by our throttles to the `Rack::Request` rack_attack::Request.include(Gitlab::RackAttack::Request) - # Send the Retry-After header so clients (e.g. python-gitlab) can make good choices about delays - Rack::Attack.throttled_response_retry_after_header = true + + # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response + Rack::Attack.throttled_response = lambda do |env| + throttled_headers = Gitlab::RackAttack.throttled_response_headers( + env['rack.attack.matched'], env['rack.attack.match_data'] + ) + [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] + end + # Configure the throttles configure_throttles(rack_attack) configure_user_allowlist end + # Rate Limit HTTP headers are not standardized anywhere. This is the latest + # draft submitted to IETF: + # https://github.com/ietf-wg-httpapi/ratelimit-headers/blob/main/draft-ietf-httpapi-ratelimit-headers.md + # + # This method implement the most viable parts of the headers. Those headers + # will be sent back to the client when it gets throttled. + # + # - RateLimit-Limit: indicates the request quota associated to the client + # in 60 seconds. The time window for the quota here is supposed to be + # mirrored to throttle_*_period_in_seconds application settings. However, + # our HAProxy as well as some ecosystem libraries are using a fixed + # 60-second window. Therefore, the returned limit is approximately rounded + # up to fit into that window. + # + # - RateLimit-Observed: indicates the current request amount associated to + # the client within the time window. + # + # - RateLimit-Remaining: indicates the remaining quota within the time + # window. It is the result of RateLimit-Limit - RateLimit-Remaining + # + # - Retry-After: the remaining duration in seconds until the quota is + # reset. This is a standardized HTTP header: + # https://tools.ietf.org/html/rfc7231#page-69 + # + # - RateLimit-Reset: the point of time that the request quota is reset, in Unix time + # + # - RateLimit-ResetTime: the point of time that the request quota is reset, in HTTP date format + def self.throttled_response_headers(matched, match_data) + # Match data example: + # {:discriminator=>"127.0.0.1", :count=>12, :period=>60 seconds, :limit=>1, :epoch_time=>1609833930} + # Source: https://github.com/rack/rack-attack/blob/v6.3.0/lib/rack/attack/throttle.rb#L33 + period = match_data[:period] + limit = match_data[:limit] + rounded_limit = (limit.to_f * 1.minute / match_data[:period]).ceil + observed = match_data[:count] + now = match_data[:epoch_time] + retry_after = period - (now % period) + reset_time = Time.at(now + retry_after) # rubocop:disable Rails/TimeZone + { + 'RateLimit-Name' => matched.to_s, + 'RateLimit-Limit' => rounded_limit.to_s, + 'RateLimit-Observed' => observed.to_s, + 'RateLimit-Remaining' => (limit > observed ? limit - observed : 0).to_s, + 'RateLimit-Reset' => reset_time.to_i.to_s, + 'RateLimit-ResetTime' => reset_time.httpdate, + 'Retry-After' => retry_after.to_s + } + end + def self.configure_user_allowlist @user_allowlist = nil user_allowlist diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index 231d5aea129..7ef6ab32bd4 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -13,7 +13,8 @@ module Gitlab end def feature_enabled?(actor = nil) - feature.enabled?(actor) + # Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around. + Feature.enabled?(:sourcegraph, actor) end private diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index e84937ec4ad..b659bff52ad 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -23,7 +23,12 @@ module Gitlab end def content - @finder.read(@path) + blob = @finder.read(@path) + [description, blob].compact.join("\n") + end + + def description + # override with a comment to be placed at the top of the blob. end # Present for compatibility with license templates, which can replace text diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index 3b516bb862a..09643cfb619 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -3,9 +3,8 @@ module Gitlab module Template class DockerfileTemplate < BaseTemplate - def content - explanation = "# This file is a template, and might need editing before it works on your project." - [explanation, super].join("\n") + def description + "# This file is a template, and might need editing before it works on your project." end class << self diff --git a/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb b/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb new file mode 100644 index 00000000000..3bf3a28d3c5 --- /dev/null +++ b/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Template + class GitlabCiSyntaxYmlTemplate < BaseTemplate + class << self + def extension + '.gitlab-ci.yml' + end + + def categories + { + 'General' => '' + } + end + + def base_dir + Rails.root.join('lib/gitlab/ci/syntax_templates') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new( + self.base_dir, self.extension, self.categories + ) + end + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index e12af6bf0a4..c295cc75da5 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -5,9 +5,8 @@ module Gitlab class GitlabCiYmlTemplate < BaseTemplate BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze - def content - explanation = "# This file is a template, and might need editing before it works on your project." - [explanation, super].join("\n") + def description + "# This file is a template, and might need editing before it works on your project." end class << self diff --git a/lib/gitlab/template/metrics_dashboard_template.rb b/lib/gitlab/template/metrics_dashboard_template.rb index 88fc3007b63..469f97d7cb1 100644 --- a/lib/gitlab/template/metrics_dashboard_template.rb +++ b/lib/gitlab/template/metrics_dashboard_template.rb @@ -3,9 +3,8 @@ module Gitlab module Template class MetricsDashboardTemplate < BaseTemplate - def content - explanation = "# This file is a template, and might need editing before it works on your project." - [explanation, super].join("\n") + def description + "# This file is a template, and might need editing before it works on your project." end class << self diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb index aebf8d92cb3..520075012e8 100644 --- a/lib/gitlab/throttle.rb +++ b/lib/gitlab/throttle.rb @@ -2,6 +2,8 @@ module Gitlab class Throttle + DEFAULT_RATE_LIMITING_RESPONSE_TEXT = 'Retry later' + def self.settings Gitlab::CurrentSettings.current_application_settings end @@ -46,5 +48,9 @@ module Gitlab { limit: limit_proc, period: period_proc } end + + def self.rate_limiting_response_text + (settings.rate_limiting_response_text.presence || DEFAULT_RATE_LIMITING_RESPONSE_TEXT) + "\n" + end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 618e359211b..ca4afb4c19c 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -24,7 +24,9 @@ module Gitlab Gitlab::CurrentSettings.snowplow_enabled? end - def event(category, action, label: nil, property: nil, value: nil, context: nil) + def event(category, action, label: nil, property: nil, value: nil, context: [], standard_context: nil) + context.push(standard_context.to_context) if standard_context + snowplow.event(category, action, label: label, property: property, value: value, context: context) product_analytics.event(category, action, label: label, property: property, value: value, context: context) end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb new file mode 100644 index 00000000000..71dfe27dd5a --- /dev/null +++ b/lib/gitlab/tracking/standard_context.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + class StandardContext + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-1'.freeze + + def initialize(namespace: nil, project: nil, **data) + @namespace = namespace + @project = project + @data = data + end + + def namespace_id + namespace&.id + end + + def project_id + @project&.id + end + + def to_context + SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) + end + + private + + def namespace + @namespace || @project&.namespace + end + + def to_h + public_methods(false).each_with_object({}) do |method, hash| + next if method == :to_context + + hash[method] = public_send(method) # rubocop:disable GitlabSecurity/PublicSend + end.merge(@data) + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index ce59e10241e..f98c488bbe5 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -18,6 +18,8 @@ module Gitlab def build(object, **options) # Objects are sometimes wrapped in a BatchLoader instance case object.itself + when Board + board_url(object, **options) when ::Ci::Build instance.project_job_url(object.project, object, **options) when Commit @@ -52,6 +54,14 @@ module Gitlab end # rubocop:enable Metrics/CyclomaticComplexity + def board_url(board, **options) + if board.project_board? + instance.project_board_url(board.resource_parent, board, **options) + else + instance.group_board_url(board.resource_parent, board, **options) + end + end + def commit_url(commit, **options) return '' unless commit.project diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb new file mode 100644 index 00000000000..e1648c78168 --- /dev/null +++ b/lib/gitlab/usage/metric.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + class Metric + include ActiveModel::Model + + InvalidMetricError = Class.new(RuntimeError) + + attr_accessor :default_generation_path, :value + + validates :default_generation_path, presence: true + + def definition + self.class.definitions[default_generation_path] + end + + def unflatten_default_path + unflatten(default_generation_path.split('.'), value) + end + + class << self + def definitions + @definitions ||= Gitlab::Usage::MetricDefinition.definitions + end + + def dictionary + definitions.map { |key, definition| definition.to_dictionary } + end + end + + private + + def unflatten(keys, value) + loop do + value = { keys.pop.to_sym => value } + break if keys.blank? + end + value + end + end + end +end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb new file mode 100644 index 00000000000..96e572bb3db --- /dev/null +++ b/lib/gitlab/usage/metric_definition.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + class MetricDefinition + METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') + + attr_reader :path + attr_reader :attributes + + def initialize(path, opts = {}) + @path = path + @attributes = opts + end + + # The key is defined by default_generation and full_path + def key + full_path[default_generation.to_sym] + end + + def to_h + attributes + end + + def validate! + self.class.schemer.validate(attributes.stringify_keys).map do |error| + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + end + end + + alias_method :to_dictionary, :to_h + + class << self + def paths + @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')] + end + + def definitions + @definitions ||= load_all! + end + + def schemer + @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) + end + + private + + def load_all! + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + + def load_from_file(path) + definition = File.read(path) + definition = YAML.safe_load(definition) + definition.deep_symbolize_keys! + + self.new(path, definition).tap(&:validate!) + rescue => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new(e.message)) + end + + def load_all_from_path!(definitions, glob_path) + Dir.glob(glob_path).each do |path| + definition = load_from_file(path) + + if previous = definitions[definition.key] + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'")) + end + + definitions[definition.key] = definition + end + end + end + + private + + def method_missing(method, *args) + attributes[method] || super + end + end + end +end + +Gitlab::Usage::MetricDefinition.prepend_if_ee('EE::Gitlab::Usage::MetricDefinition') diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index ca7699e64e1..ed9dad37f3e 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -3,7 +3,7 @@ module Gitlab module UsageDataCounters COUNTERS = [ - GuestPackageEventCounter, + PackageEventCounter, WikiPageCounter, WebIdeCounter, NoteCounter, diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index b7c0abae227..4966afd534a 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -8,6 +8,9 @@ # Corresponding feature flag should have `default_enabled` attribute set to `false`. # This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. --- +- name: compliance_features_track_unique_visits_union + operator: OR + events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] - name: product_analytics_test_metrics_union operator: OR events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] @@ -22,7 +25,6 @@ 'incident_management_alert_todo', 'incident_management_alert_create_incident' ] - feature_flag: usage_data_incident_management_alerts_total_unique_counts - name: incident_management_incidents_total_unique_counts operator: OR events: [ @@ -38,4 +40,3 @@ 'incident_management_incident_unrelate', 'incident_management_incident_change_confidential' ] - feature_flag: usage_data_incident_management_incidents_total_unique_counts diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb new file mode 100644 index 00000000000..572ad866895 --- /dev/null +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class CiTemplateUniqueCounter + REDIS_SLOT = 'ci_templates'.freeze + + TEMPLATE_TO_EVENT = { + 'Auto-DevOps.gitlab-ci.yml' => 'auto_devops', + 'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' => 'aws_cf_deploy_ec2', + 'AWS/Deploy-ECS.gitlab-ci.yml' => 'aws_deploy_ecs', + 'Jobs/Build.gitlab-ci.yml' => 'auto_devops_build', + 'Jobs/Deploy.gitlab-ci.yml' => 'auto_devops_deploy', + 'Jobs/Deploy.latest.gitlab-ci.yml' => 'auto_devops_deploy_latest', + 'Security/SAST.gitlab-ci.yml' => 'security_sast', + 'Security/Secret-Detection.gitlab-ci.yml' => 'security_secret_detection', + 'Terraform/Base.latest.gitlab-ci.yml' => 'terraform_base_latest' + }.freeze + + class << self + def track_unique_project_event(project_id:, template:) + return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml) + + if event = unique_project_event(template) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) + end + end + + private + + def unique_project_event(template) + if name = TEMPLATE_TO_EVENT[template] + "p_#{REDIS_SLOT}_#{name}" + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/counter_events/guest_package_events.yml b/lib/gitlab/usage_data_counters/counter_events/guest_package_events.yml deleted file mode 100644 index a9b9f8ea235..00000000000 --- a/lib/gitlab/usage_data_counters/counter_events/guest_package_events.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- i_package_composer_guest_delete -- i_package_composer_guest_pull -- i_package_composer_guest_push -- i_package_conan_guest_delete -- i_package_conan_guest_pull -- i_package_conan_guest_push -- i_package_container_guest_delete -- i_package_container_guest_pull -- i_package_container_guest_push -- i_package_debian_guest_delete -- i_package_debian_guest_pull -- i_package_debian_guest_push -- i_package_generic_guest_delete -- i_package_generic_guest_pull -- i_package_generic_guest_push -- i_package_golang_guest_delete -- i_package_golang_guest_pull -- i_package_golang_guest_push -- i_package_maven_guest_delete -- i_package_maven_guest_pull -- i_package_maven_guest_push -- i_package_npm_guest_delete -- i_package_npm_guest_pull -- i_package_npm_guest_push -- i_package_nuget_guest_delete -- i_package_nuget_guest_pull -- i_package_nuget_guest_push -- i_package_pypi_guest_delete -- i_package_pypi_guest_pull -- i_package_pypi_guest_push -- i_package_tag_guest_delete -- i_package_tag_guest_pull -- i_package_tag_guest_push diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml new file mode 100644 index 00000000000..f6bddabdd44 --- /dev/null +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -0,0 +1,46 @@ +--- +- i_package_composer_delete_package +- i_package_composer_pull_package +- i_package_composer_push_package +- i_package_conan_delete_package +- i_package_conan_pull_package +- i_package_conan_push_package +- i_package_container_delete_package +- i_package_container_pull_package +- i_package_container_push_package +- i_package_debian_delete_package +- i_package_debian_pull_package +- i_package_debian_push_package +- i_package_delete_package +- i_package_delete_package_by_deploy_token +- i_package_delete_package_by_guest +- i_package_delete_package_by_user +- i_package_generic_delete_package +- i_package_generic_pull_package +- i_package_generic_push_package +- i_package_golang_delete_package +- i_package_golang_pull_package +- i_package_golang_push_package +- i_package_maven_delete_package +- i_package_maven_pull_package +- i_package_maven_push_package +- i_package_npm_delete_package +- i_package_npm_pull_package +- i_package_npm_push_package +- i_package_nuget_delete_package +- i_package_nuget_pull_package +- i_package_nuget_push_package +- i_package_pull_package +- i_package_pull_package_by_deploy_token +- i_package_pull_package_by_guest +- i_package_pull_package_by_user +- i_package_push_package +- i_package_push_package_by_deploy_token +- i_package_push_package_by_guest +- i_package_push_package_by_user +- i_package_pypi_delete_package +- i_package_pypi_pull_package +- i_package_pypi_push_package +- i_package_tag_delete_package +- i_package_tag_pull_package +- i_package_tag_push_package diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index eeb26c11bfa..bef3fc7b504 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -53,7 +53,7 @@ module Gitlab return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) end def count_unique(actions, date_from, date_to) diff --git a/lib/gitlab/usage_data_counters/guest_package_event_counter.rb b/lib/gitlab/usage_data_counters/guest_package_event_counter.rb deleted file mode 100644 index a9bcbfadda2..00000000000 --- a/lib/gitlab/usage_data_counters/guest_package_event_counter.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - class GuestPackageEventCounter < BaseCounter - KNOWN_EVENTS_PATH = File.expand_path('counter_events/guest_package_events.yml', __dir__) - KNOWN_EVENTS = YAML.safe_load(File.read(KNOWN_EVENTS_PATH)).freeze - PREFIX = 'package_guest' - end - end -end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index b61720c7638..47361d831b2 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -39,20 +39,31 @@ module Gitlab # # Usage: # - # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard') + # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_dashboard', values: user_id) # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self include Gitlab::Utils::UsageData - def track_event(value, event_name, time = Time.zone.now) - track(value, event_name, time: time) - end - - def track_event_in_context(value, event_name, context, time = Time.zone.now) + # Track unique events + # + # event_name - The event name. + # values - One or multiple values counted. + # time - Time of the action, set to Time.current. + def track_event(event_name, values:, time: Time.current) + track(values, event_name, time: time) + end + + # Track unique events + # + # event_name - The event name. + # values - One or multiple values counted. + # context - Event context, plan level tracking. + # time - Time of the action, set to Time.current. + def track_event_in_context(event_name, values:, context:, time: Time.zone.now) return if context.blank? return unless context.in?(valid_context_list) - track(value, event_name, context: context, time: time) + track(values, event_name, context: context, time: time) end def unique_events(event_names:, start_date:, end_date:, context: '') @@ -114,16 +125,16 @@ module Gitlab private - def track(value, event_name, context: '', time: Time.zone.now) + def track(values, event_name, context: '', time: Time.zone.now) return unless Gitlab::CurrentSettings.usage_ping_enabled? event = event_for(event_name) raise UnknownEvent, "Unknown event #{event_name}" unless event.present? - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: value, expiry: expiry(event)) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) end - # The aray of valid context on which we allow tracking + # The array of valid context on which we allow tracking def valid_context_list Plan.all_plans end diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 0fed8e1c211..f649e7f407d 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -148,7 +148,7 @@ module Gitlab return unless Feature.enabled?(:track_issue_activity_actions, default_enabled: true) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) end end end diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 25cf388aedf..4cbde0c0372 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -248,6 +248,26 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_test_case_parsed +- name: i_testing_metrics_report_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_metrics_report_widget_total +- name: i_testing_group_code_coverage_visit_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_group_code_coverage_visit_total +- name: i_testing_full_code_quality_report_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_full_code_quality_report_total +- name: i_testing_web_performance_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_web_performance_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -425,3 +445,126 @@ redis_slot: snippets aggregation: weekly feature_flag: usage_data_i_snippets_show +# Merge request counters +- name: i_code_review_mr_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_diffs +- name: i_code_review_user_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_single_file_diffs +- name: i_code_review_mr_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_single_file_diffs +- name: i_code_review_user_create_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr +- name: i_code_review_user_close_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_close_mr +- name: i_code_review_user_reopen_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_reopen_mr +- name: i_code_review_user_merge_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_merge_mr +- name: i_code_review_user_create_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr_comment +- name: i_code_review_user_edit_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_mr_comment +- name: i_code_review_user_remove_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_mr_comment +- name: i_code_review_user_create_review_note + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_review_note +- name: i_code_review_user_publish_review + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_publish_review +- name: i_code_review_user_create_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment +- name: i_code_review_user_edit_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment +- name: i_code_review_user_remove_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment +# Terraform +- name: p_terraform_state_api_unique_users + category: terraform + redis_slot: terraform + aggregation: weekly + feature_flag: usage_data_p_terraform_state_api_unique_users +# CI templates +- name: p_ci_templates_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_aws_cf_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_auto_devops_deploy_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_terraform_base_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index 4c3138dc000..78a2a587b34 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -1,331 +1,111 @@ --- -- name: i_package_composer_deploy_token_delete - category: composer_packages +- name: i_package_composer_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_deploy_token_pull - category: composer_packages +- name: i_package_composer_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_deploy_token_push - category: composer_packages +- name: i_package_conan_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_user_delete - category: composer_packages +- name: i_package_conan_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_user_pull - category: composer_packages +- name: i_package_container_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_user_push - category: composer_packages +- name: i_package_container_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_deploy_token_delete - category: conan_packages +- name: i_package_debian_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_deploy_token_pull - category: conan_packages +- name: i_package_debian_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_deploy_token_push - category: conan_packages +- name: i_package_generic_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_user_delete - category: conan_packages +- name: i_package_generic_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_user_pull - category: conan_packages +- name: i_package_golang_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_user_push - category: conan_packages +- name: i_package_golang_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_deploy_token_delete - category: container_packages +- name: i_package_maven_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_deploy_token_pull - category: container_packages +- name: i_package_maven_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_deploy_token_push - category: container_packages +- name: i_package_npm_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_user_delete - category: container_packages +- name: i_package_npm_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_user_pull - category: container_packages +- name: i_package_nuget_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_user_push - category: container_packages +- name: i_package_nuget_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_deploy_token_delete - category: debian_packages +- name: i_package_pypi_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_deploy_token_pull - category: debian_packages +- name: i_package_pypi_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_deploy_token_push - category: debian_packages +- name: i_package_tag_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_user_delete - category: debian_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_debian_user_pull - category: debian_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_debian_user_push - category: debian_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_deploy_token_delete - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_deploy_token_pull - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_deploy_token_push - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_user_delete - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_user_pull - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_user_push - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_deploy_token_delete - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_deploy_token_pull - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_deploy_token_push - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_user_delete - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_user_pull - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_user_push - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_deploy_token_delete - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_deploy_token_pull - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_deploy_token_push - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_user_delete - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_user_pull - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_user_push - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_deploy_token_delete - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_deploy_token_pull - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_deploy_token_push - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_user_delete - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_user_pull - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_user_push - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_deploy_token_delete - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_deploy_token_pull - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_deploy_token_push - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_user_delete - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_user_pull - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_user_push - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_deploy_token_delete - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_deploy_token_pull - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_deploy_token_push - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_user_delete - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_user_pull - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_user_push - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_deploy_token_delete - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_deploy_token_pull - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_deploy_token_push - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_user_delete - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_user_pull - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_user_push - category: tag_packages +- name: i_package_tag_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb new file mode 100644 index 00000000000..11d59257ed9 --- /dev/null +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module MergeRequestActivityUniqueCounter + MR_DIFFS_ACTION = 'i_code_review_mr_diffs' + MR_DIFFS_SINGLE_FILE_ACTION = 'i_code_review_mr_single_file_diffs' + MR_DIFFS_USER_SINGLE_FILE_ACTION = 'i_code_review_user_single_file_diffs' + MR_CREATE_ACTION = 'i_code_review_user_create_mr' + MR_CLOSE_ACTION = 'i_code_review_user_close_mr' + MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' + MR_MERGE_ACTION = 'i_code_review_user_merge_mr' + MR_CREATE_COMMENT_ACTION = 'i_code_review_user_create_mr_comment' + MR_EDIT_COMMENT_ACTION = 'i_code_review_user_edit_mr_comment' + MR_REMOVE_COMMENT_ACTION = 'i_code_review_user_remove_mr_comment' + MR_CREATE_REVIEW_NOTE_ACTION = 'i_code_review_user_create_review_note' + MR_PUBLISH_REVIEW_ACTION = 'i_code_review_user_publish_review' + MR_CREATE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_create_multiline_mr_comment' + MR_EDIT_MULTILINE_COMMENT_ACTION = 'i_code_review_user_edit_multiline_mr_comment' + MR_REMOVE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_remove_multiline_mr_comment' + + class << self + def track_mr_diffs_action(merge_request:) + track_unique_action_by_merge_request(MR_DIFFS_ACTION, merge_request) + end + + def track_mr_diffs_single_file_action(merge_request:, user:) + track_unique_action_by_merge_request(MR_DIFFS_SINGLE_FILE_ACTION, merge_request) + track_unique_action_by_user(MR_DIFFS_USER_SINGLE_FILE_ACTION, user) + end + + def track_create_mr_action(user:) + track_unique_action_by_user(MR_CREATE_ACTION, user) + end + + def track_close_mr_action(user:) + track_unique_action_by_user(MR_CLOSE_ACTION, user) + end + + def track_merge_mr_action(user:) + track_unique_action_by_user(MR_MERGE_ACTION, user) + end + + def track_reopen_mr_action(user:) + track_unique_action_by_user(MR_REOPEN_ACTION, user) + end + + def track_create_comment_action(note:) + track_unique_action_by_user(MR_CREATE_COMMENT_ACTION, note.author) + track_multiline_unique_action(MR_CREATE_MULTILINE_COMMENT_ACTION, note) + end + + def track_edit_comment_action(note:) + track_unique_action_by_user(MR_EDIT_COMMENT_ACTION, note.author) + track_multiline_unique_action(MR_EDIT_MULTILINE_COMMENT_ACTION, note) + end + + def track_remove_comment_action(note:) + track_unique_action_by_user(MR_REMOVE_COMMENT_ACTION, note.author) + track_multiline_unique_action(MR_REMOVE_MULTILINE_COMMENT_ACTION, note) + end + + def track_create_review_note_action(user:) + track_unique_action_by_user(MR_CREATE_REVIEW_NOTE_ACTION, user) + end + + def track_publish_review_action(user:) + track_unique_action_by_user(MR_PUBLISH_REVIEW_ACTION, user) + end + + private + + def track_unique_action_by_merge_request(action, merge_request) + track_unique_action(action, merge_request.id) + end + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + + def track_multiline_unique_action(action, note) + return unless note.is_a?(DiffNote) && note.multiline? + + track_unique_action_by_user(action, note.author) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/package_event_counter.rb b/lib/gitlab/usage_data_counters/package_event_counter.rb new file mode 100644 index 00000000000..700b518eae3 --- /dev/null +++ b/lib/gitlab/usage_data_counters/package_event_counter.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class PackageEventCounter < BaseCounter + KNOWN_EVENTS_PATH = File.expand_path('counter_events/package_events.yml', __dir__) + KNOWN_EVENTS = YAML.safe_load(File.read(KNOWN_EVENTS_PATH)).freeze + PREFIX = 'package_events' + end + end +end diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 95380ae0b1d..20da9665876 100644 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -43,7 +43,7 @@ module Gitlab return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(transformed_action.to_s, values: author_id, time: time) track_git_write_action(author_id, transformed_action, time) end @@ -73,7 +73,7 @@ module Gitlab def track_git_write_action(author_id, transformed_action, time) return unless GIT_WRITE_ACTIONS.include?(transformed_action) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, GIT_WRITE_ACTION, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(GIT_WRITE_ACTION, values: author_id, time: time) end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 3df54e74b4f..29f02a5912a 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -174,6 +174,18 @@ module Gitlab rescue IPAddr::InvalidAddressError end + # A safe alternative to String#downcase! + # + # This will make copies of frozen strings but downcase unfrozen + # strings in place, reducing allocations. + def safe_downcase!(str) + if str.frozen? + str.downcase + else + str.downcase! || str + end + end + # Converts a string to an Addressable::URI object. # If the string is not a valid URI, it returns nil. # Param uri_string should be a String object. diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 0d28a1cd035..baccadd9594 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -61,7 +61,10 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::PostgresHll::BatchDistinctCounter.new(relation, column).estimate_distinct_count(batch_size: batch_size, start: start, finish: finish) + Gitlab::Database::PostgresHll::BatchDistinctCounter + .new(relation, column) + .execute(batch_size: batch_size, start: start, finish: finish) + .estimated_distinct_count rescue ActiveRecord::StatementInvalid FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue @@ -119,7 +122,7 @@ module Gitlab def track_usage_event(event_name, values) return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end private @@ -142,7 +145,8 @@ module Gitlab def prometheus_server_address if Gitlab::Prometheus::Internal.prometheus_enabled? - Gitlab::Prometheus::Internal.server_address + # Stripping protocol from URI + Gitlab::Prometheus::Internal.uri&.strip&.sub(%r{^https?://}, '') elsif Gitlab::Consul::Internal.api_url Gitlab::Consul::Internal.discover_prometheus_server_address end diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index 12a4efabc44..80caf2c6788 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -9,6 +9,7 @@ module Gitlab production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze + UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze PACK_PATTERN = "NnnnnN".freeze @@ -17,6 +18,10 @@ module Gitlab Digest::UUID.uuid_v5(namespace_id, name) end + def v5?(string) + string.match(UUID_V5_PATTERN).present? + end + private def default_namespace_id diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index a22740ab9b7..76cf769d041 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -123,14 +123,6 @@ module Gitlab end end - def visibility_level_decreased? - return false unless visibility_level_previous_changes - - before, after = visibility_level_previous_changes - - before && after && after < before - end - def visibility_level_previous_changes previous_changes[:visibility_level] end diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index 5873d9c2b99..9c967d99e3a 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -69,8 +69,8 @@ module Gitlab def manifest if Gitlab.config.webpack.dev_server.enabled - # Don't cache if we're in dev server mode, manifest may change ... - load_manifest + # Only cache at request level if we're in dev server mode, manifest may change ... + Gitlab::SafeRequestStore.fetch('manifest.json') { load_manifest } else # ... otherwise cache at class level, as JSON loading/parsing can be expensive strong_memoize(:manifest) { load_manifest } diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index ec9dd20ccc0..b0974e02edd 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -11,7 +11,7 @@ class GitlabDanger karma database commit_messages - product_analytics + product_intelligence utility_css pajamas pipeline diff --git a/lib/release_highlights/validator.rb b/lib/release_highlights/validator.rb new file mode 100644 index 00000000000..6f3f90b5f30 --- /dev/null +++ b/lib/release_highlights/validator.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module ReleaseHighlights + class Validator + attr_reader :errors, :file + + def initialize(file:) + @file = file + @errors = [] + end + + def valid? + document = YAML.parse(File.read(file)) + + document.root.children.each do |entry| + entry = ReleaseHighlights::Validator::Entry.new(entry) + + errors.push(entry.errors.full_messages) unless entry.valid? + end + + errors.none? + end + + def self.validate_all! + @all_errors = [] + + ReleaseHighlight.file_paths.each do |file_path| + instance = self.new(file: file_path) + + @all_errors.push([instance.errors, instance.file]) unless instance.valid? + end + + @all_errors.none? + end + + def self.error_message + io = StringIO.new + + @all_errors.each do |errors, file| + message = "Validation failed for #{file}" + line = -> { io.puts "-" * message.length } + + line.call + io.puts message + line.call + + errors.flatten.each { |error| io.puts "* #{error}" } + io.puts + end + + io.string + end + end +end diff --git a/lib/release_highlights/validator/entry.rb b/lib/release_highlights/validator/entry.rb new file mode 100644 index 00000000000..0dbe0cdf882 --- /dev/null +++ b/lib/release_highlights/validator/entry.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module ReleaseHighlights + class Validator::Entry + include ActiveModel::Validations + include ActiveModel::Validations::Callbacks + + PACKAGES = %w(Core Starter Premium Ultimate).freeze + + attr_reader :entry + + validates :title, :body, :stage, presence: true + validates :'self-managed', :'gitlab-com', inclusion: { in: [true, false], message: "must be a boolean" } + validates :url, :image_url, format: { with: URI::DEFAULT_PARSER.make_regexp, message: 'must be a URL' } + validates :release, numericality: true + validate :validate_published_at + validate :validate_packages + + after_validation :add_line_numbers_to_errors! + + def initialize(entry) + @entry = entry + end + + def validate_published_at + published_at = value_for('published_at') + + return if published_at.is_a?(Date) + + errors.add(:published_at, 'must be valid Date') + end + + def validate_packages + packages = value_for('packages') + + if !packages.is_a?(Array) || packages.empty? || packages.any? { |p| PACKAGES.exclude?(p) } + errors.add(:packages, "must be one of #{PACKAGES}") + end + end + + def read_attribute_for_validation(key) + value_for(key) + end + + private + + def add_line_numbers_to_errors! + errors.messages.each do |attribute, messages| + messages.map! { |m| "#{m} (line #{line_number_for(attribute)})" } + end + end + + def line_number_for(key) + node = find_node(key) + + (node&.start_line || @entry.start_line) + 1 + end + + def value_for(key) + node = find_node(key) + + return if node.nil? + + index = entry.children.find_index(node) + + next_node = entry.children[index + 1] + next_node&.to_ruby + end + + def find_node(key) + entry.children.find {|node| node.try(:value) == key.to_s } + end + end +end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 8a53b51d4fe..abb15f29328 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -21,5 +21,39 @@ namespace :gitlab do failures.each { |f| puts "- #{f}" } end end + + # Example for all projects: + # + # $ bin/rake gitlab:git:checksum_projects + # 1,cfa3f06ba235c13df0bb28e079bcea62c5848af2 + # 2, + # 3,3f3fb58a8106230e3a6c6b48adc2712fb3b6ef87 + # 4,0000000000000000000000000000000000000000 + # + # Example with a list of project IDs: + # + # $ CHECKSUM_PROJECT_IDS="1,3" bin/rake gitlab:git:checksum_projects + # 1,cfa3f06ba235c13df0bb28e079bcea62c5848af2 + # 3,3f3fb58a8106230e3a6c6b48adc2712fb3b6ef87 + # + # - If a repository does not exist, the project ID is output with a blank checksum + # - If a repository exists but is empty, the output checksum is `0000000000000000000000000000000000000000` + # - If given specific IDs, projects which do not exist are skipped + desc 'GitLab | Git | Generate checksum of project repository refs' + task checksum_projects: :environment do + project_ids = ENV['CHECKSUM_PROJECT_IDS']&.split(',') + relation = Project + relation = relation.where(id: project_ids) if project_ids.present? + + relation.find_each(batch_size: 100) do |project| + next unless project.repo_exists? + + result = project.repository.checksum + rescue => e + result = "Ignored error: #{e.message}".squish.truncate(255) + ensure + puts "#{project.id},#{result}" + end + end end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index 5a583183924..f708114c226 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -33,6 +33,44 @@ namespace :gitlab do ) namespace :graphql do + desc 'Gitlab | GraphQL | Validate queries' + task validate: [:environment, :enable_feature_flags] do |t, args| + queries = if args.to_a.present? + args.to_a.flat_map { |path| Gitlab::Graphql::Queries.find(path) } + else + Gitlab::Graphql::Queries.all + end + + failed = queries.flat_map do |defn| + summary, errs = defn.validate(GitlabSchema) + + case summary + when :client_query + warn("SKIP #{defn.file}: client query") + else + warn("OK #{defn.file}") if errs.empty? + errs.each do |err| + warn(<<~MSG) + ERROR #{defn.file}: #{err.message} (at #{err.path.join('.')}) + MSG + end + end + + errs.empty? ? [] : [defn.file] + end + + if failed.present? + format_output( + "#{failed.count} GraphQL #{'query'.pluralize(failed.count)} out of #{queries.count} failed validation:", + *failed.map do |name| + known_failure = Gitlab::Graphql::Queries.known_failure?(name) + "- #{name}" + (known_failure ? ' (known failure)' : '') + end + ) + abort unless failed.all? { |name| Gitlab::Graphql::Queries.known_failure?(name) } + end + end + desc 'GitLab | GraphQL | Generate GraphQL docs' task compile_docs: [:environment, :enable_feature_flags] do renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) @@ -78,11 +116,11 @@ def render_options } end -def format_output(str) +def format_output(*strs) heading = '#' * 10 puts heading puts '#' - puts "# #{str}" + strs.each { |str| puts "# #{str}" } puts '#' puts heading end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index ca507fb5320..cfe97984dda 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -5,18 +5,18 @@ namespace :gitlab do namespace :packages do namespace :events do task generate: :environment do - Rake::Task["gitlab:packages:events:generate_guest"].invoke + Rake::Task["gitlab:packages:events:generate_counts"].invoke Rake::Task["gitlab:packages:events:generate_unique"].invoke rescue => e logger.error("Error building events list: #{e}") end - task generate_guest: :environment do + task generate_counts: :environment do logger = Logger.new(STDOUT) logger.info('Building list of package events...') - path = Gitlab::UsageDataCounters::GuestPackageEventCounter::KNOWN_EVENTS_PATH - File.open(path, "w") { |file| file << guest_events_list.to_yaml } + path = Gitlab::UsageDataCounters::PackageEventCounter::KNOWN_EVENTS_PATH + File.open(path, "w") { |file| file << counter_events_list.to_yaml } logger.info("Events file `#{path}` generated successfully") rescue => e @@ -43,26 +43,32 @@ namespace :gitlab do def generate_unique_events_list events = event_pairs.each_with_object([]) do |(event_type, event_scope), events| - Packages::Event.originator_types.keys.excluding('guest').each do |originator| - if name = Packages::Event.allowed_event_name(event_scope, event_type, originator) - events << { - "name" => name, - "category" => "#{event_scope}_packages", + Packages::Event.originator_types.keys.excluding('guest').each do |originator_type| + events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name| + { + "name" => event_name, + "category" => "#{originator_type}_packages", "aggregation" => "weekly", "redis_slot" => "package", "feature_flag" => "collect_package_events_redis" } end + + events.concat(events_definition) end end - events.sort_by { |event| event["name"] } + events.sort_by { |event| event["name"] }.uniq end - def guest_events_list - event_pairs.map do |event_type, event_scope| - Packages::Event.allowed_event_name(event_scope, event_type, "guest") - end.compact.sort + def counter_events_list + counters = event_pairs.flat_map do |event_type, event_scope| + Packages::Event.originator_types.keys.flat_map do |originator_type| + Packages::Event.counters_for(event_scope, event_type, originator_type) + end + end + + counters.compact.sort.uniq end end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake new file mode 100644 index 00000000000..e15cbb4e32e --- /dev/null +++ b/lib/tasks/gitlab/pages.rake @@ -0,0 +1,35 @@ +require 'logger' + +namespace :gitlab do + namespace :pages do + desc "GitLab | Pages | Migrate legacy storage to zip format" + task migrate_legacy_storage: :gitlab_environment do + logger = Logger.new(STDOUT) + logger.info('Starting to migrate legacy pages storage to zip deployments') + processed_projects = 0 + + ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch| + batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum| + project = metadatum.project + + result = nil + time = Benchmark.realtime do + result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute + end + processed_projects += 1 + + if result[:status] == :success + logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds") + else + logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time} seconds: #{result[:message]}") + end + rescue => e + logger.error("#{e.message} project_id: #{project&.id}") + Gitlab::ErrorTracking.track_exception(e, project_id: project&.id) + end + + logger.info("#{processed_projects} pages projects are processed") + end + end + end +end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index ccc96b7edfb..f7819fd974b 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -116,6 +116,21 @@ namespace :gitlab do helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) end + desc 'Gitlab | Storage | Prune projects using Hashed Storage. Remove all hashed directories that do not have a project associated' + task prune_hashed_projects: [:environment, :gitlab_environment] do + if Rails.env.production? + abort('This destructive action may only be run in development') + end + + helper = Gitlab::HashedStorage::RakeHelper + name = 'projects using Hashed Storage' + relation = Project.with_storage_feature(:repository) + root = Gitlab.config.repositories.storages['default'].legacy_disk_path + dry_run = !ENV['FORCE'].present? + + helper.prune(name, relation, dry_run: dry_run, root: root) + end + desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' task legacy_attachments: :environment do helper = Gitlab::HashedStorage::RakeHelper diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 9b034d1c6c2..79920968603 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -42,10 +42,7 @@ class UploadedFile @remote_id = remote_id end - # TODO this function is meant to replace .from_params when the feature flag - # upload_middleware_jwt_params_handler is removed - # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps - def self.from_params_without_field(params, upload_paths) + def self.from_params(params, upload_paths) path = params['path'] remote_id = params['remote_id'] return if path.blank? && remote_id.blank? @@ -71,33 +68,6 @@ class UploadedFile ) end - # Deprecated. Don't use it. - # .from_params_without_field will replace this one - # See .from_params_without_field and - # https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps - def self.from_params(params, field, upload_paths, path_override = nil) - path = path_override || params["#{field}.path"] - remote_id = params["#{field}.remote_id"] - return if path.blank? && remote_id.blank? - - if remote_id.present? # don't use file_path if remote_id is set - file_path = nil - elsif path.present? - file_path = File.realpath(path) - - unless self.allowed_path?(file_path, Array(upload_paths).compact) - raise InvalidPathError, "insecure path used '#{file_path}'" - end - end - - UploadedFile.new(file_path, - filename: params["#{field}.name"], - content_type: params["#{field}.type"] || 'application/octet-stream', - sha256: params["#{field}.sha256"], - remote_id: remote_id, - size: params["#{field}.size"]) - end - def self.allowed_path?(file_path, paths) paths.any? do |path| File.exist?(path) && file_path.start_with?(File.realpath(path)) |