diff options
Diffstat (limited to 'lib')
441 files changed, 6244 insertions, 1893 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 5305b25538f..7e3d70a210a 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class AccessRequests < Grape::API::Instance + class AccessRequests < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb index 8721d94d642..44c389d6f94 100644 --- a/lib/api/admin/ci/variables.rb +++ b/lib/api/admin/ci/variables.rb @@ -3,7 +3,7 @@ module API module Admin module Ci - class Variables < Grape::API::Instance + class Variables < ::API::Base include PaginationParams before { authenticated_as_admin! } diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index 8208d10c089..ce1bdd65eff 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -2,7 +2,7 @@ module API module Admin - class InstanceClusters < Grape::API::Instance + class InstanceClusters < ::API::Base include PaginationParams before do @@ -37,6 +37,7 @@ module API requires :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' @@ -70,6 +71,7 @@ module API optional :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, desc: 'Enable or disable Gitlab\'s connection to your Kubernetes cluster' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb index f4c84f2eee8..c2e9de5fb4e 100644 --- a/lib/api/admin/sidekiq.rb +++ b/lib/api/admin/sidekiq.rb @@ -2,7 +2,7 @@ module API module Admin - class Sidekiq < Grape::API::Instance + class Sidekiq < ::API::Base before { authenticated_as_admin! } namespace 'admin' do diff --git a/lib/api/api.rb b/lib/api/api.rb index b37751e1b47..84b4d5a5835 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class API < Grape::API::Instance + class API < ::API::Base include APIGuard LOG_FILENAME = Rails.root.join("log", "api_json.log") @@ -153,6 +153,9 @@ module API mount ::API::Environments mount ::API::ErrorTracking mount ::API::Events + mount ::API::FeatureFlags + mount ::API::FeatureFlagScopes + mount ::API::FeatureFlagsUserLists mount ::API::Features mount ::API::Files mount ::API::FreezePeriods @@ -196,6 +199,8 @@ module API mount ::API::ComposerPackages mount ::API::ConanProjectPackages mount ::API::ConanInstancePackages + mount ::API::DebianGroupPackages + mount ::API::DebianProjectPackages mount ::API::MavenPackages mount ::API::NpmPackages mount ::API::GenericPackages @@ -216,6 +221,7 @@ module API mount ::API::ProjectStatistics mount ::API::ProjectTemplates mount ::API::Terraform::State + mount ::API::Terraform::StateVersion mount ::API::ProtectedBranches mount ::API::ProtectedTags mount ::API::Releases @@ -236,6 +242,7 @@ module API mount ::API::Templates mount ::API::Todos mount ::API::Triggers + mount ::API::Unleash mount ::API::UsageData mount ::API::UserCounts mount ::API::Users @@ -245,6 +252,7 @@ module API end mount ::API::Internal::Base + mount ::API::Internal::Lfs mount ::API::Internal::Pages mount ::API::Internal::Kubernetes diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index bf5044b4832..0a486307653 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -19,6 +19,7 @@ module API end use AdminModeMiddleware + use ResponseCoercerMiddleware helpers HelperMethods @@ -188,6 +189,44 @@ module API end end + # Prior to Rack v2.1.x, returning a body of [nil] or [201] worked + # because the body was coerced to a string. However, this no longer + # works in Rack v2.1.0+. The Rack spec + # (https://github.com/rack/rack/blob/master/SPEC.rdoc#the-body-) + # says: + # + # The Body must respond to `each` and must only yield String values + # + # Because it's easy to return the wrong body type, this middleware + # will: + # + # 1. Inspect each element of the body if it is an Array. + # 2. Coerce each value to a string if necessary. + # 3. Flag a test and development error. + class ResponseCoercerMiddleware < ::Grape::Middleware::Base + def call(env) + response = super(env) + + status = response[0] + body = response[2] + + return response if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY[status] + return response unless body.is_a?(Array) + + body.map! do |part| + if part.is_a?(String) + part + else + err = ArgumentError.new("The response body should be a String, but it is of type #{part.class}") + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err) + part.to_s + end + end + + response + end + end + class AdminModeMiddleware < ::Grape::Middleware::Base def after # Use a Grape middleware since the Grape `after` blocks might run diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index f98004af480..00b495bbc1e 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Appearance < Grape::API::Instance + class Appearance < ::API::Base before { authenticated_as_admin! } helpers do diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 4f2c3ee79ef..2afe8763d9d 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -2,7 +2,7 @@ module API # External applications API - class Applications < Grape::API::Instance + class Applications < ::API::Base before { authenticated_as_admin! } resource :applications do diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index 9501e777fff..5a9b9940fcf 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Avatar < Grape::API::Instance + class Avatar < ::API::Base resource :avatar do desc 'Return avatar url for a user' do success Entities::Avatar diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 0a3df3ed96e..6d40ae8f5ff 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class AwardEmoji < Grape::API::Instance + class AwardEmoji < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/badges.rb b/lib/api/badges.rb index f9728ffc446..fc00594c9ec 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Badges < Grape::API::Instance + class Badges < ::API::Base include PaginationParams before { authenticate_non_get! } diff --git a/lib/api/base.rb b/lib/api/base.rb new file mode 100644 index 00000000000..e174cef3bad --- /dev/null +++ b/lib/api/base.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module API + class Base < Grape::API::Instance # rubocop:disable API/Base + end +end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 1f5086127a8..d2d1628aff4 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Boards < Grape::API::Instance + class Boards < ::API::Base include BoardsResponses include PaginationParams diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 68497a08fb8..6a86c02bf4a 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -45,7 +45,7 @@ module API def destroy_list(list) destroy_conditionally!(list) do |list| service = ::Boards::Lists::DestroyService.new(board_parent, current_user) - unless service.execute(list) + if service.execute(list).error? render_api_error!({ error: 'List could not be deleted!' }, 400) end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 44f7610384e..37cce6eafba 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -3,7 +3,7 @@ require 'mime/types' module API - class Branches < Grape::API::Instance + class Branches < ::API::Base include PaginationParams BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index dcf950d7a03..8ce7694bbfd 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class BroadcastMessages < Grape::API::Instance + class BroadcastMessages < ::API::Base include PaginationParams resource :broadcast_messages do diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb index 1afdb0ad34c..18caf85f109 100644 --- a/lib/api/ci/pipeline_schedules.rb +++ b/lib/api/ci/pipeline_schedules.rb @@ -2,7 +2,7 @@ module API module Ci - class PipelineSchedules < Grape::API::Instance + class PipelineSchedules < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 045f81074a7..61e03ed1a95 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -2,7 +2,7 @@ module API module Ci - class Pipelines < Grape::API::Instance + class Pipelines < ::API::Base include PaginationParams before { authenticate_non_get! } @@ -128,7 +128,7 @@ module API pipeline = user_project.all_pipelines.find(params[:pipeline_id]) - if Feature.enabled?(:ci_jobs_finder_refactor) + if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) builds = ::Ci::JobsFinder .new(current_user: current_user, pipeline: pipeline, params: params) .execute @@ -157,7 +157,7 @@ module API pipeline = user_project.all_pipelines.find(params[:pipeline_id]) - if Feature.enabled?(:ci_jobs_finder_refactor) + if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) bridges = ::Ci::JobsFinder .new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge) .execute diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 08903dce3dc..ef679147c9f 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -2,7 +2,7 @@ module API module Ci - class Runner < Grape::API::Instance + class Runner < ::API::Base helpers ::API::Helpers::Runner resource :runners do @@ -72,6 +72,7 @@ module API post '/verify' do authenticate_runner! status 200 + body "200" end end @@ -181,7 +182,9 @@ module API .new(job, declared_params(include_missing: false)) service.execute.then do |result| + header 'X-GitLab-Trace-Update-Interval', result.backoff status result.status + body result.status.to_s end end @@ -292,6 +295,7 @@ module API if result[:status] == :success status :created + body "201" else render_api_error!(result[:message], result[:http_status]) end diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index 7bca72f8028..d37f10fe631 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -2,7 +2,7 @@ module API module Ci - class Runners < Grape::API::Instance + class Runners < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 9f5a6e87505..af103b8c1f8 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -3,7 +3,7 @@ require 'mime/types' module API - class CommitStatuses < Grape::API::Instance + class CommitStatuses < ::API::Base params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 20877fb5c5f..582ccd41847 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -3,7 +3,7 @@ require 'mime/types' module API - class Commits < Grape::API::Instance + class Commits < ::API::Base include PaginationParams before do @@ -62,19 +62,15 @@ module API first_parent: first_parent, order: order) - commit_count = - if all || path || before || after || first_parent - user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent) - else - # Cacheable commit count. - user_project.repository.commit_count_for_ref(ref) - end + serializer = with_stats ? Entities::CommitWithStats : Entities::Commit + # This tells kaminari that there is 1 more commit after the one we've + # loaded, meaning there will be a next page, if the currently loaded set + # of commits is equal to the requested page size. + commit_count = offset + commits.size + 1 paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - serializer = with_stats ? Entities::CommitWithStats : Entities::Commit - - present paginate(paginated_commits), with: serializer + present paginate(paginated_commits, exclude_total_headers: true), with: serializer end desc 'Commit multiple file changes as one commit' do diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 31d097c4bea..1becbd668a3 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -2,7 +2,7 @@ # PHP composer support (https://getcomposer.org/) module API - class ComposerPackages < Grape::API::Instance + class ComposerPackages < ::API::Base helpers ::API::Helpers::PackagesManagerClientsHelpers helpers ::API::Helpers::RelatedResourcesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers @@ -26,6 +26,10 @@ module API render_api_error!(e.message, 400) end + rescue_from Packages::Composer::ComposerJsonService::InvalidJson do |e| + render_api_error!(e.message, 422) + end + helpers do def packages strong_memoize(:packages) do @@ -123,7 +127,7 @@ module API bad_request! end - package_event('push_package') + track_package_event('push_package', :composer) ::Packages::Composer::CreatePackageService .new(authorized_user_project, current_user, declared_params) diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb index 209748d79fa..08265201328 100644 --- a/lib/api/conan_instance_packages.rb +++ b/lib/api/conan_instance_packages.rb @@ -2,7 +2,7 @@ # Conan Instance-Level Package Manager Client API module API - class ConanInstancePackages < Grape::API::Instance + class ConanInstancePackages < ::API::Base namespace 'packages/conan/v1' do include ConanPackageEndpoints end diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb index 445447cfcd2..9b6867a328b 100644 --- a/lib/api/conan_package_endpoints.rb +++ b/lib/api/conan_package_endpoints.rb @@ -246,7 +246,7 @@ module API delete do authorize!(:destroy_package, project) - package_event('delete_package', category: 'API::ConanPackages') + track_package_event('delete_package', :conan, category: 'API::ConanPackages') package.destroy end diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb index c51992231a7..db8cd187811 100644 --- a/lib/api/conan_project_packages.rb +++ b/lib/api/conan_project_packages.rb @@ -2,7 +2,7 @@ # Conan Project-Level Package Manager Client API module API - class ConanProjectPackages < Grape::API::Instance + class ConanProjectPackages < ::API::Base params do requires :id, type: Integer, desc: 'The ID of a project', regexp: %r{\A[1-9]\d*\z} end diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb index 0b7c35cadbd..6c4b80b612a 100644 --- a/lib/api/container_registry_event.rb +++ b/lib/api/container_registry_event.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ContainerRegistryEvent < Grape::API::Instance + class ContainerRegistryEvent < ::API::Base DOCKER_DISTRIBUTION_EVENTS_V1_JSON = 'application/vnd.docker.distribution.events.v1+json' before { authenticate_registry_notification! } diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb new file mode 100644 index 00000000000..e3cacc4132f --- /dev/null +++ b/lib/api/debian_group_packages.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + class DebianGroupPackages < ::API::Base + params do + requires :id, type: String, desc: 'The ID of a group' + end + + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + not_found! unless ::Feature.enabled?(:debian_packages, user_group) + + authorize_read_package!(user_group) + end + + namespace ':id/-/packages/debian' do + include DebianPackageEndpoints + end + end + end +end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb new file mode 100644 index 00000000000..168b3ca7a4f --- /dev/null +++ b/lib/api/debian_package_endpoints.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +module API + module DebianPackageEndpoints + extend ActiveSupport::Concern + + DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze + COMPONENT_REGEX = %r{[a-z-]+}.freeze + ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze + LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX + DISTRIBUTION_REQUIREMENTS = { + distribution: DISTRIBUTION_REGEX + }.freeze + COMPONENT_ARCHITECTURE_REQUIREMENTS = { + component: COMPONENT_REGEX, + architecture: ARCHITECTURE_REGEX + }.freeze + COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { + component: COMPONENT_REGEX, + letter: LETTER_REGEX, + source_package: PACKAGE_REGEX + }.freeze + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + included do + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + + format :txt + + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + before do + require_packages_enabled! + end + + params do + requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex + end + + namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg + desc 'The Release file signature' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'Release.gpg' do + not_found! + end + + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release + desc 'The unsigned Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'Release' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Release' + end + + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease + desc 'The signed Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'InRelease' do + not_found! + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + end + + namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + desc 'The binary files index' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'Packages' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Packages' + end + end + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' + requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex + end + + namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do + # GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name + params do + requires :file_name, type: String, desc: 'The Debian File Name' + end + desc 'The package' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get ':file_name', requirements: FILE_NAME_REQUIREMENTS do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO File' + end + end + end + end +end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb new file mode 100644 index 00000000000..bcb4e8c8cbc --- /dev/null +++ b/lib/api/debian_project_packages.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module API + class DebianProjectPackages < ::API::Base + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + before do + not_found! unless ::Feature.enabled?(:debian_packages, user_project) + + authorize_read_package! + end + + namespace ':id/-/packages/debian' do + include DebianPackageEndpoints + + params do + requires :file_name, type: String, desc: 'The file name' + end + + namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do + # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + put do + authorize_upload!(authorized_user_project) + bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size) + + track_package_event('push_package', :debian) + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) + + forbidden! + end + + # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + post 'authorize' do + authorize_workhorse!( + subject: authorized_user_project, + has_length: false, + maximum_size: authorized_user_project.actual_limits.debian_max_file_size + ) + end + end + end + end + end +end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index ad37b7578ad..314f5b6ee1d 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class DeployKeys < Grape::API::Instance + class DeployKeys < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 96aa2445f56..1c156b8b3bb 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class DeployTokens < Grape::API::Instance + class DeployTokens < ::API::Base include PaginationParams helpers do diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 87144fd31cc..ff06bdbae16 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -2,7 +2,7 @@ module API # Deployments RESTful API endpoints - class Deployments < Grape::API::Instance + class Deployments < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index c431ec8e1e4..3d2608c8c5a 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Discussions < Grape::API::Instance + class Discussions < ::API::Base include PaginationParams helpers ::API::Helpers::NotesHelpers helpers ::RendersNotes diff --git a/lib/api/entities/ci/lint/result.rb b/lib/api/entities/ci/lint/result.rb new file mode 100644 index 00000000000..0e4aa238ba2 --- /dev/null +++ b/lib/api/entities/ci/lint/result.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + module Ci + module Lint + class Result < Grape::Entity + expose :valid?, as: :valid + expose :errors + expose :warnings + expose :merged_yaml + end + end + end + end +end diff --git a/lib/api/entities/cluster.rb b/lib/api/entities/cluster.rb index 4cb54e988ce..67459092a33 100644 --- a/lib/api/entities/cluster.rb +++ b/lib/api/entities/cluster.rb @@ -4,7 +4,7 @@ module API module Entities class Cluster < Grape::Entity expose :id, :name, :created_at, :domain - expose :provider_type, :platform_type, :environment_scope, :cluster_type + expose :provider_type, :platform_type, :environment_scope, :cluster_type, :namespace_per_environment expose :user, using: Entities::UserBasic expose :platform_kubernetes, using: Entities::Platform::Kubernetes expose :provider_gcp, using: Entities::Provider::Gcp diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index cff627ab50a..c430b73580b 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -16,6 +16,7 @@ module API expose :project_id expose :location expose :created_at + expose :expiration_policy_started_at, as: :cleanup_policy_started_at expose :tags_count, if: -> (_, options) { options[:tags_count] } expose :tags, using: Tag, if: -> (_, options) { options[:tags] } end diff --git a/lib/api/entities/feature_flag.rb b/lib/api/entities/feature_flag.rb new file mode 100644 index 00000000000..82fdb20af00 --- /dev/null +++ b/lib/api/entities/feature_flag.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + expose :name + expose :description + expose :active + expose :version, if: :feature_flags_new_version_enabled + expose :created_at + expose :updated_at + expose :scopes, using: FeatureFlag::LegacyScope + expose :strategies, using: FeatureFlag::Strategy, if: :feature_flags_new_version_enabled + end + end +end diff --git a/lib/api/entities/feature_flag/detailed_legacy_scope.rb b/lib/api/entities/feature_flag/detailed_legacy_scope.rb new file mode 100644 index 00000000000..47078c1dfde --- /dev/null +++ b/lib/api/entities/feature_flag/detailed_legacy_scope.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class DetailedLegacyScope < LegacyScope + expose :name + end + end + end +end diff --git a/lib/api/entities/feature_flag/legacy_scope.rb b/lib/api/entities/feature_flag/legacy_scope.rb new file mode 100644 index 00000000000..7329f71c599 --- /dev/null +++ b/lib/api/entities/feature_flag/legacy_scope.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class LegacyScope < Grape::Entity + expose :id + expose :active + expose :environment_scope + expose :strategies + expose :created_at + expose :updated_at + end + end + end +end diff --git a/lib/api/entities/feature_flag/scope.rb b/lib/api/entities/feature_flag/scope.rb new file mode 100644 index 00000000000..906fe718257 --- /dev/null +++ b/lib/api/entities/feature_flag/scope.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class Scope < Grape::Entity + expose :id + expose :environment_scope + end + end + end +end diff --git a/lib/api/entities/feature_flag/strategy.rb b/lib/api/entities/feature_flag/strategy.rb new file mode 100644 index 00000000000..32699be0ee3 --- /dev/null +++ b/lib/api/entities/feature_flag/strategy.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class Strategy < Grape::Entity + expose :id + expose :name + expose :parameters + expose :scopes, using: FeatureFlag::Scope + end + end + end +end diff --git a/lib/api/entities/feature_flag/user_list.rb b/lib/api/entities/feature_flag/user_list.rb new file mode 100644 index 00000000000..bc8b12ea22e --- /dev/null +++ b/lib/api/entities/feature_flag/user_list.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + module Entities + class FeatureFlag < Grape::Entity + class UserList < Grape::Entity + include RequestAwareEntity + + expose :id + expose :iid + expose :project_id + expose :created_at + expose :updated_at + expose :name + expose :user_xids + + expose :path do |list| + project_feature_flags_user_list_path(list.project, list) + end + + expose :edit_path do |list| + edit_project_feature_flags_user_list_path(list.project, list) + end + end + end + end +end diff --git a/lib/api/entities/job_request/cache.rb b/lib/api/entities/job_request/cache.rb index a75affbaf84..cd533d7e5b3 100644 --- a/lib/api/entities/job_request/cache.rb +++ b/lib/api/entities/job_request/cache.rb @@ -4,7 +4,7 @@ module API module Entities module JobRequest class Cache < Grape::Entity - expose :key, :untracked, :paths, :policy + expose :key, :untracked, :paths, :policy, :when end end end diff --git a/lib/api/entities/member.rb b/lib/api/entities/member.rb index 14e97f41e77..ad62f92e5a0 100644 --- a/lib/api/entities/member.rb +++ b/lib/api/entities/member.rb @@ -5,6 +5,7 @@ module API class Member < Grape::Entity expose :user, merge: true, using: UserBasic expose :access_level + expose :created_at expose :expires_at end end diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index d903f50befa..b54f0e04a9d 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -7,7 +7,19 @@ module API extend ::API::Entities::EntityHelpers expose :id - expose :name + + expose :name do |package| + if package.conan? + package.conan_recipe + else + package.name + end + end + + expose :conan_package_name, if: ->(package) { package.conan? } do |package| + package.name + end + expose :version expose :package_type diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index fb599d68d72..82a44c75382 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -84,6 +84,7 @@ module API expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :ci_default_git_depth + expose :ci_forward_deployment_enabled expose :public_builds, as: :public_jobs expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options| project.build_allow_git_fetch ? 'fetch' : 'clone' diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb index 40488eb882d..85148c03d18 100644 --- a/lib/api/entities/snippet.rb +++ b/lib/api/entities/snippet.rb @@ -17,7 +17,7 @@ module API expose :file_name do |snippet| snippet.file_name_on_repo || snippet.file_name end - expose :files, if: ->(snippet, options) { snippet_multiple_files?(snippet, options[:current_user]) } do |snippet, options| + expose :files do |snippet, options| snippet.list_files.map do |file| { path: file, @@ -25,10 +25,6 @@ module API } end end - - def snippet_multiple_files?(snippet, current_user) - ::Feature.enabled?(:snippet_multiple_files, current_user) && snippet.repository_exists? - end end end end diff --git a/lib/api/entities/unleash_feature.rb b/lib/api/entities/unleash_feature.rb new file mode 100644 index 00000000000..8ee87d1fc11 --- /dev/null +++ b/lib/api/entities/unleash_feature.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashFeature < Grape::Entity + expose :name + expose :description, unless: ->(feature) { feature.description.nil? } + expose :active, as: :enabled + expose :strategies do |flag| + flag.strategies.map do |strategy| + if legacy_strategy?(strategy) + UnleashLegacyStrategy.represent(strategy) + elsif gitlab_user_list_strategy?(strategy) + UnleashGitlabUserListStrategy.represent(strategy) + else + UnleashStrategy.represent(strategy) + end + end + end + + private + + def legacy_strategy?(strategy) + !strategy.respond_to?(:name) + end + + def gitlab_user_list_strategy?(strategy) + strategy.name == ::Operations::FeatureFlags::Strategy::STRATEGY_GITLABUSERLIST + end + end + end +end diff --git a/lib/api/entities/unleash_gitlab_user_list_strategy.rb b/lib/api/entities/unleash_gitlab_user_list_strategy.rb new file mode 100644 index 00000000000..5617f8002d9 --- /dev/null +++ b/lib/api/entities/unleash_gitlab_user_list_strategy.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashGitlabUserListStrategy < Grape::Entity + expose :name do |_strategy| + ::Operations::FeatureFlags::Strategy::STRATEGY_USERWITHID + end + expose :parameters do |strategy| + { userIds: strategy.user_list.user_xids } + end + end + end +end diff --git a/lib/api/entities/unleash_legacy_strategy.rb b/lib/api/entities/unleash_legacy_strategy.rb new file mode 100644 index 00000000000..5d5954f8da0 --- /dev/null +++ b/lib/api/entities/unleash_legacy_strategy.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashLegacyStrategy < Grape::Entity + expose :name do |strategy| + strategy['name'] + end + expose :parameters do |strategy| + strategy['parameters'] + end + end + end +end diff --git a/lib/api/entities/unleash_strategy.rb b/lib/api/entities/unleash_strategy.rb new file mode 100644 index 00000000000..7627ce3873c --- /dev/null +++ b/lib/api/entities/unleash_strategy.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class UnleashStrategy < Grape::Entity + expose :name + expose :parameters + end + end +end diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb index c225ade6eb6..ab7bc738ff8 100644 --- a/lib/api/entities/user_with_admin.rb +++ b/lib/api/entities/user_with_admin.rb @@ -8,3 +8,5 @@ module API end end end + +API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin') diff --git a/lib/api/environments.rb b/lib/api/environments.rb index b825904e2c5..0e780d4ef36 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -2,7 +2,7 @@ module API # Environments RESTfull API endpoints - class Environments < Grape::API::Instance + class Environments < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb index 64ec6f0a57a..03f83477954 100644 --- a/lib/api/error_tracking.rb +++ b/lib/api/error_tracking.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ErrorTracking < Grape::API::Instance + class ErrorTracking < ::API::Base before { authenticate! } params do diff --git a/lib/api/events.rb b/lib/api/events.rb index 0b79431a76d..43efacf9c0b 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Events < Grape::API::Instance + class Events < ::API::Base include PaginationParams include APIGuard helpers ::API::Helpers::EventsHelpers diff --git a/lib/api/feature_flag_scopes.rb b/lib/api/feature_flag_scopes.rb new file mode 100644 index 00000000000..d77e243aa88 --- /dev/null +++ b/lib/api/feature_flag_scopes.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +module API + class FeatureFlagScopes < ::API::Base + include PaginationParams + + ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS = FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS + .merge(environment_scope: API::NO_SLASH_URL_PART_REGEX) + + before do + authorize_read_feature_flags! + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :feature_flag_scopes do + desc 'Get all effective feature flags under the environment' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag::DetailedLegacyScope + end + params do + requires :environment, type: String, desc: 'The environment name' + end + get do + present scopes_for_environment, with: ::API::Entities::FeatureFlag::DetailedLegacyScope + end + end + + params do + requires :name, type: String, desc: 'The name of the feature flag' + end + resource 'feature_flags/:name', requirements: FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS do + resource :scopes do + desc 'Get all scopes of a feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag::LegacyScope + end + params do + use :pagination + end + get do + present paginate(feature_flag.scopes), with: ::API::Entities::FeatureFlag::LegacyScope + end + + desc 'Create a scope of a feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag::LegacyScope + end + params do + requires :environment_scope, type: String, desc: 'The environment scope of the scope' + requires :active, type: Boolean, desc: 'Whether the scope is active' + requires :strategies, type: JSON, desc: 'The strategies of the scope' + end + post do + authorize_update_feature_flag! + + result = ::FeatureFlags::UpdateService + .new(user_project, current_user, scopes_attributes: [declared_params]) + .execute(feature_flag) + + if result[:status] == :success + present scope, with: ::API::Entities::FeatureFlag::LegacyScope + else + render_api_error!(result[:message], result[:http_status]) + end + end + + params do + requires :environment_scope, type: String, desc: 'URL-encoded environment scope' + end + resource ':environment_scope', requirements: ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS do + desc 'Get a scope of a feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag::LegacyScope + end + get do + present scope, with: ::API::Entities::FeatureFlag::LegacyScope + end + + desc 'Update a scope of a feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag::LegacyScope + end + params do + optional :active, type: Boolean, desc: 'Whether the scope is active' + optional :strategies, type: JSON, desc: 'The strategies of the scope' + end + put do + authorize_update_feature_flag! + + scope_attributes = declared_params.merge(id: scope.id) + + result = ::FeatureFlags::UpdateService + .new(user_project, current_user, scopes_attributes: [scope_attributes]) + .execute(feature_flag) + + if result[:status] == :success + updated_scope = result[:feature_flag].scopes + .find { |scope| scope.environment_scope == params[:environment_scope] } + + present updated_scope, with: ::API::Entities::FeatureFlag::LegacyScope + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Delete a scope from a feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag::LegacyScope + end + delete do + authorize_update_feature_flag! + + param = { scopes_attributes: [{ id: scope.id, _destroy: true }] } + + result = ::FeatureFlags::UpdateService + .new(user_project, current_user, param) + .execute(feature_flag) + + if result[:status] == :success + status :no_content + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + end + end + end + + helpers do + def authorize_read_feature_flags! + authorize! :read_feature_flag, user_project + end + + def authorize_update_feature_flag! + authorize! :update_feature_flag, feature_flag + end + + def feature_flag + @feature_flag ||= user_project.operations_feature_flags + .find_by_name!(params[:name]) + end + + def scope + @scope ||= feature_flag.scopes + .find_by_environment_scope!(CGI.unescape(params[:environment_scope])) + end + + def scopes_for_environment + Operations::FeatureFlagScope + .for_unleash_client(user_project, params[:environment]) + end + end + end +end diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb new file mode 100644 index 00000000000..613c3fb0f5b --- /dev/null +++ b/lib/api/feature_flags.rb @@ -0,0 +1,266 @@ +# frozen_string_literal: true + +module API + class FeatureFlags < ::API::Base + include PaginationParams + + FEATURE_FLAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS + .merge(name: API::NO_SLASH_URL_PART_REGEX) + + before do + authorize_read_feature_flags! + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :feature_flags do + desc 'Get all feature flags of a project' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag + end + params do + optional :scope, type: String, desc: 'The scope of feature flags', + values: %w[enabled disabled] + use :pagination + end + get do + feature_flags = ::FeatureFlagsFinder + .new(user_project, current_user, declared_params(include_missing: false)) + .execute + + present_entity(paginate(feature_flags)) + end + + desc 'Create a new feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag + end + params do + requires :name, type: String, desc: 'The name of feature flag' + optional :description, type: String, desc: 'The description of the feature flag' + optional :active, type: Boolean, desc: 'Active/inactive value of the flag' + optional :version, type: String, desc: 'The version of the feature flag' + optional :scopes, type: Array do + requires :environment_scope, type: String, desc: 'The environment scope of the scope' + requires :active, type: Boolean, desc: 'Active/inactive of the scope' + requires :strategies, type: JSON, desc: 'The strategies of the scope' + end + optional :strategies, type: Array do + requires :name, type: String, desc: 'The strategy name' + requires :parameters, type: JSON, desc: 'The strategy parameters' + optional :scopes, type: Array do + requires :environment_scope, type: String, desc: 'The environment scope of the scope' + end + end + end + post do + authorize_create_feature_flag! + + attrs = declared_params(include_missing: false) + + ensure_post_version_2_flags_enabled! if attrs[:version] == 'new_version_flag' + + rename_key(attrs, :scopes, :scopes_attributes) + rename_key(attrs, :strategies, :strategies_attributes) + update_value(attrs, :strategies_attributes) do |strategies| + strategies.map { |s| rename_key(s, :scopes, :scopes_attributes) } + end + + result = ::FeatureFlags::CreateService + .new(user_project, current_user, attrs) + .execute + + if result[:status] == :success + present_entity(result[:feature_flag]) + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + + params do + requires :feature_flag_name, type: String, desc: 'The name of the feature flag' + end + resource 'feature_flags/:feature_flag_name', requirements: FEATURE_FLAG_ENDPOINT_REQUIREMENTS do + desc 'Get a feature flag of a project' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag + end + get do + authorize_read_feature_flag! + + present_entity(feature_flag) + end + + desc 'Enable a strategy for a feature flag on an environment' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag + end + params do + requires :environment_scope, type: String, desc: 'The environment scope of the feature flag' + requires :strategy, type: JSON, desc: 'The strategy to be enabled on the scope' + end + post :enable do + not_found! unless Feature.enabled?(:feature_flag_api, user_project) + render_api_error!('Version 2 flags not supported', :unprocessable_entity) if new_version_flag_present? + + result = ::FeatureFlags::EnableService + .new(user_project, current_user, params).execute + + if result[:status] == :success + status :ok + present_entity(result[:feature_flag]) + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Disable a strategy for a feature flag on an environment' do + detail 'This feature is going to be introduced in GitLab 12.5 if `feature_flag_api` feature flag is removed' + success ::API::Entities::FeatureFlag + end + params do + requires :environment_scope, type: String, desc: 'The environment scope of the feature flag' + requires :strategy, type: JSON, desc: 'The strategy to be disabled on the scope' + end + post :disable do + not_found! unless Feature.enabled?(:feature_flag_api, user_project) + render_api_error!('Version 2 flags not supported', :unprocessable_entity) if feature_flag.new_version_flag? + + result = ::FeatureFlags::DisableService + .new(user_project, current_user, params).execute + + if result[:status] == :success + status :ok + present_entity(result[:feature_flag]) + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Update a feature flag' do + detail 'This feature will be introduced in GitLab 13.1 if feature_flags_new_version feature flag is removed' + success ::API::Entities::FeatureFlag + end + params do + optional :name, type: String, desc: 'The name of the feature flag' + optional :description, type: String, desc: 'The description of the feature flag' + optional :active, type: Boolean, desc: 'Active/inactive value of the flag' + optional :strategies, type: Array do + optional :id, type: Integer, desc: 'The strategy id' + optional :name, type: String, desc: 'The strategy type' + optional :parameters, type: JSON, desc: 'The strategy parameters' + optional :_destroy, type: Boolean, desc: 'Delete the strategy when true' + optional :scopes, type: Array do + optional :id, type: Integer, desc: 'The environment scope id' + optional :environment_scope, type: String, desc: 'The environment scope of the scope' + optional :_destroy, type: Boolean, desc: 'Delete the scope when true' + end + end + end + put do + not_found! unless feature_flags_new_version_enabled? + authorize_update_feature_flag! + render_api_error!('PUT operations are not supported for legacy feature flags', :unprocessable_entity) if feature_flag.legacy_flag? + + attrs = declared_params(include_missing: false) + + rename_key(attrs, :strategies, :strategies_attributes) + update_value(attrs, :strategies_attributes) do |strategies| + strategies.map { |s| rename_key(s, :scopes, :scopes_attributes) } + end + + result = ::FeatureFlags::UpdateService + .new(user_project, current_user, attrs) + .execute(feature_flag) + + if result[:status] == :success + present_entity(result[:feature_flag]) + else + render_api_error!(result[:message], result[:http_status]) + end + end + + desc 'Delete a feature flag' do + detail 'This feature was introduced in GitLab 12.5' + success ::API::Entities::FeatureFlag + end + delete do + authorize_destroy_feature_flag! + + result = ::FeatureFlags::DestroyService + .new(user_project, current_user, declared_params(include_missing: false)) + .execute(feature_flag) + + if result[:status] == :success + present_entity(result[:feature_flag]) + else + render_api_error!(result[:message], result[:http_status]) + end + end + end + end + + helpers do + def authorize_read_feature_flags! + authorize! :read_feature_flag, user_project + end + + def authorize_read_feature_flag! + authorize! :read_feature_flag, feature_flag + end + + def authorize_create_feature_flag! + authorize! :create_feature_flag, user_project + end + + def authorize_update_feature_flag! + authorize! :update_feature_flag, feature_flag + end + + def authorize_destroy_feature_flag! + authorize! :destroy_feature_flag, feature_flag + end + + def present_entity(result) + present result, + with: ::API::Entities::FeatureFlag, + feature_flags_new_version_enabled: feature_flags_new_version_enabled? + end + + def ensure_post_version_2_flags_enabled! + unless feature_flags_new_version_enabled? + render_api_error!('Version 2 flags are not enabled for this project', :unprocessable_entity) + end + end + + def feature_flag + @feature_flag ||= if feature_flags_new_version_enabled? + user_project.operations_feature_flags.find_by_name!(params[:feature_flag_name]) + else + user_project.operations_feature_flags.legacy_flag.find_by_name!(params[:feature_flag_name]) + end + end + + def new_version_flag_present? + user_project.operations_feature_flags.new_version_flag.find_by_name(params[:name]).present? + end + + def feature_flags_new_version_enabled? + Feature.enabled?(:feature_flags_new_version, user_project, default_enabled: true) + end + + def rename_key(hash, old_key, new_key) + hash[new_key] = hash.delete(old_key) if hash.key?(old_key) + hash + end + + def update_value(hash, key) + hash[key] = yield(hash[key]) if hash.key?(key) + hash + end + end + end +end diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb new file mode 100644 index 00000000000..e5218cfd7f1 --- /dev/null +++ b/lib/api/feature_flags_user_lists.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module API + class FeatureFlagsUserLists < ::API::Base + include PaginationParams + + error_formatter :json, -> (message, _backtrace, _options, _env, _original_exception) { + message.is_a?(String) ? { message: message }.to_json : message.to_json + } + + before do + authorize_admin_feature_flags_user_lists! + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :feature_flags_user_lists do + desc 'Get all feature flags user lists of a project' do + detail 'This feature was introduced in GitLab 12.10' + success ::API::Entities::FeatureFlag::UserList + end + params do + use :pagination + end + get do + present paginate(user_project.operations_feature_flags_user_lists), + with: ::API::Entities::FeatureFlag::UserList + end + + desc 'Create a feature flags user list for a project' do + detail 'This feature was introduced in GitLab 12.10' + success ::API::Entities::FeatureFlag::UserList + end + params do + requires :name, type: String, desc: 'The name of the list' + requires :user_xids, type: String, desc: 'A comma separated list of external user ids' + end + post do + list = user_project.operations_feature_flags_user_lists.create(declared_params) + + if list.save + present list, with: ::API::Entities::FeatureFlag::UserList + else + render_api_error!(list.errors.full_messages, :bad_request) + end + end + end + + params do + requires :iid, type: String, desc: 'The internal id of the user list' + end + resource 'feature_flags_user_lists/:iid' do + desc 'Get a single feature flag user list belonging to a project' do + detail 'This feature was introduced in GitLab 12.10' + success ::API::Entities::FeatureFlag::UserList + end + get do + present user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]), + with: ::API::Entities::FeatureFlag::UserList + end + + desc 'Update a feature flag user list' do + detail 'This feature was introduced in GitLab 12.10' + success ::API::Entities::FeatureFlag::UserList + end + params do + optional :name, type: String, desc: 'The name of the list' + optional :user_xids, type: String, desc: 'A comma separated list of external user ids' + end + put do + list = user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]) + + if list.update(declared_params(include_missing: false)) + present list, with: ::API::Entities::FeatureFlag::UserList + else + render_api_error!(list.errors.full_messages, :bad_request) + end + end + + desc 'Delete a feature flag user list' do + detail 'This feature was introduced in GitLab 12.10' + end + delete do + list = user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]) + unless list.destroy + render_api_error!(list.errors.full_messages, :conflict) + end + end + end + end + + helpers do + def authorize_admin_feature_flags_user_lists! + authorize! :admin_feature_flags_user_lists, user_project + end + end + end +end diff --git a/lib/api/features.rb b/lib/api/features.rb index 9d011d658f6..5d2e545abd6 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Features < Grape::API::Instance + class Features < ::API::Base before { authenticated_as_admin! } helpers do diff --git a/lib/api/files.rb b/lib/api/files.rb index 748bdfa894d..6833fc429e2 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Files < Grape::API::Instance + class Files < ::API::Base include APIGuard FILE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb index b8254ee9ab4..a83e36165a2 100644 --- a/lib/api/freeze_periods.rb +++ b/lib/api/freeze_periods.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class FreezePeriods < Grape::API::Instance + class FreezePeriods < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 98b8a40c7c9..a0c33ab65b9 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -1,7 +1,12 @@ # frozen_string_literal: true module API - class GenericPackages < Grape::API::Instance + class GenericPackages < ::API::Base + GENERIC_PACKAGES_REQUIREMENTS = { + package_name: API::NO_SLASH_URL_PART_REGEX, + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + before do require_packages_enabled! authenticate! @@ -17,17 +22,94 @@ module API route_setting :authentication, job_token_allowed: true namespace ':id/packages/generic' do - get 'ping' do - :pong + namespace ':package_name/*package_version/:file_name', requirements: GENERIC_PACKAGES_REQUIREMENTS do + desc 'Workhorse authorize generic package file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, job_token_allowed: true + + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true + requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + end + + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.generic_packages_max_file_size) + end + + desc 'Upload package file' do + detail 'This feature was introduced in GitLab 13.5' + end + + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true + requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, job_token_allowed: true + + put do + authorize_upload!(project) + bad_request!('File is too large') if max_file_size_exceeded? + + track_event('push_package') + + create_package_file_params = declared_params.merge(build: current_authenticated_job) + ::Packages::Generic::CreatePackageFileService + .new(project, current_user, create_package_file_params) + .execute + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id }) + + forbidden! + end + + desc 'Download package file' do + detail 'This feature was introduced in GitLab 13.5' + end + + params do + requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true + requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex + requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + end + + route_setting :authentication, job_token_allowed: true + + get do + authorize_read_package!(project) + + package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) + package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! + + track_event('pull_package') + + present_carrierwave_file!(package_file.file) + end end end end helpers do include ::API::Helpers::PackagesHelpers + include ::API::Helpers::Packages::BasicAuthHelpers def require_generic_packages_available! - not_found! unless Feature.enabled?(:generic_packages, user_project) + not_found! unless Feature.enabled?(:generic_packages, project, default_enabled: true) + end + + def project + authorized_user_project + end + + def max_file_size_exceeded? + project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size) end end end diff --git a/lib/api/github/entities.rb b/lib/api/github/entities.rb index c28a0b8eb7e..fe228c9a2d2 100644 --- a/lib/api/github/entities.rb +++ b/lib/api/github/entities.rb @@ -119,7 +119,9 @@ module API expose :username, as: :login expose :user_url, as: :url expose :user_url, as: :html_url - expose :avatar_url + expose :avatar_url do |user| + user.avatar_url(only_path: false) + end private diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index c0207f9169c..30f0cfb4dfd 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module API - class GoProxy < Grape::API::Instance + class GoProxy < ::API::Base helpers Gitlab::Golang helpers ::API::Helpers::PackagesHelpers diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 7efc12121d2..d4574b22d99 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupBoards < Grape::API::Instance + class GroupBoards < ::API::Base include BoardsResponses include PaginationParams diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index ae41d9f13b8..75429cf7a5c 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupClusters < Grape::API::Instance + class GroupClusters < ::API::Base include PaginationParams before { authenticate! } @@ -41,6 +41,7 @@ module API requires :name, type: String, desc: 'Cluster name' optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' @@ -74,6 +75,7 @@ module API optional :name, type: String, desc: 'Cluster name' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index 25b3059f63b..1bb26b3931c 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true module API - class GroupContainerRepositories < Grape::API::Instance + class GroupContainerRepositories < ::API::Base include PaginationParams + helpers ::API::Helpers::PackagesHelpers + before { authorize_read_group_container_images! } REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( @@ -27,7 +29,7 @@ module API user: current_user, subject: user_group ).execute - track_event('list_repositories') + track_package_event('list_repositories', :container) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index dc14813eefc..6ebaa8de185 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupExport < Grape::API::Instance + class GroupExport < ::API::Base helpers Helpers::RateLimiter before do diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb index b82d9fc519a..e703a217fd5 100644 --- a/lib/api/group_import.rb +++ b/lib/api/group_import.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupImport < Grape::API::Instance + class GroupImport < ::API::Base helpers Helpers::FileUploadHelpers helpers do diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 56f2b769464..8443ddf10ce 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupLabels < Grape::API::Instance + class GroupLabels < ::API::Base include PaginationParams helpers ::API::Helpers::LabelHelpers diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 82f5df79356..aef9877b84c 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupMilestones < Grape::API::Instance + class GroupMilestones < ::API::Base include MilestoneResponses include PaginationParams diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index aa047e260f5..5b6290df0dd 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupPackages < Grape::API::Instance + class GroupPackages < ::API::Base include PaginationParams before do diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index e7b8cd10197..ee110d67fa5 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class GroupVariables < Grape::API::Instance + class GroupVariables < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 813e41b4d39..bf3d6c3c7e0 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Groups < Grape::API::Instance + class Groups < ::API::Base include PaginationParams include Helpers::CustomAttributes @@ -29,7 +29,12 @@ module API # rubocop: disable CodeReuse/ActiveRecord def find_groups(params, parent_id = nil) - find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level) + find_params = params.slice( + :all_available, + :custom_attributes, + :owned, :min_access_level, + :include_parent_descendants + ) find_params[:parent] = if params[:top_level_only] [nil] @@ -309,6 +314,19 @@ module API present_groups params, groups end + desc 'Get a list of descendant groups of this group.' do + success Entities::Group + end + params do + use :group_list_params + use :with_custom_attributes + end + get ":id/descendant_groups" do + finder_params = declared_params(include_missing: false).merge(include_parent_descendants: true) + groups = find_groups(finder_params, params[:id]) + present_groups params, groups + end + desc 'Transfer a project to the group namespace. Available only for admin.' do success Entities::GroupDetail end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 1912a06682e..c8aee1f3479 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -522,7 +522,7 @@ module API else header(*Gitlab::Workhorse.send_url(file.url)) status :ok - body + body "" end end @@ -544,7 +544,6 @@ module API feature_name = "usage_data_#{event_name}" return unless Feature.enabled?(feature_name) - return unless Gitlab::CurrentSettings.usage_ping_enabled? Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name) rescue => error diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index f3dfc093926..ba07a70ee32 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -24,6 +24,7 @@ module API optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' + optional :shared_runners_setting, type: String, values: ::Namespace::SHARED_RUNNERS_SETTINGS, desc: 'Enable/disable shared runners for the group and its subgroups and projects' end params :optional_params_ee do diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb index e4163c63575..9b38eeb1e72 100644 --- a/lib/api/helpers/merge_requests_helpers.rb +++ b/lib/api/helpers/merge_requests_helpers.rb @@ -73,6 +73,13 @@ module API optional :not, type: Hash, desc: 'Parameters to negate' do use :merge_requests_negatable_params end + + optional :deployed_before, + 'Return merge requests deployed before the given date/time' + optional :deployed_after, + 'Return merge requests deployed after the given date/time' + optional :environment, + 'Returns merge requests deployed to the given environment' end params :optional_scope_param do diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index dcbf933a4e1..934e18bdd0a 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -158,7 +158,7 @@ module API conan_package_reference: params[:conan_package_reference] ).execute! - package_event('pull_package', category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY + track_package_event('pull_package', :conan, category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY present_carrierwave_file!(package_file.file) end @@ -169,7 +169,7 @@ module API def track_push_package_event if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate - package_event('push_package', category: 'API::ConanPackages') + track_package_event('push_package', :conan, category: 'API::ConanPackages') end end diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index 254af7690a2..577ba97d68a 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -10,6 +10,7 @@ module API def redirect_registry_request(forward_to_registry, package_type, options) if forward_to_registry && redirect_registry_request_available? + track_event("#{package_type}_request_forward") redirect(registry_url(package_type, options)) else yield diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index 403f5ea3851..e1898d28ef7 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -40,7 +40,7 @@ module API params = { has_length: has_length } params[:maximum_size] = maximum_size unless has_length - ::Packages::PackageFileUploader.workhorse_authorize(params) + ::Packages::PackageFileUploader.workhorse_authorize(**params) end def authorize_upload!(subject = user_project) @@ -48,7 +48,8 @@ module API require_gitlab_workhorse! end - def package_event(event_name, **args) + def track_package_event(event_name, scope, **args) + ::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute track_event(event_name, **args) end end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index a6ae9a87f98..227aec224e5 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -3,8 +3,8 @@ module API module Helpers module Pagination - def paginate(relation) - Gitlab::Pagination::OffsetPagination.new(self).paginate(relation) + def paginate(*args) + Gitlab::Pagination::OffsetPagination.new(self).paginate(*args) end end end diff --git a/lib/api/helpers/presentable.rb b/lib/api/helpers/presentable.rb index a5186cc56ea..40e1b266df5 100644 --- a/lib/api/helpers/presentable.rb +++ b/lib/api/helpers/presentable.rb @@ -23,7 +23,7 @@ module API def initialize(object, options = {}) options = options.opts_hash if options.is_a?(Grape::Entity::Options) - super(object.present(options), options) + super(object.present(**options), options) end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 8c20f5b8fc2..0364ba2ad9e 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -83,9 +83,18 @@ module API params :optional_filter_params_ee do end + params :optional_update_params_ce do + optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Skip older deployment jobs that are still pending' + end + params :optional_update_params_ee do end + params :optional_update_params do + use :optional_update_params_ce + use :optional_update_params_ee + end + params :optional_container_expiration_policy_params do optional :cadence, type: String, desc: 'Container expiration policy cadence for recurring job' optional :keep_n, type: String, desc: 'Container expiration policy number of images to keep' @@ -108,6 +117,7 @@ module API :builds_access_level, :ci_config_path, :ci_default_git_depth, + :ci_forward_deployment_enabled, :container_registry_enabled, :container_expiration_policy_attributes, :default_branch, diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 34a2fb09875..1c85669a626 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -51,9 +51,7 @@ module API job_forbidden!(job, 'Job is not running') unless job.running? end - if Gitlab::Ci::Features.job_heartbeats_runner?(job.project) - job.runner&.heartbeat(get_runner_ip) - end + job.runner&.heartbeat(get_runner_ip) job end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 4bceda51900..4adb27a7414 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -381,6 +381,12 @@ module API type: String, desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…' }, + { + required: false, + name: :branches_to_be_notified, + type: String, + desc: 'Branches for which notifications are to be sent' + }, chat_notification_events ].flatten, 'hipchat' => [ diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb index 65aec6ae2e7..451e578fdd6 100644 --- a/lib/api/helpers/settings_helpers.rb +++ b/lib/api/helpers/settings_helpers.rb @@ -12,6 +12,7 @@ module API def self.optional_attributes [*::ApplicationSettingsHelper.visible_attributes, *::ApplicationSettingsHelper.external_authorization_service_attributes, + *::ApplicationSettingsHelper.deprecated_attributes, :performance_bar_allowed_group_id].freeze end end diff --git a/lib/api/helpers/snippets_helpers.rb b/lib/api/helpers/snippets_helpers.rb index 9224381735f..42f56680ded 100644 --- a/lib/api/helpers/snippets_helpers.rb +++ b/lib/api/helpers/snippets_helpers.rb @@ -93,7 +93,7 @@ module API def validate_params_for_multiple_files(snippet) return unless params[:content] || params[:file_name] - if Feature.enabled?(:snippet_multiple_files, current_user) && snippet.multiple_files? + if snippet.multiple_files? render_api_error!({ error: _('To update Snippets with multiple files, you must use the `files` parameter') }, 400) end end diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb index df3235420e9..a0238c24f3b 100644 --- a/lib/api/import_bitbucket_server.rb +++ b/lib/api/import_bitbucket_server.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ImportBitbucketServer < Grape::API::Instance + class ImportBitbucketServer < ::API::Base helpers do def client @client ||= BitbucketServer::Client.new(credentials) diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 0bab891eada..61fce7a2c1b 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ImportGithub < Grape::API::Instance + class ImportGithub < ::API::Base rescue_from Octokit::Unauthorized, with: :provider_unauthorized before do diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index ff687a57888..6d8f13c36e6 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -3,7 +3,7 @@ module API # Internal access API module Internal - class Base < Grape::API::Instance + class Base < ::API::Base before { authenticate_by_gitlab_shell_token! } before do @@ -99,6 +99,14 @@ module API @project = @container = access_checker.container end end + + def validate_actor_key(actor, key_id) + return 'Could not find a user without a key' unless key_id + + return 'Could not find the given key' unless actor.key + + 'Could not find a user for the given key' unless actor.user + end end namespace 'internal' do @@ -163,28 +171,23 @@ module API redis: redis_ping } end + post '/two_factor_recovery_codes' do status 200 actor.update_last_used_at! user = actor.user - if params[:key_id] - unless actor.key - break { success: false, message: 'Could not find the given key' } - end - - if actor.key.is_a?(DeployKey) - break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } - end + error_message = validate_actor_key(actor, params[:key_id]) - unless user - break { success: false, message: 'Could not find a user for the given key' } - end - elsif params[:user_id] && user.nil? + if params[:user_id] && user.nil? break { success: false, message: 'Could not find the given user' } + elsif error_message + break { success: false, message: error_message } end + break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } if actor.key.is_a?(DeployKey) + unless user.two_factor_enabled? break { success: false, message: 'Two-factor authentication is not enabled for this user' } end @@ -204,20 +207,14 @@ module API actor.update_last_used_at! user = actor.user - if params[:key_id] - unless actor.key - break { success: false, message: 'Could not find the given key' } - end + error_message = validate_actor_key(actor, params[:key_id]) - if actor.key.is_a?(DeployKey) - break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' } - end + break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' } if actor.key.is_a?(DeployKey) - unless user - break { success: false, message: 'Could not find a user for the given key' } - end - elsif params[:user_id] && user.nil? + if params[:user_id] && user.nil? break { success: false, message: 'Could not find the given user' } + elsif error_message + break { success: false, message: error_message } end if params[:name].blank? @@ -269,6 +266,53 @@ module API present response, with: Entities::InternalPostReceive::Response end + + post '/two_factor_config' do + status 200 + + break { success: false } unless Feature.enabled?(:two_factor_for_cli) + + actor.update_last_used_at! + user = actor.user + + error_message = validate_actor_key(actor, params[:key_id]) + + if error_message + { success: false, message: error_message } + elsif actor.key.is_a?(DeployKey) + { success: true, two_factor_required: false } + else + { + success: true, + two_factor_required: user.two_factor_enabled? + } + end + end + + post '/two_factor_otp_check' do + status 200 + + break { success: false } unless Feature.enabled?(:two_factor_for_cli) + + actor.update_last_used_at! + user = actor.user + + error_message = validate_actor_key(actor, params[:key_id]) + + break { success: false, message: error_message } if error_message + + break { success: false, message: 'Deploy keys cannot be used for Two Factor' } if actor.key.is_a?(DeployKey) + + break { success: false, message: 'Two-factor authentication is not enabled for this user' } unless user.two_factor_enabled? + + otp_validation_result = ::Users::ValidateOtpService.new(user).execute(params.fetch(:otp_attempt)) + + if otp_validation_result[:status] == :success + { success: true } + else + { success: false, message: 'Invalid OTP' } + end + end end end end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 6d5dfd086e7..8175b81f900 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -3,7 +3,7 @@ module API # Kubernetes Internal API module Internal - class Kubernetes < Grape::API::Instance + class Kubernetes < ::API::Base before do check_feature_enabled authenticate_gitlab_kas_request! diff --git a/lib/api/internal/lfs.rb b/lib/api/internal/lfs.rb new file mode 100644 index 00000000000..630f0ec77a8 --- /dev/null +++ b/lib/api/internal/lfs.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module API + module Internal + class Lfs < ::API::Base + use Rack::Sendfile + + before { authenticate_by_gitlab_shell_token! } + + helpers do + def find_lfs_object(lfs_oid) + LfsObject.find_by_oid(lfs_oid) + end + end + + namespace 'internal' do + namespace 'lfs' do + desc 'Get LFS URL for object ID' do + detail 'This feature was introduced in GitLab 13.5.' + end + params do + requires :oid, type: String, desc: 'The object ID to query' + requires :gl_repository, type: String, desc: "Project identifier (e.g. project-1)" + end + get "/" do + lfs_object = find_lfs_object(params[:oid]) + + not_found! unless lfs_object + + _, project, repo_type = Gitlab::GlRepository.parse(params[:gl_repository]) + + not_found! unless repo_type.project? && project + not_found! unless lfs_object.project_allowed_access?(project) + + file = lfs_object.file + + not_found! unless file&.exists? + + content_type 'application/octet-stream' + + if file.file_storage? + sendfile file.path + else + workhorse_headers = Gitlab::Workhorse.send_url(file.url) + header workhorse_headers[0], workhorse_headers[1] + env['api.format'] = :binary + body "" + end + end + end + end + end + end +end diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 5f8d23f15fa..51136144c19 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -3,7 +3,7 @@ module API # Pages Internal API module Internal - class Pages < Grape::API::Instance + class Pages < ::API::Base before do authenticate_gitlab_pages_request! end diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index 6cc5b344f47..db4979c9052 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class IssueLinks < Grape::API::Instance + class IssueLinks < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0e5b0fae6e2..143f9e40736 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Issues < Grape::API::Instance + class Issues < ::API::Base include PaginationParams helpers Helpers::IssuesHelpers helpers Helpers::RateLimiter @@ -231,9 +231,6 @@ module API authorize! :create_issue, user_project - params.delete(:created_at) unless current_user.can?(:set_issue_created_at, user_project) - params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project) - issue_params = declared_params(include_missing: false) issue_params[:system_note_timestamp] = params[:created_at] @@ -279,8 +276,6 @@ module API issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue - # Setting updated_at is allowed only for admins and owners - params.delete(:updated_at) unless current_user.can?(:set_issue_updated_at, user_project) issue.system_note_timestamp = params[:updated_at] update_params = declared_params(include_missing: false).merge(request: request, api: true) diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index bc7bc956580..536b361b308 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class JobArtifacts < Grape::API::Instance + class JobArtifacts < ::API::Base before { authenticate_non_get! } # EE::API::JobArtifacts would override the following helpers diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index ad46d948f3b..bdb23b4a9be 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Jobs < Grape::API::Instance + class Jobs < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/keys.rb b/lib/api/keys.rb index c014641ca04..2e4568029b5 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -2,7 +2,7 @@ module API # Keys API - class Keys < Grape::API::Instance + class Keys < ::API::Base before { authenticate! } resource :keys do diff --git a/lib/api/labels.rb b/lib/api/labels.rb index edf4a8ca14e..0cc9f33bd07 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Labels < Grape::API::Instance + class Labels < ::API::Base include PaginationParams helpers ::API::Helpers::LabelHelpers diff --git a/lib/api/lint.rb b/lib/api/lint.rb index f7796b1e969..bfd152f70b1 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -1,24 +1,48 @@ # frozen_string_literal: true module API - class Lint < Grape::API::Instance + class Lint < ::API::Base namespace :ci do desc 'Validation of .gitlab-ci.yml content' params do requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' end post '/lint' do - error = Gitlab::Ci::YamlProcessor.validation_message(params[:content], - user: current_user) + result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute + error = result.errors.first status 200 - if error.blank? - { status: 'valid', errors: [] } - else - { status: 'invalid', errors: [error] } + response = if error.blank? + { status: 'valid', errors: [] } + else + { status: 'invalid', errors: [error] } + end + + response.tap do |response| + response[:merged_yaml] = result.merged_yaml if params[:include_merged_yaml] end end end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Validation of .gitlab-ci.yml content' do + detail 'This feature was introduced in GitLab 13.5.' + end + params do + optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' + end + get ':id/ci/lint' do + authorize! :download_code, user_project + + content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default) + result = Gitlab::Ci::Lint + .new(project: user_project, current_user: current_user) + .validate(content, dry_run: params[:dry_run]) + + present result, with: Entities::Ci::Lint::Result, current_user: current_user + end + end end end diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index a0822271cca..97549abd273 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Markdown < Grape::API::Instance + class Markdown < ::API::Base params do requires :text, type: String, desc: "The markdown text to render" optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown" diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index e6d9a9a7c20..a3e2fa84c32 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module API - class MavenPackages < Grape::API::Instance + class MavenPackages < ::API::Base MAVEN_ENDPOINT_REQUIREMENTS = { file_name: API::NO_SLASH_URL_PART_REGEX }.freeze @@ -32,10 +32,10 @@ module API end def verify_package_file(package_file, uploaded_file) - stored_sha1 = Digest::SHA256.hexdigest(package_file.file_sha1) - expected_sha1 = uploaded_file.sha256 + stored_sha256 = Digest::SHA256.hexdigest(package_file.file_sha1) + expected_sha256 = uploaded_file.sha256 - if stored_sha1 == expected_sha1 + if stored_sha256 == expected_sha256 no_content! else conflict! @@ -107,7 +107,7 @@ module API when 'sha1' package_file.file_sha1 else - package_event('pull_package') if jar_file?(format) + track_package_event('pull_package', :maven) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end end @@ -145,7 +145,7 @@ module API when 'sha1' package_file.file_sha1 else - package_event('pull_package') if jar_file?(format) + track_package_event('pull_package', :maven) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -181,7 +181,7 @@ module API when 'sha1' package_file.file_sha1 else - package_event('pull_package') if jar_file?(format) + track_package_event('pull_package', :maven) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -231,9 +231,9 @@ module API verify_package_file(package_file, params[:file]) when 'md5' - nil + '' else - package_event('push_package') if jar_file?(format) + track_package_event('push_package', :maven) if jar_file?(format) file_params = { file: params[:file], diff --git a/lib/api/members.rb b/lib/api/members.rb index 4edf94c6350..c28b3b1cc7c 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Members < Grape::API::Instance + class Members < ::API::Base include PaginationParams before { authenticate! } @@ -88,8 +88,8 @@ module API success Entities::Member end params do - requires :user_id, type: Integer, desc: 'The user ID of the new member' requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end # rubocop: disable CodeReuse/ActiveRecord @@ -97,20 +97,26 @@ module API source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - member = source.members.find_by(user_id: params[:user_id]) - conflict!('Member already exists') if member + if params[:user_id].to_s.include?(',') + create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id] }) - user = User.find_by_id(params[:user_id]) - not_found!('User') unless user + ::Members::CreateService.new(current_user, create_service_params).execute(source) + elsif params[:user_id].present? + member = source.members.find_by(user_id: params[:user_id]) + conflict!('Member already exists') if member - member = create_member(current_user, user, source, params) + user = User.find_by_id(params[:user_id]) + not_found!('User') unless user - if !member - not_allowed! # This currently can only be reached in EE - elsif member.valid? && member.persisted? - present_members(member) - else - render_validation_error!(member) + member = create_member(current_user, user, source, params) + + if !member + not_allowed! # This currently can only be reached in EE + elsif member.valid? && member.persisted? + present_members(member) + else + render_validation_error!(member) + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 035ed9f0e04..14d6e3995ea 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class MergeRequestApprovals < ::Grape::API::Instance + class MergeRequestApprovals < ::API::Base before { authenticate_non_get! } helpers do diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 3e43fe8b257..22023888bbd 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -2,7 +2,7 @@ module API # MergeRequestDiff API - class MergeRequestDiffs < Grape::API::Instance + class MergeRequestDiffs < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 4bd72b267a9..b24dd870c8b 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class MergeRequests < Grape::API::Instance + class MergeRequests < ::API::Base include PaginationParams CONTEXT_COMMITS_POST_LIMIT = 20 diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb index e07762ac6d3..b6bc0af2202 100644 --- a/lib/api/metrics/dashboard/annotations.rb +++ b/lib/api/metrics/dashboard/annotations.rb @@ -3,7 +3,7 @@ module API module Metrics module Dashboard - class Annotations < Grape::API::Instance + class Annotations < ::API::Base desc 'Create a new monitoring dashboard annotation' do success Entities::Metrics::Dashboard::Annotation end diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb index 263d2394276..cb6e7099247 100644 --- a/lib/api/metrics/user_starred_dashboards.rb +++ b/lib/api/metrics/user_starred_dashboards.rb @@ -2,7 +2,7 @@ module API module Metrics - class UserStarredDashboards < Grape::API::Instance + class UserStarredDashboards < ::API::Base resource :projects do desc 'Marks selected metrics dashboard as starred' do success Entities::Metrics::UserStarredDashboard diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index e1f279df045..f98a1f6dd1d 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Namespaces < Grape::API::Instance + class Namespaces < ::API::Base include PaginationParams before { authenticate! } @@ -32,7 +32,9 @@ module API get do namespaces = current_user.admin ? Namespace.all : current_user.namespaces - namespaces = namespaces.include_gitlab_subscription if Gitlab.ee? + namespaces = namespaces.include_route + + namespaces = namespaces.include_gitlab_subscription_with_hosted_plan if Gitlab.ee? namespaces = namespaces.search(params[:search]) if params[:search].present? diff --git a/lib/api/notes.rb b/lib/api/notes.rb index e4989243f3d..0db537ca616 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Notes < Grape::API::Instance + class Notes < ::API::Base include PaginationParams helpers ::API::Helpers::NotesHelpers diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index f8b621c1c38..bad3f5ead7a 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -2,7 +2,7 @@ module API # notification_settings API - class NotificationSettings < Grape::API::Instance + class NotificationSettings < ::API::Base before { authenticate! } helpers ::API::Helpers::MembersHelpers diff --git a/lib/api/npm_packages.rb b/lib/api/npm_packages.rb index fca405b76b7..1443b28c1ee 100644 --- a/lib/api/npm_packages.rb +++ b/lib/api/npm_packages.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module API - class NpmPackages < Grape::API::Instance + class NpmPackages < ::API::Base helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::DependencyProxyHelpers @@ -141,7 +141,7 @@ module API package_file = ::Packages::PackageFileFinder .new(package, params[:file_name]).execute! - package_event('pull_package') + track_package_event('pull_package', package) present_carrierwave_file!(package_file.file) end @@ -157,7 +157,7 @@ module API put ':id/packages/npm/:package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do authorize_create_package!(user_project) - package_event('push_package') + track_package_event('push_package', :npm) created_package = ::Packages::Npm::CreatePackageService .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute diff --git a/lib/api/nuget_packages.rb b/lib/api/nuget_packages.rb index f84a3acbe6d..0f2c956a9df 100644 --- a/lib/api/nuget_packages.rb +++ b/lib/api/nuget_packages.rb @@ -6,7 +6,7 @@ # called by the NuGet package manager client when users run commands # like `nuget install` or `nuget push`. module API - class NugetPackages < Grape::API::Instance + class NugetPackages < ::API::Base helpers ::API::Helpers::PackagesManagerClientsHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers @@ -42,7 +42,7 @@ module API def package_finder(finder_params = {}) ::Packages::Nuget::PackageFinder.new( authorized_user_project, - finder_params.merge(package_name: params[:package_name]) + **finder_params.merge(package_name: params[:package_name]) ) end end @@ -73,7 +73,7 @@ module API get 'index', format: :json do authorize_read_package!(authorized_user_project) - track_event('nuget_service_index') + track_package_event('cli_metadata', :nuget) present ::Packages::Nuget::ServiceIndexPresenter.new(authorized_user_project), with: ::API::Entities::Nuget::ServiceIndex @@ -105,7 +105,7 @@ module API package_file = ::Packages::CreatePackageFileService.new(package, file_params) .execute - package_event('push_package') + track_package_event('push_package', :nuget) ::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker @@ -198,7 +198,7 @@ module API not_found!('Package') unless package_file - package_event('pull_package') + track_package_event('pull_package', :nuget) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false present_carrierwave_file!(package_file.file, supports_direct_download: false) @@ -233,7 +233,7 @@ module API .new(authorized_user_project, params[:q], search_options) .execute - package_event('search_package') + track_package_event('search_package', :nuget) present ::Packages::Nuget::SearchResultsPresenter.new(search), with: ::API::Entities::Nuget::SearchResults diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index 17b92df629c..c1fc9a6e4d8 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class PackageFiles < Grape::API::Instance + class PackageFiles < ::API::Base include PaginationParams before do diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 79a6b527581..813307c498f 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Pages < Grape::API::Instance + class Pages < ::API::Base before do require_pages_config_enabled! authenticated_with_can_read_all_resources! diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 7d27b575efa..00c51298c45 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class PagesDomains < Grape::API::Instance + class PagesDomains < ::API::Base include PaginationParams PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 0e5605984e6..46ccb4ba1a0 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectClusters < Grape::API::Instance + class ProjectClusters < ::API::Base include PaginationParams before { authenticate! } @@ -45,6 +45,7 @@ module API optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true' requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do @@ -78,6 +79,7 @@ module API optional :name, type: String, desc: 'Cluster name' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' + optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 8f2a62bc5a4..d565531d372 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true module API - class ProjectContainerRepositories < Grape::API::Instance + class ProjectContainerRepositories < ::API::Base include PaginationParams + helpers ::API::Helpers::PackagesHelpers REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) - before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) } before { authorize_read_container_images! } params do @@ -28,7 +28,7 @@ module API user: current_user, subject: user_project ).execute - track_event( 'list_repositories') + track_package_event('list_repositories', :container) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end @@ -43,7 +43,7 @@ module API authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker - track_event('delete_repository') + track_package_event('delete_repository', :container) status :accepted end @@ -60,7 +60,7 @@ module API authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) - track_event('list_tags') + track_package_event('list_tags', :container) present paginate(tags), with: Entities::ContainerRegistry::Tag end @@ -89,7 +89,7 @@ module API declared_params.except(:repository_id).merge(container_expiration_policy: false)) # rubocop:enable CodeReuse/Worker - track_event('delete_tag_bulk') + track_package_event('delete_tag_bulk', :container) status :accepted end @@ -125,7 +125,7 @@ module API .execute(repository) if result[:status] == :success - track_event('delete_tag') + track_package_event('delete_tag', :container) status :ok else diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb index 726e693826e..3765473bc0e 100644 --- a/lib/api/project_events.rb +++ b/lib/api/project_events.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectEvents < Grape::API::Instance + class ProjectEvents < ::API::Base include PaginationParams include APIGuard helpers ::API::Helpers::EventsHelpers diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 377d61689b3..184f89200ab 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectExport < Grape::API::Instance + class ProjectExport < ::API::Base helpers Helpers::RateLimiter before do @@ -55,7 +55,7 @@ module API export_strategy = if after_export_params[:url].present? params = after_export_params.slice(:url, :http_method).symbolize_keys - Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params) + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(**params) end if export_strategy&.invalid? diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index e68a3b106b1..bc2d8c816a8 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectHooks < Grape::API::Instance + class ProjectHooks < ::API::Base include PaginationParams before { authenticate! } @@ -104,7 +104,9 @@ module API delete ":id/hooks/:hook_id" do hook = user_project.hooks.find(params.delete(:hook_id)) - destroy_conditionally!(hook) + destroy_conditionally!(hook) do + WebHooks::DestroyService.new(current_user).execute(hook) + end end end end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 9f43c3c7993..5c4e1d73ee1 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -1,11 +1,9 @@ # frozen_string_literal: true module API - class ProjectImport < Grape::API::Instance + class ProjectImport < ::API::Base include PaginationParams - MAXIMUM_FILE_SIZE = 50.megabytes - helpers Helpers::ProjectsHelpers helpers Helpers::FileUploadHelpers helpers Helpers::RateLimiter diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index 2f8dd1085dc..a81118f44bd 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectMilestones < Grape::API::Instance + class ProjectMilestones < ::API::Base include PaginationParams include MilestoneResponses diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 359514f1f78..b8d97b1243a 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectPackages < Grape::API::Instance + class ProjectPackages < ::API::Base include PaginationParams before do diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb index c318907542b..38eb74663d3 100644 --- a/lib/api/project_repository_storage_moves.rb +++ b/lib/api/project_repository_storage_moves.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectRepositoryStorageMoves < Grape::API::Instance + class ProjectRepositoryStorageMoves < ::API::Base include PaginationParams before { authenticated_as_admin! } @@ -69,7 +69,7 @@ module API success Entities::ProjectRepositoryStorageMove end params do - requires :destination_storage_name, type: String, desc: 'The destination storage shard' + optional :destination_storage_name, type: String, desc: 'The destination storage shard' end post ':id/repository_storage_moves' do storage_move = user_project.repository_storage_moves.build( diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb index 360000861fc..e19afb6e8e4 100644 --- a/lib/api/project_snapshots.rb +++ b/lib/api/project_snapshots.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectSnapshots < Grape::API::Instance + class ProjectSnapshots < ::API::Base helpers ::API::Helpers::ProjectSnapshotsHelpers before { authorize_read_git_snapshot! } diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index f6e87fece89..b4de260fe49 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true module API - class ProjectSnippets < Grape::API::Instance + class ProjectSnippets < ::API::Base include PaginationParams - before { authenticate! } before { check_snippets_enabled } params do @@ -37,6 +36,8 @@ module API use :pagination end get ":id/snippets" do + authenticate! + present paginate(snippets_for_current_user), with: Entities::ProjectSnippet, current_user: current_user end @@ -48,6 +49,9 @@ module API end get ":id/snippets/:snippet_id" do snippet = snippets_for_current_user.find(params[:snippet_id]) + + not_found!('Snippet') unless snippet + present snippet, with: Entities::ProjectSnippet, current_user: current_user end @@ -63,6 +67,8 @@ module API use :create_file_params end post ":id/snippets" do + authenticate! + authorize! :create_snippet, user_project snippet_params = process_create_params(declared_params(include_missing: false)) @@ -97,6 +103,8 @@ module API end # rubocop: disable CodeReuse/ActiveRecord put ":id/snippets/:snippet_id" do + authenticate! + snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) not_found!('Snippet') unless snippet @@ -125,6 +133,8 @@ module API end # rubocop: disable CodeReuse/ActiveRecord delete ":id/snippets/:snippet_id" do + authenticate! + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) not_found!('Snippet') unless snippet diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb index 2196801096f..1ead969fc81 100644 --- a/lib/api/project_statistics.rb +++ b/lib/api/project_statistics.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectStatistics < Grape::API::Instance + class ProjectStatistics < ::API::Base before do authenticate! authorize! :daily_statistics, user_project diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index 48c3dbed3b0..7d851de0237 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProjectTemplates < Grape::API::Instance + class ProjectTemplates < ::API::Base include PaginationParams TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze diff --git a/lib/api/projects.rb b/lib/api/projects.rb index abbdb11a3f7..ecee76ae60c 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -3,7 +3,7 @@ require_dependency 'declarative_policy' module API - class Projects < Grape::API::Instance + class Projects < ::API::Base include PaginationParams include Helpers::CustomAttributes @@ -353,7 +353,7 @@ module API optional :path, type: String, desc: 'The path of the repository' use :optional_project_params - use :optional_update_params_ee + use :optional_update_params at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of) end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index b0a7f898eec..a448682d8bd 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProtectedBranches < Grape::API::Instance + class ProtectedBranches < ::API::Base include PaginationParams BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index aaa31cb7cc6..dd3e407ffc9 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ProtectedTags < Grape::API::Instance + class ProtectedTags < ::API::Base include PaginationParams TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index c07db68f8a8..5622bc6e42d 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -6,7 +6,7 @@ # called by the PyPI package manager client when users run commands # like `pip install` or `twine upload`. module API - class PypiPackages < Grape::API::Instance + class PypiPackages < ::API::Base helpers ::API::Helpers::PackagesManagerClientsHelpers helpers ::API::Helpers::RelatedResourcesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers @@ -33,7 +33,7 @@ module API def find_package_versions packages = packages_finder - .with_name(params[:package_name]) + .with_normalized_pypi_name(params[:package_name]) not_found!('Package') if packages.empty? @@ -72,7 +72,7 @@ module API package = packages_finder(project).by_file_name_and_sha256(filename, params[:sha256]) package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute - package_event('pull_package') + track_package_event('pull_package', :pypi) present_carrierwave_file!(package_file.file, supports_direct_download: true) end @@ -91,7 +91,7 @@ module API get 'simple/*package_name', format: :txt do authorize_read_package!(authorized_user_project) - package_event('list_package') + track_package_event('list_package', :pypi) packages = find_package_versions presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) @@ -122,7 +122,7 @@ module API authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size) - package_event('push_package') + track_package_event('push_package', :pypi) ::Packages::Pypi::CreatePackageService .new(authorized_user_project, current_user, declared_params) diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index 9624b8924e5..23de9f9fc9f 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -2,7 +2,7 @@ module API module Release - class Links < Grape::API::Instance + class Links < ::API::Base include PaginationParams RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 3c38721129f..3bd6ea77403 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Releases < Grape::API::Instance + class Releases < ::API::Base include PaginationParams RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS @@ -19,9 +19,13 @@ module API end params do use :pagination + optional :order_by, type: String, values: %w[released_at created_at], default: 'released_at', + desc: 'Return releases ordered by `released_at` or `created_at`.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return releases sorted in `asc` or `desc` order.' end get ':id/releases' do - releases = ::ReleasesFinder.new(user_project, current_user).execute + releases = ::ReleasesFinder.new(user_project, current_user, declared_params.slice(:order_by, :sort)).execute present paginate(releases), with: Entities::Release, current_user: current_user end @@ -152,7 +156,7 @@ module API end def authorize_create_evidence! - # This is a separate method so that EE can extend its behaviour + # extended in EE end def release @@ -160,15 +164,15 @@ module API end def log_release_created_audit_event(release) - # This is a separate method so that EE can extend its behaviour + # extended in EE end def log_release_updated_audit_event - # This is a separate method so that EE can extend its behaviour + # extended in EE end def log_release_milestones_updated_audit_event - # This is a separate method so that EE can extend its behaviour + # extended in EE end end end diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index d1def05808b..f63ea04a529 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class RemoteMirrors < Grape::API::Instance + class RemoteMirrors < ::API::Base include PaginationParams before do diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 81702f8f02a..38ac1f22a48 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -3,7 +3,7 @@ require 'mime/types' module API - class Repositories < Grape::API::Instance + class Repositories < ::API::Base include PaginationParams helpers ::API::Helpers::HeadersHelpers diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb index a8d3419528c..d3a219f0810 100644 --- a/lib/api/resource_label_events.rb +++ b/lib/api/resource_label_events.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ResourceLabelEvents < Grape::API::Instance + class ResourceLabelEvents < ::API::Base include PaginationParams helpers ::API::Helpers::NotesHelpers diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb index a8f221f8740..21411f68dd5 100644 --- a/lib/api/resource_milestone_events.rb +++ b/lib/api/resource_milestone_events.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ResourceMilestoneEvents < Grape::API::Instance + class ResourceMilestoneEvents < ::API::Base include PaginationParams helpers ::API::Helpers::NotesHelpers diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb index 1c1a90c09a3..9bfda39be90 100644 --- a/lib/api/resource_state_events.rb +++ b/lib/api/resource_state_events.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class ResourceStateEvents < Grape::API::Instance + class ResourceStateEvents < ::API::Base include PaginationParams helpers ::API::Helpers::NotesHelpers diff --git a/lib/api/search.rb b/lib/api/search.rb index b9c6a823f4f..85f0a8e2e60 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Search < Grape::API::Instance + class Search < ::API::Base include PaginationParams before { authenticate! } @@ -33,6 +33,7 @@ module API scope: params[:scope], search: params[:search], state: params[:state], + confidential: params[:confidential], snippets: snippets?, page: params[:page], per_page: params[:per_page] @@ -62,12 +63,6 @@ module API # Defining this method here as a noop allows us to easily extend it in # EE, without having to modify this file directly. end - - def check_users_search_allowed! - if params[:scope].to_sym == :users && Feature.disabled?(:users_search, default_enabled: true) - render_api_error!({ error: _("Scope not supported with disabled 'users_search' feature!") }, 400) - end - end end resource :search do @@ -81,11 +76,11 @@ module API desc: 'The scope of the search', values: Helpers::SearchHelpers.global_search_scopes optional :state, type: String, desc: 'Filter results by state', values: Helpers::SearchHelpers.search_states + optional :confidential, type: Boolean, desc: 'Filter results by confidentiality' use :pagination end get do verify_search_scope!(resource: nil) - check_users_search_allowed! present search, with: entity end @@ -103,11 +98,11 @@ module API desc: 'The scope of the search', values: Helpers::SearchHelpers.group_search_scopes optional :state, type: String, desc: 'Filter results by state', values: Helpers::SearchHelpers.search_states + optional :confidential, type: Boolean, desc: 'Filter results by confidentiality' use :pagination end get ':id/(-/)search' do verify_search_scope!(resource: user_group) - check_users_search_allowed! present search(group_id: user_group.id), with: entity end @@ -126,11 +121,10 @@ module API values: Helpers::SearchHelpers.project_search_scopes optional :ref, type: String, desc: 'The name of a repository branch or tag. If not given, the default branch is used' optional :state, type: String, desc: 'Filter results by state', values: Helpers::SearchHelpers.search_states + optional :confidential, type: Boolean, desc: 'Filter results by confidentiality' use :pagination end get ':id/(-/)search' do - check_users_search_allowed! - present search({ project_id: user_project.id, repository_ref: params[:ref] }), with: entity end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 9ee1822339c..5f3d14010a8 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module API - class Services < Grape::API::Instance + class Services < ::API::Base services = Helpers::ServicesHelpers.services service_classes = Helpers::ServicesHelpers.service_classes diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 6e5534d0c9a..dc917d9c529 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Settings < Grape::API::Instance + class Settings < ::API::Base before { authenticated_as_admin! } helpers Helpers::SettingsHelpers @@ -29,7 +29,8 @@ module API success Entities::ApplicationSetting end params do - optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :admin_notification_email, type: String, desc: 'Deprecated: Use :abuse_notification_email instead. Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :abuse_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' @@ -73,6 +74,7 @@ module API optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help' optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page and help dropdown' + optional :help_page_documentation_base_url, type: String, desc: 'Alternate documentation pages URL' optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' @@ -194,6 +196,11 @@ module API attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:admin_notification_email) + attrs[:abuse_notification_email] = attrs.delete(:admin_notification_email) + end + # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0 attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled) diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index 77f2b1e871e..b025dbfab37 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -3,7 +3,7 @@ require 'sidekiq/api' module API - class SidekiqMetrics < Grape::API::Instance + class SidekiqMetrics < ::API::Base before { authenticated_as_admin! } helpers do diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index c6ef35875fc..2e67b9649bc 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -2,11 +2,9 @@ module API # Snippets API - class Snippets < Grape::API::Instance + class Snippets < ::API::Base include PaginationParams - before { authenticate! } - resource :snippets do helpers Helpers::SnippetsHelpers helpers do @@ -23,7 +21,7 @@ module API end end - desc 'Get a snippets list for authenticated user' do + desc 'Get a snippets list for an authenticated user' do detail 'This feature was introduced in GitLab 8.15.' success Entities::Snippet end @@ -31,6 +29,8 @@ module API use :pagination end get do + authenticate! + present paginate(snippets_for_current_user), with: Entities::Snippet, current_user: current_user end @@ -42,6 +42,8 @@ module API use :pagination end get 'public' do + authenticate! + present paginate(public_snippets), with: Entities::PersonalSnippet, current_user: current_user end @@ -74,6 +76,8 @@ module API use :create_file_params end post do + authenticate! + authorize! :create_snippet attrs = process_create_params(declared_params(include_missing: false)) @@ -109,6 +113,8 @@ module API use :minimum_update_params end put ':id' do + authenticate! + snippet = snippets_for_current_user.find_by_id(params.delete(:id)) break not_found!('Snippet') unless snippet @@ -139,6 +145,8 @@ module API requires :id, type: Integer, desc: 'The ID of a snippet' end delete ':id' do + authenticate! + snippet = snippets_for_current_user.find_by_id(params.delete(:id)) break not_found!('Snippet') unless snippet diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb index 3869fd3ac76..fa7176491ba 100644 --- a/lib/api/statistics.rb +++ b/lib/api/statistics.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Statistics < Grape::API::Instance + class Statistics < ::API::Base before { authenticated_as_admin! } COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue, diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb index 34d21d3d7d8..e2ceb49c119 100644 --- a/lib/api/submodules.rb +++ b/lib/api/submodules.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Submodules < Grape::API::Instance + class Submodules < ::API::Base before { authenticate! } helpers do diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 533663fb087..35a28da4736 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Subscriptions < Grape::API::Instance + class Subscriptions < ::API::Base helpers ::API::Helpers::LabelHelpers before { authenticate! } diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index 38e96c080f2..f23d279c3f4 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Suggestions < Grape::API::Instance + class Suggestions < ::API::Base before { authenticate! } resource :suggestions do diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index d8e0a425625..2820d305d0f 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class SystemHooks < Grape::API::Instance + class SystemHooks < ::API::Base include PaginationParams before do @@ -70,7 +70,9 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook - destroy_conditionally!(hook) + destroy_conditionally!(hook) do + WebHooks::DestroyService.new(current_user).execute(hook) + end end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index c1fbd3ca7c6..b969394ec47 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Tags < Grape::API::Instance + class Tags < ::API::Base include PaginationParams TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 80a97aae429..0b427bbf5b9 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Templates < Grape::API::Instance + class Templates < ::API::Base include PaginationParams GLOBAL_TEMPLATE_TYPES = { diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index 7063a3d08b5..4168cce21ef 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -4,7 +4,7 @@ require_dependency 'api/validations/validators/limit' module API module Terraform - class State < Grape::API::Instance + class State < ::API::Base include ::Gitlab::Utils::StrongMemoize default_format :json diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb new file mode 100644 index 00000000000..b4a0efd7a2b --- /dev/null +++ b/lib/api/terraform/state_version.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module API + module Terraform + class StateVersion < ::API::Base + default_format :json + + before do + authenticate! + authorize! :read_terraform_state, user_project + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/terraform/state/:name/versions/:serial' do + params do + requires :name, type: String, desc: 'The name of a Terraform state' + requires :serial, type: Integer, desc: 'The version number of the state' + end + + helpers do + def remote_state_handler + ::Terraform::RemoteStateHandler.new(user_project, current_user, name: params[:name]) + end + + def find_version(serial) + remote_state_handler.find_with_lock do |state| + version = state.versions.find_by_version(serial) + + if version.present? + yield version + else + not_found! + end + end + end + end + + desc 'Get a terraform state version' + route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get do + find_version(params[:serial]) do |version| + env['api.format'] = :binary # Bypass json serialization + body version.file.read + status :ok + end + end + + desc 'Delete a terraform state version' + route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + delete do + authorize! :admin_terraform_state, user_project + + find_version(params[:serial]) do |version| + version.destroy! + + body false + status :no_content + end + end + end + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 5eae92a251e..ce07d13cc9a 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Todos < Grape::API::Instance + class Todos < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index f398bbf3e32..960d004a04c 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Triggers < Grape::API::Instance + class Triggers < ::API::Base include PaginationParams HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb new file mode 100644 index 00000000000..907422118f1 --- /dev/null +++ b/lib/api/unleash.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module API + class Unleash < ::API::Base + include PaginationParams + + namespace :feature_flags do + resource :unleash, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :project_id, type: String, desc: 'The ID of a project' + optional :instance_id, type: String, desc: 'The Instance ID of Unleash Client' + optional :app_name, type: String, desc: 'The Application Name of Unleash Client' + end + route_param :project_id do + before do + authorize_by_unleash_instance_id! + end + + get do + # not supported yet + status :ok + end + + desc 'Get a list of features (deprecated, v2 client support)' + get 'features' do + present :version, 1 + present :features, feature_flags, with: ::API::Entities::UnleashFeature + end + + desc 'Get a list of features' + get 'client/features' do + present :version, 1 + present :features, feature_flags, with: ::API::Entities::UnleashFeature + end + + post 'client/register' do + # not supported yet + status :ok + end + + post 'client/metrics' do + # not supported yet + status :ok + end + end + end + end + + helpers do + def project + @project ||= find_project(params[:project_id]) + end + + def unleash_instance_id + env['HTTP_UNLEASH_INSTANCEID'] || params[:instance_id] + end + + def unleash_app_name + env['HTTP_UNLEASH_APPNAME'] || params[:app_name] + end + + def authorize_by_unleash_instance_id! + unauthorized! unless Operations::FeatureFlagsClient + .find_for_project_and_token(project, unleash_instance_id) + end + + def feature_flags + return [] unless unleash_app_name.present? + + legacy_flags = Operations::FeatureFlagScope.for_unleash_client(project, unleash_app_name) + new_version_flags = Operations::FeatureFlag.for_unleash_client(project, unleash_app_name) + + legacy_flags + new_version_flags + end + end + end +end diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index a1512197ee1..fa5bfc1cbe9 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true module API - class UsageData < Grape::API::Instance + class UsageData < ::API::Base before { authenticate! } namespace 'usage_data' do before do - not_found! unless Feature.enabled?(:usage_data_api) + not_found! unless Feature.enabled?(:usage_data_api, default_enabled: true) forbidden!('Invalid CSRF token is provided') unless verified_request? end diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 90127ecbc73..6d9db53fec8 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class UserCounts < Grape::API::Instance + class UserCounts < ::API::Base resource :user_counts do desc 'Return the user specific counts' do detail 'Open MR Count' diff --git a/lib/api/users.rb b/lib/api/users.rb index 73bb43b88fc..e7c1d644324 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Users < Grape::API::Instance + class Users < ::API::Base include PaginationParams include APIGuard include Helpers::CustomAttributes @@ -348,7 +348,7 @@ module API end # rubocop: enable CodeReuse/ActiveRecord - desc 'Get the GPG keys of a specified user. Available only for admins.' do + desc 'Get the GPG keys of a specified user.' do detail 'This feature was added in GitLab 10.0' success Entities::GpgKey end @@ -358,8 +358,6 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/gpg_keys' do - authenticated_as_admin! - user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -367,6 +365,26 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Get a specific GPG key for a given user.' do + detail 'This feature was added in GitLab 13.5' + success Entities::GpgKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the GPG key' + end + # rubocop: disable CodeReuse/ActiveRecord + get ':id/gpg_keys/:key_id' do + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.gpg_keys.find_by(id: params[:key_id]) + not_found!('GPG Key') unless key + + present key, with: Entities::GpgKey + end + # rubocop: enable CodeReuse/ActiveRecord + desc 'Delete an existing GPG key from a specified user. Available only for admins.' do detail 'This feature was added in GitLab 10.0' end @@ -529,10 +547,15 @@ module API unless user.can_be_deactivated? forbidden!('A blocked user cannot be deactivated by the API') if user.blocked? + forbidden!('An internal user cannot be deactivated by the API') if user.internal? forbidden!("The user you are trying to deactivate has been active in the past #{::User::MINIMUM_INACTIVE_DAYS} days and cannot be deactivated") end - user.deactivate + if user.deactivate + true + else + render_api_error!(user.errors.full_messages, 400) + end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 593f90460ac..aed88e6091c 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -7,7 +7,7 @@ # module API module V3 - class Github < Grape::API::Instance + class Github < ::API::Base NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze ENDPOINT_REQUIREMENTS = { namespace: NO_SLASH_URL_PART_REGEX, @@ -51,7 +51,7 @@ module API def find_project_with_access(params) project = find_project!( - ::Gitlab::Jira::Dvcs.restore_full_path(params.slice(:namespace, :project).symbolize_keys) + ::Gitlab::Jira::Dvcs.restore_full_path(**params.slice(:namespace, :project).symbolize_keys) ) not_found! unless can?(current_user, :download_code, project) project diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 0b3ec10f1b4..f5de3d844e6 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Variables < Grape::API::Instance + class Variables < ::API::Base include PaginationParams before { authenticate! } diff --git a/lib/api/version.rb b/lib/api/version.rb index 6a480fc2bd9..841b55f8d6c 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Version < Grape::API::Instance + class Version < ::API::Base helpers ::API::Helpers::GraphqlHelpers include APIGuard diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 4eba12157bd..21f457046f1 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module API - class Wikis < Grape::API::Instance + class Wikis < ::API::Base helpers ::API::Helpers::WikisHelpers helpers do diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index c2266f0bad6..6a45baa60ec 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Artifacts < Files + class Artifacts < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 5e795a449de..9c3b7165de7 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Builds < Files + class Builds < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb index 0dfe56e214f..514d52d7f65 100644 --- a/lib/backup/lfs.rb +++ b/lib/backup/lfs.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Lfs < Files + class Lfs < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb index d7aab33d7cb..ae293073ba2 100644 --- a/lib/backup/pages.rb +++ b/lib/backup/pages.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Pages < Files + class Pages < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb index d16ed2facf1..9645a07dfb8 100644 --- a/lib/backup/registry.rb +++ b/lib/backup/registry.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Registry < Files + class Registry < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb new file mode 100644 index 00000000000..4248a86dc7c --- /dev/null +++ b/lib/backup/repositories.rb @@ -0,0 +1,310 @@ +# frozen_string_literal: true + +require 'yaml' + +module Backup + class Repositories + attr_reader :progress + + def initialize(progress) + @progress = progress + end + + def dump(max_concurrency:, max_storage_concurrency:) + prepare + + if max_concurrency <= 1 && max_storage_concurrency <= 1 + return dump_consecutive + end + + check_valid_storages! + + semaphore = Concurrent::Semaphore.new(max_concurrency) + errors = Queue.new + + threads = Gitlab.config.repositories.storages.keys.map do |storage| + Thread.new do + Rails.application.executor.wrap do + dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) + rescue => e + errors << e + end + end + end + + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + threads.each(&:join) + end + + raise errors.pop unless errors.empty? + end + + def restore + Project.find_each(batch_size: 1000) do |project| + restore_repository(project, Gitlab::GlRepository::PROJECT) + restore_repository(project, Gitlab::GlRepository::WIKI) + restore_repository(project, Gitlab::GlRepository::DESIGN) + end + + invalid_ids = Snippet.find_each(batch_size: 1000) + .map { |snippet| restore_snippet_repository(snippet) } + .compact + + cleanup_snippets_without_repositories(invalid_ids) + + restore_object_pools + end + + private + + def check_valid_storages! + [ProjectRepository, SnippetRepository].each do |klass| + if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? + raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}" + end + end + end + + def backup_repos_path + @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories') + end + + def prepare + FileUtils.rm_rf(backup_repos_path) + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.mkdir(backup_repos_path, mode: 0700) + end + + def dump_consecutive + dump_consecutive_projects + dump_consecutive_snippets + end + + def dump_consecutive_projects + project_relation.find_each(batch_size: 1000) do |project| + dump_project(project) + end + end + + def dump_consecutive_snippets + Snippet.find_each(batch_size: 1000) { |snippet| dump_snippet(snippet) } + end + + def dump_storage(storage, semaphore, max_storage_concurrency:) + errors = Queue.new + queue = InterlockSizedQueue.new(1) + + threads = Array.new(max_storage_concurrency) do + Thread.new do + Rails.application.executor.wrap do + while container = queue.pop + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + semaphore.acquire + end + + begin + case container + when Project + dump_project(container) + when Snippet + dump_snippet(container) + end + rescue => e + errors << e + break + ensure + semaphore.release + end + end + end + end + end + + enqueue_records_for_storage(storage, queue, errors) + + raise errors.pop unless errors.empty? + ensure + queue.close + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + threads.each(&:join) + end + end + + def dump_project(project) + backup_repository(project, Gitlab::GlRepository::PROJECT) + backup_repository(project, Gitlab::GlRepository::WIKI) + backup_repository(project, Gitlab::GlRepository::DESIGN) + end + + def dump_snippet(snippet) + backup_repository(snippet, Gitlab::GlRepository::SNIPPET) + end + + def enqueue_records_for_storage(storage, queue, errors) + records_to_enqueue(storage).each do |relation| + relation.find_each(batch_size: 100) do |project| + break unless errors.empty? + + queue.push(project) + end + end + end + + def records_to_enqueue(storage) + [projects_in_storage(storage), snippets_in_storage(storage)] + end + + def projects_in_storage(storage) + project_relation.id_in(ProjectRepository.for_repository_storage(storage).select(:project_id)) + end + + def project_relation + Project.includes(:route, :group, namespace: :owner) + end + + def snippets_in_storage(storage) + Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id)) + end + + def backup_repository(container, type) + BackupRestore.new( + progress, + type.repository_for(container), + backup_repos_path + ).backup + end + + def restore_repository(container, type) + BackupRestore.new( + progress, + type.repository_for(container), + backup_repos_path + ).restore(always_create: type.project?) + end + + def restore_object_pools + PoolRepository.includes(:source_project).find_each do |pool| + progress.puts " - Object pool #{pool.disk_path}..." + + pool.source_project ||= pool.member_projects.first.root_of_fork_network + pool.state = 'none' + pool.save + + pool.schedule + end + end + + def restore_snippet_repository(snippet) + restore_repository(snippet, Gitlab::GlRepository::SNIPPET) + + response = Snippets::RepositoryValidationService.new(nil, snippet).execute + + if response.error? + snippet.repository.remove + + progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") + + snippet.id + else + nil + end + end + + # Snippets without a repository should be removed because they failed to import + # due to having invalid repositories + def cleanup_snippets_without_repositories(ids) + Snippet.id_in(ids).delete_all + end + + class BackupRestore + attr_accessor :progress, :repository, :backup_repos_path + + def initialize(progress, repository, backup_repos_path) + @progress = progress + @repository = repository + @backup_repos_path = backup_repos_path + end + + def backup + progress.puts " * #{display_repo_path} ... " + + if repository.empty? + progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan) + return + end + + FileUtils.mkdir_p(repository_backup_path) + + repository.bundle_to_disk(path_to_bundle) + repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue => e + progress.puts "[Failed] backing up #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + def restore(always_create: false) + progress.puts " * #{display_repo_path} ... " + + repository.remove rescue nil + + if File.exist?(path_to_bundle) + repository.create_from_bundle(path_to_bundle) + restore_custom_hooks + elsif always_create + repository.create_repository + end + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue => e + progress.puts "[Failed] restoring #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + private + + def display_repo_path + "#{repository.full_path} (#{repository.disk_path})" + end + + def repository_backup_path + @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) + end + + def path_to_bundle + @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') + end + + def restore_custom_hooks + return unless File.exist?(custom_hooks_tar) + + repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + end + + def custom_hooks_tar + File.join(repository_backup_path, "custom_hooks.tar") + end + end + + class InterlockSizedQueue < SizedQueue + extend ::Gitlab::Utils::Override + + override :pop + def pop(*) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + super + end + end + + override :push + def push(*) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + super + end + end + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb deleted file mode 100644 index eb0b230904e..00000000000 --- a/lib/backup/repository.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: true - -require 'yaml' - -module Backup - class Repository - attr_reader :progress - - def initialize(progress) - @progress = progress - end - - def dump(max_concurrency:, max_storage_concurrency:) - prepare - - if max_concurrency <= 1 && max_storage_concurrency <= 1 - return dump_consecutive - end - - if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? - raise Error, 'repositories.storages in gitlab.yml is misconfigured' - end - - semaphore = Concurrent::Semaphore.new(max_concurrency) - errors = Queue.new - - threads = Gitlab.config.repositories.storages.keys.map do |storage| - Thread.new do - Rails.application.executor.wrap do - dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) - rescue => e - errors << e - end - end - end - - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - - raise errors.pop unless errors.empty? - end - - def backup_project(project) - path_to_project_bundle = path_to_bundle(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .create_bundle(path_to_project_bundle) - - backup_custom_hooks(project) - rescue => e - progress_warn(project, e, 'Failed to backup repo') - end - - def backup_custom_hooks(project) - FileUtils.mkdir_p(project_backup_path(project)) - - custom_hooks_path = custom_hooks_tar(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .backup_custom_hooks(custom_hooks_path) - end - - def restore_custom_hooks(project) - return unless Dir.exist?(project_backup_path(project)) - return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? - - custom_hooks_path = custom_hooks_tar(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .restore_custom_hooks(custom_hooks_path) - end - - def restore - Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " - - restore_repo_success = - begin - try_restore_repository(project) - rescue => err - progress.puts "Error: #{err}".color(:red) - false - end - - if restore_repo_success - progress.puts "[DONE]".color(:green) - else - progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) - end - - wiki = ProjectWiki.new(project) - wiki.repository.remove rescue nil - path_to_wiki_bundle = path_to_bundle(wiki) - - if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.full_path} ... " - begin - wiki.repository.create_from_bundle(path_to_wiki_bundle) - restore_custom_hooks(wiki) - - progress.puts "[DONE]".color(:green) - rescue => e - progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) - progress.puts "Error #{e}".color(:red) - end - end - end - - restore_object_pools - end - - protected - - def try_restore_repository(project) - path_to_project_bundle = path_to_bundle(project) - project.repository.remove rescue nil - - if File.exist?(path_to_project_bundle) - project.repository.create_from_bundle(path_to_project_bundle) - restore_custom_hooks(project) - else - project.repository.create_repository - end - - true - end - - def path_to_bundle(project) - File.join(backup_repos_path, project.disk_path + '.bundle') - end - - def project_backup_path(project) - File.join(backup_repos_path, project.disk_path) - end - - def custom_hooks_tar(project) - File.join(project_backup_path(project), "custom_hooks.tar") - end - - def backup_repos_path - File.join(Gitlab.config.backup.path, 'repositories') - end - - def prepare - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) - end - - private - - def dump_consecutive - Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project| - dump_project(project) - end - end - - def dump_storage(storage, semaphore, max_storage_concurrency:) - errors = Queue.new - queue = InterlockSizedQueue.new(1) - - threads = Array.new(max_storage_concurrency) do - Thread.new do - Rails.application.executor.wrap do - while project = queue.pop - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - semaphore.acquire - end - - begin - dump_project(project) - rescue => e - errors << e - break - ensure - semaphore.release - end - end - end - end - end - - Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project| - break unless errors.empty? - - queue.push(project) - end - - raise errors.pop unless errors.empty? - ensure - queue.close - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - end - - def dump_project(project) - progress.puts " * #{display_repo_path(project)} ... " - - if project.hashed_storage?(:repository) - FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) - else - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace - end - - if !empty_repo?(project) - backup_project(project) - progress.puts " * #{display_repo_path(project)} ... " + "[DONE]".color(:green) - else - progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED]".color(:cyan) - end - - wiki = ProjectWiki.new(project) - - if !empty_repo?(wiki) - backup_project(wiki) - progress.puts " * #{display_repo_path(project)} ... " + "[DONE] Wiki".color(:green) - else - progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED] Wiki".color(:cyan) - end - end - - def progress_warn(project, cmd, output) - progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange) - end - - def empty_repo?(project_or_wiki) - project_or_wiki.repository.expire_emptiness_caches - project_or_wiki.repository.empty? - end - - def display_repo_path(project) - project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path - end - - def restore_object_pools - PoolRepository.includes(:source_project).find_each do |pool| - progress.puts " - Object pool #{pool.disk_path}..." - - pool.source_project ||= pool.member_projects.first.root_of_fork_network - pool.state = 'none' - pool.save - - pool.schedule - end - end - - class InterlockSizedQueue < SizedQueue - extend ::Gitlab::Utils::Override - - override :pop - def pop(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - - override :push - def push(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - end - end -end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index b6a62bc3f29..9665624f71b 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true -require 'backup/files' - module Backup - class Uploads < Files + class Uploads < Backup::Files attr_reader :progress def initialize(progress) diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb index 2ab47c5c6db..1754fec93d4 100644 --- a/lib/banzai/filter/design_reference_filter.rb +++ b/lib/banzai/filter/design_reference_filter.rb @@ -3,8 +3,6 @@ module Banzai module Filter class DesignReferenceFilter < AbstractReferenceFilter - FEATURE_FLAG = :design_management_reference_filter_gfm_pipeline - class Identifier include Comparable attr_reader :issue_iid, :filename @@ -35,14 +33,6 @@ module Banzai self.reference_type = :design - # This filter must be enabled by setting the - # design_management_reference_filter_gfm_pipeline flag - def call - return doc unless enabled? - - super - end - def find_object(project, identifier) records_per_parent[project][identifier] end @@ -112,10 +102,6 @@ module Banzai .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) } end - - def enabled? - Feature.enabled?(FEATURE_FLAG, parent, default_enabled: true) - end end end end diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb index c08d3364a87..8b5d689a984 100644 --- a/lib/banzai/reference_parser.rb +++ b/lib/banzai/reference_parser.rb @@ -2,6 +2,8 @@ module Banzai module ReferenceParser + InvalidReferenceType = Class.new(StandardError) + # Returns the reference parser class for the given type # # Example: @@ -11,6 +13,8 @@ module Banzai # This would return the `Banzai::ReferenceParser::IssueParser` class. def self.[](name) const_get("#{name.to_s.camelize}Parser", false) + rescue NameError + raise InvalidReferenceType end end end diff --git a/lib/banzai/reference_parser/mentioned_group_parser.rb b/lib/banzai/reference_parser/mentioned_group_parser.rb index a0892e15df8..75d05ef59f9 100644 --- a/lib/banzai/reference_parser/mentioned_group_parser.rb +++ b/lib/banzai/reference_parser/mentioned_group_parser.rb @@ -16,7 +16,7 @@ module Banzai end def nodes_visible_to_user(user, nodes) - groups = lazy { grouped_objects_for_nodes(nodes, Group, GROUP_ATTR) } + groups = lazy { grouped_objects_for_nodes(nodes, references_relation, GROUP_ATTR) } nodes.select do |node| node.has_attribute?(GROUP_ATTR) && can_read_group_reference?(node, user, groups) diff --git a/lib/banzai/reference_redactor.rb b/lib/banzai/reference_redactor.rb index 936436982e7..81e4fd45966 100644 --- a/lib/banzai/reference_redactor.rb +++ b/lib/banzai/reference_redactor.rb @@ -111,6 +111,7 @@ module Banzai parser = Banzai::ReferenceParser[type].new(context) visible.merge(parser.nodes_visible_to_user(user, nodes)) + rescue Banzai::ReferenceParser::InvalidReferenceType end visible diff --git a/lib/feature.rb b/lib/feature.rb index 71241e98723..1f8c530bee5 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -138,7 +138,7 @@ class Feature def register_definitions return unless check_feature_flags_definition? - Feature::Definition.load_all! + Feature::Definition.reload! end def register_hot_reloader diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index ee779a86952..0ba1bdc4799 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -84,17 +84,14 @@ class Feature end def definitions - @definitions ||= {} + # We lazily load all definitions + # The hot reloading might request a feature flag + # before we can properly call `load_all!` + @definitions ||= load_all! end - def load_all! - definitions.clear - - paths.each do |glob_path| - load_all_from_path!(glob_path) - end - - definitions + def reload! + @definitions = load_all! end def valid_usage!(key, type:, default_enabled:) @@ -110,9 +107,7 @@ class Feature def register_hot_reloader! # Reload feature flags on change of this file or any `.yml` file_watcher = Rails.configuration.file_watcher.new(reload_files, reload_directories) do - # We use `Feature::Definition` as on Ruby code-reload - # a new class definition is created - Feature::Definition.load_all! + Feature::Definition.reload! end Rails.application.reloaders << file_watcher @@ -123,6 +118,16 @@ class Feature private + def load_all! + # We currently do not load feature flag definitions + # in production environments + return [] unless Gitlab.dev_or_test_env? + + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + def load_from_file(path) definition = File.read(path) definition = YAML.safe_load(definition) @@ -133,7 +138,7 @@ class Feature raise Feature::InvalidFeatureFlagError, "Invalid definition for `#{path}`: #{e.message}" end - def load_all_from_path!(glob_path) + def load_all_from_path!(definitions, glob_path) Dir.glob(glob_path).each do |path| definition = load_from_file(path) @@ -146,7 +151,7 @@ class Feature end def reload_files - [File.expand_path(__FILE__)] + [] end def reload_directories diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index c06f699ef27..9ec56ee6b52 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -9,12 +9,14 @@ class Feature # optional: defines if a on-disk definition is required for this feature flag type # rollout_issue: defines if `bin/feature-flag` asks for rollout issue # default_enabled: defines a default state of a feature flag when created by `bin/feature-flag` + # ee_only: defines that a feature flag can only be created in a context of EE # example: usage being shown when exception is raised TYPES = { development: { description: 'Short lived, used to enable unfinished code to be deployed', - optional: true, + optional: false, rollout_issue: true, + ee_only: false, default_enabled: false, example: <<-EOS Feature.enabled?(:my_feature_flag, project) @@ -26,6 +28,7 @@ class Feature description: "Long-lived feature flags that control operational aspects of GitLab's behavior", optional: true, rollout_issue: false, + ee_only: false, default_enabled: false, example: <<-EOS Feature.enabled?(:my_ops_flag, type: ops) @@ -36,6 +39,7 @@ class Feature description: 'Permanent feature flags used to temporarily disable licensed features.', optional: true, rollout_issue: false, + ee_only: true, default_enabled: true, example: <<-EOS project.feature_available?(:my_licensed_feature) @@ -44,13 +48,15 @@ class Feature } }.freeze + # The ordering of PARAMS defines an order in YAML + # This is done to ease the file comparison PARAMS = %i[ name - default_enabled - type introduced_by_url rollout_issue_url + type group + default_enabled ].freeze end end diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb deleted file mode 100644 index 3bb839c1114..00000000000 --- a/lib/gitlab/alert_management/alert_params.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module AlertManagement - class AlertParams - MONITORING_TOOLS = { - prometheus: 'Prometheus' - }.freeze - - def self.from_generic_alert(project:, payload:) - parsed_payload = Gitlab::Alerting::NotificationPayloadParser.call(payload, project).with_indifferent_access - annotations = parsed_payload[:annotations] - - { - project_id: project.id, - title: annotations[:title], - description: annotations[:description], - monitoring_tool: annotations[:monitoring_tool], - service: annotations[:service], - hosts: Array(annotations[:hosts]), - payload: payload, - started_at: parsed_payload['startsAt'], - ended_at: parsed_payload['endsAt'], - severity: annotations[:severity], - fingerprint: annotations[:fingerprint], - environment: annotations[:environment] - } - end - - def self.from_prometheus_alert(project:, parsed_alert:) - { - project_id: project.id, - title: parsed_alert.title, - description: parsed_alert.description, - monitoring_tool: MONITORING_TOOLS[:prometheus], - payload: parsed_alert.payload, - started_at: parsed_alert.starts_at, - ended_at: parsed_alert.ends_at, - fingerprint: parsed_alert.gitlab_fingerprint, - environment: parsed_alert.environment, - prometheus_alert: parsed_alert.gitlab_alert - } - end - end - end -end diff --git a/lib/gitlab/alert_management/alert_status_counts.rb b/lib/gitlab/alert_management/alert_status_counts.rb index 382026236e0..e55e0016599 100644 --- a/lib/gitlab/alert_management/alert_status_counts.rb +++ b/lib/gitlab/alert_management/alert_status_counts.rb @@ -6,8 +6,6 @@ module Gitlab class AlertStatusCounts include Gitlab::Utils::StrongMemoize - STATUSES = ::AlertManagement::Alert::STATUSES - attr_reader :project def self.declarative_policy_class @@ -21,7 +19,7 @@ module Gitlab end # Define method for each status - STATUSES.each_key do |status| + ::AlertManagement::Alert.status_names.each do |status| define_method(status) { counts[status] } end @@ -30,7 +28,7 @@ module Gitlab end def all - counts.values.sum # rubocop:disable CodeReuse/ActiveRecord + counts.values.sum end private @@ -44,9 +42,7 @@ module Gitlab end def counts_by_status - ::AlertManagement::AlertsFinder - .counts_by_status(current_user, project, params) - .transform_keys { |status_id| STATUSES.key(status_id) } + ::AlertManagement::AlertsFinder.counts_by_status(current_user, project, params) end end end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 74e47e5226e..0fd593a3780 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -88,19 +88,19 @@ module Gitlab # AlertManagement::Alert directly for read operations. def alert_params { - description: description, + description: description&.truncate(::AlertManagement::Alert::DESCRIPTION_MAX_LENGTH), ended_at: ends_at, environment: environment, fingerprint: gitlab_fingerprint, - hosts: Array(hosts), - monitoring_tool: monitoring_tool, + hosts: truncate_hosts(Array(hosts).flatten), + monitoring_tool: monitoring_tool&.truncate(::AlertManagement::Alert::TOOL_MAX_LENGTH), payload: payload, project_id: project.id, prometheus_alert: gitlab_alert, - service: service, + service: service&.truncate(::AlertManagement::Alert::SERVICE_MAX_LENGTH), severity: severity, started_at: starts_at, - title: title + title: title&.truncate(::AlertManagement::Alert::TITLE_MAX_LENGTH) }.transform_values(&:presence).compact end @@ -135,6 +135,18 @@ module Gitlab def plain_gitlab_fingerprint; end + def truncate_hosts(hosts) + return hosts if hosts.join.length <= ::AlertManagement::Alert::HOSTS_MAX_LENGTH + + hosts.inject([]) do |new_hosts, host| + remaining_length = ::AlertManagement::Alert::HOSTS_MAX_LENGTH - new_hosts.join.length + + break new_hosts unless remaining_length > 0 + + new_hosts << host.to_s.truncate(remaining_length, omission: '') + end + end + def value_for_paths(paths) target_path = paths.find { |path| payload&.dig(*path) } diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 7efdfac75dc..e8e85155bef 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -8,6 +8,8 @@ module Gitlab DEFAULT_TITLE = 'New: Incident' DEFAULT_SEVERITY = 'critical' + attribute :description, paths: 'description' + attribute :ends_at, paths: 'end_time', type: :time attribute :environment_name, paths: 'gitlab_environment_name' attribute :hosts, paths: 'hosts' attribute :monitoring_tool, paths: 'monitoring_tool' @@ -23,3 +25,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload::Generic.prepend_if_ee('EE::Gitlab::AlertManagement::Payload::Generic') diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb deleted file mode 100644 index 94b81b7d290..00000000000 --- a/lib/gitlab/alerting/alert.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class Alert - include ActiveModel::Model - include Gitlab::Utils::StrongMemoize - include Presentable - - attr_accessor :project, :payload, :am_alert - - def self.for_alert_management_alert(project:, alert:) - params = if alert.prometheus? - alert.payload - else - Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project) - end - - self.new(project: project, payload: params, am_alert: alert) - end - - def gitlab_alert - strong_memoize(:gitlab_alert) do - parse_gitlab_alert_from_payload - end - end - - def metric_id - strong_memoize(:metric_id) do - payload&.dig('labels', 'gitlab_alert_id') - end - end - - def gitlab_prometheus_alert_id - strong_memoize(:gitlab_prometheus_alert_id) do - payload&.dig('labels', 'gitlab_prometheus_alert_id') - end - end - - def title - strong_memoize(:title) do - gitlab_alert&.title || parse_title_from_payload - end - end - - def description - strong_memoize(:description) do - parse_description_from_payload - end - end - - def environment - strong_memoize(:environment) do - gitlab_alert&.environment || parse_environment_from_payload - end - end - - def annotations - strong_memoize(:annotations) do - parse_annotations_from_payload || [] - end - end - - def starts_at - strong_memoize(:starts_at) do - parse_datetime_from_payload('startsAt') - end - end - - def starts_at_raw - strong_memoize(:starts_at_raw) do - payload&.dig('startsAt') - end - end - - def ends_at - strong_memoize(:ends_at) do - parse_datetime_from_payload('endsAt') - end - end - - def full_query - strong_memoize(:full_query) do - gitlab_alert&.full_query || parse_expr_from_payload - end - end - - def y_label - strong_memoize(:y_label) do - parse_y_label_from_payload || title - end - end - - def alert_markdown - strong_memoize(:alert_markdown) do - parse_alert_markdown_from_payload - end - end - - def status - strong_memoize(:status) do - payload&.dig('status') - end - end - - def firing? - status == 'firing' - end - - def resolved? - status == 'resolved' - end - - def gitlab_managed? - metric_id.present? - end - - def gitlab_fingerprint - Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint) - end - - def valid? - payload.respond_to?(:dig) && project && title && starts_at - end - - def present - super(presenter_class: Projects::Prometheus::AlertPresenter) - end - - private - - def plain_gitlab_fingerprint - if gitlab_managed? - [metric_id, starts_at_raw].join('/') - else # self managed - [starts_at_raw, title, full_query].join('/') - end - end - - def parse_environment_from_payload - environment_name = payload&.dig('labels', 'gitlab_environment_name') - - return unless environment_name - - EnvironmentsFinder.new(project, nil, { name: environment_name }) - .find - &.first - end - - def parse_gitlab_alert_from_payload - alerts_found = matching_gitlab_alerts - - return if alerts_found.blank? || alerts_found.size > 1 - - alerts_found.first - end - - def matching_gitlab_alerts - return unless metric_id || gitlab_prometheus_alert_id - - Projects::Prometheus::AlertsFinder - .new(project: project, metric: metric_id, id: gitlab_prometheus_alert_id) - .execute - end - - def parse_title_from_payload - payload&.dig('annotations', 'title') || - payload&.dig('annotations', 'summary') || - payload&.dig('labels', 'alertname') - end - - def parse_description_from_payload - payload&.dig('annotations', 'description') - end - - def parse_annotations_from_payload - payload&.dig('annotations')&.map do |label, value| - Alerting::AlertAnnotation.new(label: label, value: value) - end - end - - def parse_datetime_from_payload(field) - value = payload&.dig(field) - return unless value - - # value is a rfc3339 timestamp - # Timestamps from Prometheus and Alertmanager are UTC RFC3339 timestamps like: '2018-03-12T09:06:00Z' (Z represents 0 offset or UTC) - # .utc sets the datetime zone to `UTC` - Time.rfc3339(value).utc - rescue ArgumentError - end - - # Parses `g0.expr` from `generatorURL`. - # - # Example: http://localhost:9090/graph?g0.expr=vector%281%29&g0.tab=1 - def parse_expr_from_payload - url = payload&.dig('generatorURL') - return unless url - - uri = URI(url) - - Rack::Utils.parse_query(uri.query).fetch('g0.expr') - rescue URI::InvalidURIError, KeyError - end - - def parse_alert_markdown_from_payload - payload&.dig('annotations', 'gitlab_incident_markdown') - end - - def parse_y_label_from_payload - payload&.dig('annotations', 'gitlab_y_label') - end - end - end -end diff --git a/lib/gitlab/alerting/alert_annotation.rb b/lib/gitlab/alerting/alert_annotation.rb deleted file mode 100644 index a4b3a97b08c..00000000000 --- a/lib/gitlab/alerting/alert_annotation.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class AlertAnnotation - include ActiveModel::Model - - attr_accessor :label, :value - end - end -end diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb deleted file mode 100644 index 348f851f551..00000000000 --- a/lib/gitlab/alerting/notification_payload_parser.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class NotificationPayloadParser - BadPayloadError = Class.new(StandardError) - - DEFAULT_TITLE = 'New: Incident' - DEFAULT_SEVERITY = 'critical' - - def initialize(payload, project) - @payload = payload.to_h.with_indifferent_access - @project = project - end - - def self.call(payload, project) - new(payload, project).call - end - - def call - { - 'annotations' => annotations, - 'startsAt' => starts_at, - 'endsAt' => ends_at - }.compact - end - - private - - attr_reader :payload, :project - - def title - payload[:title].presence || DEFAULT_TITLE - end - - def severity - payload[:severity].presence || DEFAULT_SEVERITY - end - - def fingerprint - Gitlab::AlertManagement::Fingerprint.generate(payload[:fingerprint]) - end - - def annotations - primary_params - .reverse_merge(flatten_secondary_params) - .transform_values(&:presence) - .compact - end - - def primary_params - { - 'title' => title, - 'description' => payload[:description], - 'monitoring_tool' => payload[:monitoring_tool], - 'service' => payload[:service], - 'hosts' => hosts.presence, - 'severity' => severity, - 'fingerprint' => fingerprint, - 'environment' => environment - } - end - - def hosts - Array(payload[:hosts]).reject(&:blank?) - end - - def current_time - Time.current.change(usec: 0).rfc3339 - end - - def starts_at - Time.parse(payload[:start_time].to_s).rfc3339 - rescue ArgumentError - current_time - end - - def ends_at - Time.parse(payload[:end_time].to_s).rfc3339 - rescue ArgumentError - nil - end - - def environment - environment_name = payload[:gitlab_environment_name] - - return unless environment_name - - EnvironmentsFinder.new(project, nil, { name: environment_name }) - .find - &.first - end - - def secondary_params - payload.except(:start_time, :end_time) - end - - def flatten_secondary_params - Gitlab::Utils::SafeInlineHash.merge_keys!(secondary_params) - rescue ArgumentError - raise BadPayloadError, 'The payload is too big' - end - end - end -end - -Gitlab::Alerting::NotificationPayloadParser.prepend_if_ee('EE::Gitlab::Alerting::NotificationPayloadParser') diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 30cb74bcf54..b4bbb309c36 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -12,7 +12,8 @@ module Gitlab Attribute.new(:namespace, Namespace), Attribute.new(:user, User), Attribute.new(:caller_id, String), - Attribute.new(:related_class, String) + Attribute.new(:related_class, String), + Attribute.new(:feature_category, String) ].freeze def self.with_context(args, &block) @@ -45,6 +46,7 @@ module Gitlab hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:caller_id] = caller_id if set_values.include?(:caller_id) hash[:related_class] = related_class if set_values.include?(:related_class) + hash[:feature_category] = feature_category if set_values.include?(:feature_category) end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 609eef5e365..001c083c778 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -371,7 +371,7 @@ module Gitlab end def find_build_by_token(token) - ::Ci::Build.running.find_by_token(token) + ::Ci::AuthJobFinder.new(token: token).execute end def user_auth_attempt!(user, success:) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index ccf52bae9a5..3d3f7212053 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -290,7 +290,7 @@ module Gitlab end def api_request? - current_request.path.starts_with?('/api/') + current_request.path.starts_with?(Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/api/')) end def archive_request? diff --git a/lib/gitlab/auth/otp/strategies/base.rb b/lib/gitlab/auth/otp/strategies/base.rb new file mode 100644 index 00000000000..718630e0e31 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/base.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + class Base + def initialize(user) + @user = user + end + + private + + attr_reader :user + + def success + { status: :success } + end + + def error(message, http_status = nil) + result = { message: message, + status: :error } + + result[:http_status] = http_status if http_status + + result + end + end + end + end + end +end diff --git a/lib/gitlab/auth/otp/strategies/devise.rb b/lib/gitlab/auth/otp/strategies/devise.rb new file mode 100644 index 00000000000..93068d6c9b0 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/devise.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + class Devise < Base + def validate(otp_code) + user.validate_and_consume_otp!(otp_code) ? success : error('invalid OTP code') + end + end + end + end + end +end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator.rb new file mode 100644 index 00000000000..fbcb9fd8cdb --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + class FortiAuthenticator < Base + def validate(otp_code) + body = { username: user.username, + token_code: otp_code } + + response = Gitlab::HTTP.post( + auth_url, + headers: { 'Content-Type': 'application/json' }, + body: body.to_json, + basic_auth: api_credentials) + + # Successful authentication results in HTTP 200: OK + # https://docs.fortinet.com/document/fortiauthenticator/6.2.0/rest-api-solution-guide/704555/authentication-auth + response.ok? ? success : error(message: response.message, http_status: response.code) + end + + private + + def auth_url + host = ::Gitlab.config.forti_authenticator.host + port = ::Gitlab.config.forti_authenticator.port + path = 'api/v1/auth/' + + "https://#{host}:#{port}/#{path}" + end + + def api_credentials + { username: ::Gitlab.config.forti_authenticator.username, + password: ::Gitlab.config.forti_authenticator.token } + end + end + end + end + end +end diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index cc4b8d887ff..36b54ba2e46 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -11,6 +11,8 @@ module Gitlab case rejection_type when :internal "This action cannot be performed by internal users" + when :blocked_pending_approval + "Your account is pending approval from your administrator and hence blocked." when :terms_not_accepted "You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\ "Please access GitLab from a web browser to accept these terms." @@ -31,6 +33,8 @@ module Gitlab def rejection_type if @user.internal? :internal + elsif @user.blocked_pending_approval? + :blocked_pending_approval elsif @user.required_terms_not_accepted? :terms_not_accepted elsif @user.deactivated? diff --git a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb new file mode 100644 index 00000000000..2148e96f6b4 --- /dev/null +++ b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Compare all current rules to project rules + class AddModifiedToApprovalMergeRequestRule + # Stubbed class to access the Group table + class Group < ActiveRecord::Base + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + end + + # Stubbed class to access the ApprovalMergeRequestRule table + class ApprovalMergeRequestRule < ActiveRecord::Base + self.table_name = 'approval_merge_request_rules' + + has_one :approval_merge_request_rule_source, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource' + has_one :approval_project_rule, through: :approval_merge_request_rule_source + has_and_belongs_to_many :groups, + class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups" + end + + # Stubbed class to access the ApprovalProjectRule table + class ApprovalProjectRule < ActiveRecord::Base + self.table_name = 'approval_project_rules' + + has_many :approval_merge_request_rule_sources, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource' + has_and_belongs_to_many :groups, + class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups" + end + + # Stubbed class to access the ApprovalMergeRequestRuleSource table + class ApprovalMergeRequestRuleSource < ActiveRecord::Base + self.table_name = 'approval_merge_request_rule_sources' + + belongs_to :approval_merge_request_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRule' + belongs_to :approval_project_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalProjectRule' + end + + def perform(start_id, stop_id) + approval_merge_requests_rules = ApprovalMergeRequestRule + .joins(:groups, :approval_merge_request_rule_source) + .where(id: start_id..stop_id) + .pluck( + 'approval_merge_request_rule_sources.id as ars_id', + 'approval_merge_request_rules_groups.id as amrg_id' + ) + + approval_project_rules = ApprovalProjectRule + .joins(:groups, approval_merge_request_rule_sources: :approval_merge_request_rule) + .where(approval_merge_request_rules: { id: start_id..stop_id }) + .pluck( + 'approval_merge_request_rule_sources.id as ars_id', + 'approval_project_rules_groups.id as apg_id' + ) + + different_names_or_approval_sources = ApprovalMergeRequestRule.joins(:approval_project_rule, :approval_merge_request_rule_source) + .where(id: start_id..stop_id) + .where('approval_merge_request_rules.name != approval_project_rules.name OR ' \ + 'approval_merge_request_rules.approvals_required != approval_project_rules.approvals_required') + .pluck('approval_merge_request_rule_sources.id as ars_id') + + intersected_set = approval_merge_requests_rules.to_set ^ approval_project_rules.to_set + source_ids = intersected_set.collect { |rule| rule[0] }.uniq + + rule_sources = ApprovalMergeRequestRuleSource.where(id: source_ids + different_names_or_approval_sources) + changed_merge_request_rules = ApprovalMergeRequestRule.where(id: rule_sources.select(:approval_merge_request_rule_id)) + + changed_merge_request_rules.update_all(modified_from_project_rule: true) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 21538000fec..8befade8c3a 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -109,7 +109,7 @@ module Gitlab end def create_commit(snippet) - snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs) + snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), **commit_attrs) end # If the user is not allowed to access git or update the snippet diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb new file mode 100644 index 00000000000..b8c14aa2573 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation +require "webauthn/u2f_migrator" + +module Gitlab + module BackgroundMigration + class MigrateU2fWebauthn + class U2fRegistration < ActiveRecord::Base + self.table_name = 'u2f_registrations' + end + + class WebauthnRegistration < ActiveRecord::Base + self.table_name = 'webauthn_registrations' + end + + def perform(start_id, end_id) + old_registrations = U2fRegistration.where(id: start_id..end_id) + old_registrations.each_slice(100) do |slice| + now = Time.now + values = slice.map do |u2f_registration| + converted_credential = WebAuthn::U2fMigrator.new( + app_id: Gitlab.config.gitlab.url, + certificate: u2f_registration.certificate, + key_handle: u2f_registration.key_handle, + public_key: u2f_registration.public_key, + counter: u2f_registration.counter + ).credential + + { + credential_xid: Base64.strict_encode64(converted_credential.id), + public_key: Base64.strict_encode64(converted_credential.public_key), + counter: u2f_registration.counter || 0, + name: u2f_registration.name || '', + user_id: u2f_registration.user_id, + u2f_registration_id: u2f_registration.id, + created_at: now, + updated_at: now + } + end + + WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb index ca64d13b118..bbe2164ae4e 100644 --- a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb +++ b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb @@ -13,8 +13,6 @@ module Gitlab end def perform(start_id, stop_id) - return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true) - relation = User .select("id AS user_id", "substring(COALESCE(bio, '') from 1 for 255) AS bio") .where("(COALESCE(bio, '') IS DISTINCT FROM '')") diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb new file mode 100644 index 00000000000..cd305adc7cd --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveDuplicatedCsFindingsWithoutVulnerabilityId + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb new file mode 100644 index 00000000000..26626aaef79 --- /dev/null +++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class ReplaceBlockedByLinks + class IssueLink < ActiveRecord::Base + self.table_name = 'issue_links' + end + + def perform(start_id, stop_id) + blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2) + + ActiveRecord::Base.transaction do + # if there is duplicit bi-directional relation (issue2 is blocked by issue1 + # and issue1 already links issue2), then we can just delete 'blocked by'. + # This should be rare as we have a pre-create check which checks if issues are + # already linked + blocked_by_links + .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id') + .where('opposite_links.link_type': 1) + .delete_all + + blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb new file mode 100644 index 00000000000..6262320128c --- /dev/null +++ b/lib/gitlab/background_migration/sync_blocking_issues_count.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class SyncBlockingIssuesCount + def perform(start_id, end_id) + end + end + end +end + +Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount') diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb new file mode 100644 index 00000000000..3def5eb3369 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + # isolated Banzai::ReferenceParser + module ReferenceParser + # Returns the reference parser class for the given type + # + # Example: + # + # Banzai::ReferenceParser['isolated_mentioned_group'] + # + # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class. + def self.[](name) + const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false) + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb new file mode 100644 index 00000000000..d3d032ba433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Group + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb new file mode 100644 index 00000000000..1d3a3af81a1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Extract possible GFM references from an arbitrary String for further processing. + class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor + REFERABLES = %i(isolated_mentioned_group).freeze + + REFERABLES.each do |type| + define_method("#{type}s") do + @references[type] ||= isolated_references(type) + end + end + + def isolated_references(type) + context = ::Banzai::RenderContext.new(project, current_user) + processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context) + + refs = processor.process(html_documents) + refs[:visible] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index 69ba3f9132b..be9c0ad2b3a 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -36,7 +36,8 @@ module Gitlab if extractor extractors[current_user] = extractor else - extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user) + extractor = extractors[current_user] ||= + Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user) extractor.reset_memoized_values end @@ -71,7 +72,7 @@ module Gitlab mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) - mentioned_groups_ids = array_to_sql(refs.mentioned_groups.pluck(:id)) + mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb new file mode 100644 index 00000000000..5cadfa45b5b --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + module Namespace + # extracted methods for recursive traversing of namespace hierarchy + module RecursiveTraversal + extend ActiveSupport::Concern + + def root_ancestor + return self if persisted? && parent_id.nil? + + strong_memoize(:root_ancestor) do + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors + .reorder(nil) + .find_by(parent_id: nil) + end + end + + # Returns all ancestors, self, and descendants of the current namespace. + def self_and_hierarchy + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .all_objects + end + + # Returns all the ancestors of the current namespaces. + def ancestors + return self.class.none unless parent_id + + Gitlab::ObjectHierarchy + .new(self.class.where(id: parent_id)) + .base_and_ancestors + end + + # returns all ancestors upto but excluding the given namespace + # when no namespace is given, all ancestors upto the top are returned + def ancestors_upto(top = nil, hierarchy_order: nil) + Gitlab::ObjectHierarchy.new(self.class.where(id: id)) + .ancestors(upto: top, hierarchy_order: hierarchy_order) + end + + def self_and_ancestors(hierarchy_order: nil) + return self.class.where(id: id) unless parent_id + + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors(hierarchy_order: hierarchy_order) + end + + # Returns all the descendants of the current namespace. + def descendants + Gitlab::ObjectHierarchy + .new(self.class.where(parent_id: id)) + .base_and_descendants + end + + def self_and_descendants + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_descendants + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb new file mode 100644 index 00000000000..bc04172b9a2 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Group model + class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace + self.store_full_sti_class = false + has_one :saml_provider + + def self.declarative_policy_class + "GroupPolicy" + end + + def max_member_access_for_user(user) + return GroupMember::NO_ACCESS unless user + + return GroupMember::OWNER if user.admin? + + max_member_access = members_with_parents.where(user_id: user) + .reorder(access_level: :desc) + .first + &.access_level + + max_member_access || GroupMember::NO_ACCESS + end + + def members_with_parents + # Avoids an unnecessary SELECT when the group has no parents + source_ids = + if has_parent? + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + group_hierarchy_members = GroupMember.active_without_invites_and_requests + .where(source_id: source_ids) + + GroupMember.from_union([group_hierarchy_members, + members_from_self_and_ancestor_group_shares]) + end + + # rubocop: disable Metrics/AbcSize + def members_from_self_and_ancestor_group_shares + group_group_link_table = GroupGroupLink.arel_table + group_member_table = GroupMember.arel_table + + source_ids = + if has_parent? + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids) + cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query) + cte_alias = cte.table.alias(GroupGroupLink.table_name) + + # Instead of members.access_level, we need to maximize that access_level at + # the respective group_group_links.group_access. + member_columns = GroupMember.attribute_names.map do |column_name| + if column_name == 'access_level' + smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]], + 'access_level') + else + group_member_table[column_name] + end + end + + GroupMember + .with(cte.to_arel) + .select(*member_columns) + .from([group_member_table, cte.alias_to(group_group_link_table)]) + .where(group_member_table[:requested_at].eq(nil)) + .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) + .where(group_member_table[:source_type].eq('Namespace')) + end + # rubocop: enable Metrics/AbcSize + + def smallest_value_arel(args, column_alias) + Arel::Nodes::As.new( + Arel::Nodes::NamedFunction.new('LEAST', args), + Arel::Nodes::SqlLiteral.new(column_alias)) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb new file mode 100644 index 00000000000..6d7b9a86e69 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Namespace < ApplicationRecord + include ::Gitlab::VisibilityLevel + include ::Gitlab::Utils::StrongMemoize + include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal + + belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + + def visibility_level_field + :visibility_level + end + + def has_parent? + parent_id.present? || parent.present? + end + + # Overridden in EE::Namespace + def feature_available?(_feature) + false + end + end + end + end + end +end + +Namespace.prepend_if_ee('::EE::Namespace') diff --git a/lib/gitlab/bulk_import/client.rb b/lib/gitlab/bulk_import/client.rb new file mode 100644 index 00000000000..c6e77a158cd --- /dev/null +++ b/lib/gitlab/bulk_import/client.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module BulkImport + class Client + API_VERSION = 'v4'.freeze + DEFAULT_PAGE = 1.freeze + DEFAULT_PER_PAGE = 30.freeze + + ConnectionError = Class.new(StandardError) + + def initialize(uri:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION) + @uri = URI.parse(uri) + @token = token&.strip + @page = page + @per_page = per_page + @api_version = api_version + end + + def get(resource, query = {}) + response = with_error_handling do + Gitlab::HTTP.get( + resource_url(resource), + headers: request_headers, + follow_redirects: false, + query: query.merge(request_query) + ) + end + + response.parsed_response + end + + private + + def request_query + { + page: @page, + per_page: @per_page + } + end + + def request_headers + { + 'Content-Type' => 'application/json', + 'Authorization' => "Bearer #{@token}" + } + end + + def with_error_handling + response = yield + + raise ConnectionError.new("Error #{response.code}") unless response.success? + + response + rescue *Gitlab::HTTP::HTTP_ERRORS => e + raise ConnectionError, e + end + + def base_uri + @base_uri ||= "#{@uri.scheme}://#{@uri.host}:#{@uri.port}" + end + + def api_url + Gitlab::Utils.append_path(base_uri, "/api/#{@api_version}") + end + + def resource_url(resource) + Gitlab::Utils.append_path(api_url, resource) + end + end + end +end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 71361b12d07..db7af0088d0 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -20,3 +20,5 @@ module Gitlab end end end + +Gitlab::Checks::MatchingMergeRequest.prepend_if_ee('EE::Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb index b3c981d252b..0d93e7ac8a1 100644 --- a/lib/gitlab/checks/post_push_message.rb +++ b/lib/gitlab/checks/post_push_message.rb @@ -44,11 +44,7 @@ module Gitlab end def url_to_repo - protocol == 'ssh' ? message_subject.ssh_url_to_repo : message_subject.http_url_to_repo - end - - def message_subject - repository.repo_type.wiki? ? project.wiki : container + protocol == 'ssh' ? container.ssh_url_to_repo : container.http_url_to_repo end end end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 0373a12ab69..6d152c052dc 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -104,23 +104,24 @@ module Gitlab action = scanner[1] timestamp = scanner[2] section = scanner[3] + options = parse_section_options(scanner[4]) section_name = sanitize_section_name(section) - if action == "start" - handle_section_start(scanner, section_name, timestamp) - elsif action == "end" + if action == 'start' + handle_section_start(scanner, section_name, timestamp, options) + elsif action == 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' end end - def handle_section_start(scanner, section, timestamp) + def handle_section_start(scanner, section, timestamp, options) # We make a new line for new section flush_current_line - @state.open_section(section, timestamp) + @state.open_section(section, timestamp, options) # we need to consume match after handling # the open of section, as we want the section @@ -157,6 +158,18 @@ module Gitlab def sanitize_section_name(section) section.to_s.downcase.gsub(/[^a-z0-9]/, '-') end + + def parse_section_options(raw_options) + return unless raw_options + + # We need to remove the square brackets and split + # by comma to get a list of the options + options = raw_options[1...-1].split ',' + + # Now split each option by equals to separate + # each in the format [key, value] + options.to_h { |option| option.split '=' } + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21aa1f84353..b1dee0e1ecc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -32,7 +32,7 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration + :section_header, :section_duration, :section_options def initialize(offset:, style:, sections: []) @offset = offset @@ -68,6 +68,10 @@ module Gitlab @sections << section end + def set_section_options(options) + @section_options = options + end + def set_as_section_header @section_header = true end @@ -90,6 +94,7 @@ module Gitlab result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header result[:section_duration] = @section_duration if @section_duration + result[:section_options] = @section_options if @section_options end end end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 38d36e6950c..b2b6ce649ed 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -26,10 +26,11 @@ module Gitlab Base64.urlsafe_encode64(state.to_json) end - def open_section(section, timestamp) + def open_section(section, timestamp, options) @open_sections[section] = timestamp @current_line.add_section(section) + @current_line.set_section_options(options) @current_line.set_as_section_header end diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index 6395a20ca99..b0fad026ec5 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -45,14 +45,6 @@ module Gitlab end def read_zip_file!(file_path) - if ::Gitlab::Ci::Features.new_artifact_file_reader_enabled?(job.project) - read_with_new_artifact_file_reader(file_path) - else - read_with_legacy_artifact_file_reader(file_path) - end - end - - def read_with_new_artifact_file_reader(file_path) job.artifacts_file.use_open_file do |file| zip_file = Zip::File.new(file, false, true) entry = zip_file.find_entry(file_path) @@ -69,25 +61,6 @@ module Gitlab end end - def read_with_legacy_artifact_file_reader(file_path) - job.artifacts_file.use_file do |archive_path| - Zip::File.open(archive_path) do |zip_file| - entry = zip_file.find_entry(file_path) - unless entry - raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!" - end - - if entry.name_is_directory? - raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!" - end - - zip_file.get_input_stream(entry) do |is| - is.read - end - end - end - end - def max_archive_size_in_mb ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE) end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 9d269831679..071a8ef830f 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -54,6 +54,10 @@ module Gitlab root.variables_value end + def variables_with_data + root.variables_entry.value_with_data + end + def stages root.stages_value end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index a8b67a1db4f..1740032e5c7 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -11,15 +11,18 @@ module Gitlab class Bridge < ::Gitlab::Config::Entry::Node include ::Gitlab::Ci::Config::Entry::Processable + ALLOWED_WHEN = %w[on_success on_failure always manual].freeze ALLOWED_KEYS = %i[trigger].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do - validates :when, - inclusion: { in: %w[on_success on_failure always], - message: 'should be on_success, on_failure or always' } + validates :allow_failure, boolean: true + validates :when, inclusion: { + in: ALLOWED_WHEN, + message: "should be one of: #{ALLOWED_WHEN.join(', ')}" + } end validate on: :composed do @@ -57,11 +60,19 @@ module Gitlab true end + def manual_action? + self.when == 'manual' + end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end + def value super.merge( trigger: (trigger_value if trigger_defined?), needs: (needs_value if needs_defined?), - ignore: !!allow_failure, + ignore: ignored?, when: self.when, scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage ).compact diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index a304d9b724f..6b036182706 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -9,14 +9,28 @@ module Gitlab # class Cache < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[key untracked paths policy].freeze + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' validations do - validates :config, allowed_keys: ALLOWED_KEYS - validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } + end end entry :key, Entry::Key, @@ -28,13 +42,15 @@ module Gitlab entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - attributes :policy + attributes :policy, :when def value result = super result[:key] = key_value result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN result end diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index 9c2e5f641d0..ad0ed00aa6f 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -10,7 +10,7 @@ module Gitlab class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable - ALLOWED_KEYS = %i[local file remote template artifact job].freeze + ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze validations do validates :config, hash_or_string: true @@ -22,6 +22,10 @@ module Gitlab if config[:artifact] && config[:job].blank? errors.add(:config, "must specify the job where to fetch the artifact from") end + + if config[:project] && config[:file].blank? + errors.add(:config, "must specify the file where to fetch the config from") + end end end end diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index b5ce42969a5..b0fd9cef10b 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a set of jobs. # - class Jobs < ::Gitlab::Config::Entry::Node + class Jobs < ::Gitlab::Config::Entry::ComposableHash include ::Gitlab::Config::Entry::Validatable validations do @@ -36,6 +36,10 @@ module Gitlab end end + def composable_class(name, config) + self.class.find_type(name, config) + end + TYPES = [Entry::Hidden, Entry::Job, Entry::Bridge].freeze private_constant :TYPES @@ -49,29 +53,6 @@ module Gitlab type.matching?(name, config) end end - - # rubocop: disable CodeReuse/ActiveRecord - def compose!(deps = nil) - super do - @config.each do |name, config| - node = self.class.find_type(name, config) - next unless node - - factory = ::Gitlab::Config::Entry::Factory.new(node) - .value(config || {}) - .metadata(name: name) - .with(key: name, parent: self, - description: "#{name} job definition.") - - @entries[name] = factory.create! - end - - @entries.each_value do |entry| - entry.compose!(deps) - end - end - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb index d7ba8624882..66cd57b8cf3 100644 --- a/lib/gitlab/ci/config/entry/needs.rb +++ b/lib/gitlab/ci/config/entry/needs.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a set of needs dependencies. # - class Needs < ::Gitlab::Config::Entry::Node + class Needs < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -29,27 +29,16 @@ module Gitlab end end - def compose!(deps = nil) - super(deps) do - [@config].flatten.each_with_index do |need, index| - @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Need) - .value(need) - .with(key: "need", parent: self, description: "need definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each_value do |entry| - entry.compose!(deps) - end - end - end - def value - values = @entries.values.select(&:type) + values = @entries.select(&:type) values.group_by(&:type).transform_values do |values| values.map(&:value) end end + + def composable_class + Entry::Need + end end end end diff --git a/lib/gitlab/ci/config/entry/ports.rb b/lib/gitlab/ci/config/entry/ports.rb index 01ffcc7dd87..d26b31deca8 100644 --- a/lib/gitlab/ci/config/entry/ports.rb +++ b/lib/gitlab/ci/config/entry/ports.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a configuration of the ports of a Docker service. # - class Ports < ::Gitlab::Config::Entry::Node + class Ports < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -16,28 +16,8 @@ module Gitlab validates :config, port_unique: true end - def compose!(deps = nil) - super do - @entries = [] - @config.each do |config| - @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Port) - .value(config || {}) - .with(key: "port", parent: self, description: "port definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each do |entry| - entry.compose!(deps) - end - end - end - - def value - @entries.map(&:value) - end - - def descendants - @entries + def composable_class + Entry::Port end end end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb index 6af809d46c1..d4ee0978e1b 100644 --- a/lib/gitlab/ci/config/entry/product/matrix.rb +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -46,13 +46,11 @@ module Gitlab end end - # rubocop:disable CodeReuse/ActiveRecord def number_of_generated_jobs value.sum do |config| config.values.reduce(1) { |acc, values| acc * values.size } end end - # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index ac4f70fb69e..2481989060e 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: 2, + minimum: :minimum, too_short: 'requires at least %{count} items' } end @@ -28,6 +28,10 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end + + def minimum + ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 + end end end end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 0ae65f43723..f2fd8ac7fd9 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -15,7 +15,7 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_management license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing].freeze + requirements coverage_fuzzing api_fuzzing].freeze attributes ALLOWED_KEYS @@ -25,6 +25,7 @@ module Gitlab with_options allow_nil: true do validates :junit, array_of_strings_or_string: true + validates :api_fuzzing, array_of_strings_or_string: true validates :coverage_fuzzing, array_of_strings_or_string: true validates :sast, array_of_strings_or_string: true validates :sast, array_of_strings_or_string: true diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index 2fbc3d9e367..bf74f995e80 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -4,7 +4,7 @@ module Gitlab module Ci class Config module Entry - class Rules < ::Gitlab::Config::Entry::Node + class Rules < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -12,24 +12,13 @@ module Gitlab validates :config, type: Array end - def compose!(deps = nil) - super(deps) do - @config.each_with_index do |rule, index| - @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule) - .value(rule) - .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each_value do |entry| - entry.compose!(deps) - end - end - end - def value @config end + + def composable_class + Entry::Rules::Rule + end end end end diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 83baa83711f..44e2903a300 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -7,7 +7,7 @@ module Gitlab ## # Entry that represents a configuration of Docker services. # - class Services < ::Gitlab::Config::Entry::Node + class Services < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable validations do @@ -15,28 +15,8 @@ module Gitlab validates :config, services_with_ports_alias_unique: true, if: ->(record) { record.opt(:with_image_ports) } end - def compose!(deps = nil) - super do - @entries = [] - @config.each do |config| - @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service) - .value(config || {}) - .with(key: "service", parent: self, description: "service definition.") # rubocop:disable CodeReuse/ActiveRecord - .create! - end - - @entries.each do |entry| - entry.compose!(deps) - end - end - end - - def value - @entries.map(&:value) - end - - def descendants - @entries + def composable_class + Entry::Service end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index c9d0c7cb568..e258f7128fc 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -10,16 +10,32 @@ module Gitlab class Variables < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + ALLOWED_VALUE_DATA = %i[value description].freeze + validations do - validates :config, variables: true + validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA } + end + + def value + Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }] end def self.default(**) {} end - def value - Hash[@config.map { |key, value| [key.to_s, value.to_s] }] + def value_with_data + Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }] + end + + private + + def expand_value(value) + if value.is_a?(Hash) + { value: value[:value].to_s, description: value[:description] } + else + { value: value.to_s, description: nil } + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index e770187b124..1b58e3ec71a 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.job_heartbeats_runner?(project) - ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end @@ -35,10 +31,6 @@ module Gitlab ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.bulk_insert_on_create?(project) - ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a parituclar project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -54,25 +46,25 @@ module Gitlab Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.coverage_report_view?(project) - ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) - end - - def self.child_of_child_pipeline_enabled?(project) - ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) - end - def self.trace_overwrite? ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end def self.accept_trace?(project) ::Feature.enabled?(:ci_enable_live_trace, project) && - ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false) + ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true) + end + + def self.log_invalid_trace_chunks?(project) + ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) + end + + def self.one_dimensional_matrix_enabled? + ::Feature.enabled?(:one_dimensional_matrix, default_enabled: true) end - def self.new_artifact_file_reader_enabled?(project) - ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) + def self.manual_bridges_enabled?(project) + ::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 86a9ebfa451..44f2ac23ce3 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -4,10 +4,11 @@ module Gitlab module Ci class Lint class Result - attr_reader :jobs, :errors, :warnings + attr_reader :jobs, :merged_yaml, :errors, :warnings - def initialize(jobs:, errors:, warnings:) + def initialize(jobs:, merged_yaml:, errors:, warnings:) @jobs = jobs + @merged_yaml = merged_yaml @errors = errors @warnings = warnings end @@ -39,6 +40,7 @@ module Gitlab Result.new( jobs: dry_run_convert_to_jobs(pipeline.stages), + merged_yaml: pipeline.merged_yaml, errors: pipeline.error_messages.map(&:content), warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content) ) @@ -54,6 +56,7 @@ module Gitlab Result.new( jobs: static_validation_convert_to_jobs(result), + merged_yaml: result.merged_yaml, errors: result.errors, warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord ) diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 5746f38ae5b..50cd703da4a 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -8,12 +8,17 @@ module Gitlab JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze - def parse!(xml_data, test_suite, **args) + def parse!(xml_data, test_suite, job:) root = Hash.from_xml(xml_data) + total_parsed = 0 + max_test_cases = job.max_test_cases_per_report all_cases(root) do |test_case| - test_case = create_test_case(test_case, args) + test_case = create_test_case(test_case, test_suite, job) test_suite.add_test_case(test_case) + total_parsed += 1 + + ensure_test_cases_limited!(total_parsed, max_test_cases) end rescue Nokogiri::XML::SyntaxError => e test_suite.set_suite_error("JUnit XML parsing failed: #{e}") @@ -23,6 +28,12 @@ module Gitlab private + def ensure_test_cases_limited!(total_parsed, limit) + return unless limit > 0 && total_parsed > limit + + raise JunitParserError.new("number of test cases exceeded the limit of #{limit}") + end + def all_cases(root, parent = nil, &blk) return unless root.present? @@ -33,20 +44,24 @@ module Gitlab all_cases(node['testsuites'], root, &blk) unless parent # we require at least one level of testsuites or testsuite - each_case(node['testcase'], &blk) if parent + each_case(node['testcase'], node['name'], &blk) if parent # we allow multiple nested 'testsuite' (eg. PHPUnit) all_cases(node['testsuite'], root, &blk) end end - def each_case(testcase, &blk) + def each_case(testcase, testsuite_name, &blk) return unless testcase.present? - [testcase].flatten.compact.map(&blk) + [testcase].flatten.compact.each do |tc| + tc['suite_name'] = testsuite_name + + yield(tc) + end end - def create_test_case(data, args) + def create_test_case(data, test_suite, job) if data.key?('failure') status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED system_output = data['failure'] @@ -63,6 +78,7 @@ module Gitlab end ::Gitlab::Ci::Reports::TestCase.new( + suite_name: data['suite_name'] || test_suite.name, classname: data['classname'], name: data['name'], file: data['file'], @@ -70,10 +86,14 @@ module Gitlab status: status, system_output: system_output, attachment: attachment, - job: args.fetch(:job) + job: job ) end + def suite_name(parent, test_suite) + parent.dig('testsuite', 'name') || test_suite.name + end + def attachment_path(data) return unless data diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d1882059dd8..06096a33f27 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -16,7 +16,7 @@ module Gitlab ) do include Gitlab::Utils::StrongMemoize - def initialize(**params) + def initialize(params = {}) params.each do |key, value| self[key] = value end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 8ccb33ffd34..c3fbd0c9e24 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -28,6 +28,8 @@ module Gitlab error(result.errors.first, config_error: true) end + @pipeline.merged_yaml = result.merged_yaml + rescue => ex Gitlab::ErrorTracking.track_exception(ex, project_id: project.id, diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 34649fe16f3..81ef3bb074d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -8,7 +8,7 @@ module Gitlab include Chain::Helpers def perform! - BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + BulkInsertableAssociations.with_bulk_insert do pipeline.save! end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index a4127ea0be2..8d6fe13c3b9 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -13,6 +13,7 @@ module Gitlab @paths = local_cache.delete(:paths) @policy = local_cache.delete(:policy) @untracked = local_cache.delete(:untracked) + @when = local_cache.delete(:when) raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end @@ -24,7 +25,8 @@ module Gitlab key: key_string, paths: @paths, policy: @policy, - untracked: @untracked + untracked: @untracked, + when: @when }.compact.presence }.compact } diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 15a3c862c9e..8c70dbb6931 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -10,9 +10,10 @@ module Gitlab STATUS_ERROR = 'error' STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze - attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job + attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job def initialize(params) + @suite_name = params.fetch(:suite_name) @name = params.fetch(:name) @classname = params.fetch(:classname) @file = params.fetch(:file, nil) @@ -23,7 +24,7 @@ module Gitlab @attachment = params.fetch(:attachment, nil) @job = params.fetch(:job, nil) - @key = sanitize_key_name("#{classname}_#{name}") + @key = hash_key("#{suite_name}_#{classname}_#{name}") end def has_attachment? @@ -42,8 +43,8 @@ module Gitlab private - def sanitize_key_name(key) - key.gsub(/[^0-9A-Za-z]/, '-') + def hash_key(key) + Digest::SHA256.hexdigest(key) end end end diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index e9b78b841e4..00920dfbd54 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -12,18 +12,24 @@ module Gitlab def initialize(name = nil) @name = name @test_cases = {} + @all_test_cases = [] @total_time = 0.0 - @duplicate_cases = [] end def add_test_case(test_case) - @duplicate_cases << test_case if existing_key?(test_case) - @test_cases[test_case.status] ||= {} @test_cases[test_case.status][test_case.key] = test_case @total_time += test_case.execution_time end + def each_test_case + @test_cases.each do |status, test_cases| + test_cases.values.each do |test_case| + yield test_case + end + end + end + # rubocop: disable CodeReuse/ActiveRecord def total_count return 0 if suite_error @@ -86,10 +92,6 @@ module Gitlab private - def existing_key?(test_case) - @test_cases[test_case.status]&.key?(test_case.key) - end - def sort_by_status @test_cases = @test_cases.sort_by { |status, _| Gitlab::Ci::Reports::TestCase::STATUS_TYPES.index(status) }.to_h end diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb new file mode 100644 index 00000000000..95d7719e9cb --- /dev/null +++ b/lib/gitlab/ci/runner/backoff.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + ## + # Runner Backoff class is an implementation of an exponential backoff + # used when a runner communicates with GitLab. We typically use it when a + # runner retries sending a build status after we created a build pending + # state. + # + # Backoff is calculated based on the backoff slot which is always a power + # of 2: + # + # 0s - 3s duration -> 1 second backoff + # 4s - 7s duration -> 2 seconds backoff + # 8s - 15s duration -> 4 seconds backoff + # 16s - 31s duration -> 8 seconds backoff + # 32s - 63s duration -> 16 seconds backoff + # 64s - 127s duration -> 32 seconds backoff + # 127s - 256s+ duration -> 64 seconds backoff + # + # It means that first 15 requests made by a runner will need to respect + # following backoffs: + # + # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff) + # 1s -> 1 second backoff + # 2s -> 1 second backoff + # 3s -> 1 seconds backoff + # (slot 1 - 2^1 backoff) + # 4s -> 2 seconds backoff + # 6s -> 2 seconds backoff + # (slot 2 - 2^2 backoff) + # 8s -> 4 seconds backoff + # 12s -> 4 seconds backoff + # (slot 3 - 2^3 backoff) + # 16s -> 8 seconds backoff + # 24s -> 8 seconds backoff + # (slot 4 - 2^4 backoff) + # 32s -> 16 seconds backoff + # 48s -> 16 seconds backoff + # (slot 5 - 2^5 backoff) + # 64s -> 32 seconds backoff + # 96s -> 32 seconds backoff + # (slot 6 - 2^6 backoff) + # 128s -> 64 seconds backoff + # + # There is a cap on the backoff - it will never exceed 64 seconds. + # + class Backoff + def initialize(started) + @started = started + + if duration < 0 + raise ArgumentError, 'backoff duration negative' + end + end + + def duration + (Time.current - @started).ceil + end + + def slot + return 0 if duration < 2 + + Math.log(duration, 2).floor - 1 + end + + def to_seconds + 2**[slot, 6].min + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/action.rb b/lib/gitlab/ci/status/bridge/action.rb new file mode 100644 index 00000000000..1ba4700d9b0 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/action.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Action < Status::Build::Action + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb index b95565b5e09..d66d4b20bba 100644 --- a/lib/gitlab/ci/status/bridge/common.rb +++ b/lib/gitlab/ci/status/bridge/common.rb @@ -14,7 +14,6 @@ module Gitlab end def details_path - return unless Feature.enabled?(:ci_bridge_pipeline_details, subject.project, default_enabled: true) return unless can?(user, :read_pipeline, downstream_pipeline) project_pipeline_path(downstream_project, downstream_pipeline) diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index 5d397dba0de..b9bd66cee71 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -6,7 +6,10 @@ module Gitlab module Bridge class Factory < Status::Factory def self.extended_statuses - [Status::Bridge::Failed] + [[Status::Bridge::Failed], + [Status::Bridge::Manual], + [Status::Bridge::Play], + [Status::Bridge::Action]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/bridge/manual.rb b/lib/gitlab/ci/status/bridge/manual.rb new file mode 100644 index 00000000000..e07e645a34d --- /dev/null +++ b/lib/gitlab/ci/status/bridge/manual.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Manual < Status::Build::Manual + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/play.rb b/lib/gitlab/ci/status/bridge/play.rb new file mode 100644 index 00000000000..ae00ef6c2ad --- /dev/null +++ b/lib/gitlab/ci/status/bridge/play.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class Play < Status::Build::Play + def has_action? + can?(user, :play_job, subject) + end + + def self.matches?(bridge, user) + bridge.playable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index 07f37732023..f173964b36c 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_canceled' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index fface4bb97b..33e67314d93 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_created' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 770ed7d4d5a..215d27734a7 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_failed' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index 50c92add400..eb376df5f22 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_manual' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index cea7e6ed938..4280ad84534 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_pending' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb index 1ebdbc482b7..e59d1d2eed1 100644 --- a/lib/gitlab/ci/status/preparing.rb +++ b/lib/gitlab/ci/status/preparing.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_preparing' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index ac7dd74cdce..eed1983e60e 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_running' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index 16ad1da89e3..e9068c326cf 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_scheduled' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index aaec1e1d201..238aa3ab4f9 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_skipped' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 020f2c5b89f..2a10e60414e 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -19,6 +19,10 @@ module Gitlab def favicon 'favicon_status_success' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb index 4c9e706bc51..2026148f752 100644 --- a/lib/gitlab/ci/status/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/waiting_for_resource.rb @@ -23,6 +23,10 @@ module Gitlab def group 'waiting-for-resource' end + + def details_path + nil + end end end end diff --git a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml new file mode 100644 index 00000000000..267027a1b8a --- /dev/null +++ b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml @@ -0,0 +1,11 @@ +stages: + - provision + - review + - production + +variables: + AUTO_DEVOPS_PLATFORM_TARGET: EC2 + +include: + - template: Jobs/CF-Provision.gitlab-ci.yml + - template: Jobs/Deploy/EC2.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 6966ce88b30..cba13f374f4 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -20,6 +20,7 @@ # * dast: DAST_DISABLED # * review: REVIEW_DISABLED # * stop_review: REVIEW_DISABLED +# * code_intelligence: CODE_INTELLIGENCE_DISABLED # # In order to deploy, you must have a Kubernetes cluster configured either # via a project integration, or via group/project variables. @@ -159,6 +160,7 @@ include: - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 368069844ea..67e58d9ee99 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,4 +1,4 @@ -# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options +# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index c3568c0d2c8..0c5850bdb8e 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,7 +1,7 @@ # Based on openjdk:8, already includes lein image: clojure:lein-2.7.0 # If you need to configure a database, add a `services` section here -# See https://docs.gitlab.com/ce/ci/services/postgres.html +# See https://docs.gitlab.com/ee/ci/services/postgres.html # Make sure you configure the connection as well before_script: diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index e9301a2638d..538f96c4084 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -4,7 +4,7 @@ image: "crystallang/crystal:latest" # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service # services: # - mysql:latest # - redis:latest diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index d35fcb0f807..c657c7e8eb1 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -4,7 +4,7 @@ image: python:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - postgres:latest @@ -13,7 +13,7 @@ variables: POSTGRES_DB: database_name # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - ~/.cache/pip/ diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 4d4c6a64cd5..7271526ab1b 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -2,7 +2,7 @@ image: elixir:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml new file mode 100644 index 00000000000..31ca68c57d7 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -0,0 +1,14 @@ +stages: + - provision + +cloud_formation: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-cloudformation:latest' + stage: provision + script: + - gl-cloudformation create-stack + rules: + - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml new file mode 100644 index 00000000000..83bc5548614 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml @@ -0,0 +1,16 @@ +code_intelligence_go: + stage: test + needs: [] + allow_failure: true + image: sourcegraph/lsif-go:v1 + rules: + - if: $CODE_INTELLIGENCE_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.go' + script: + - lsif-go + artifacts: + reports: + lsif: dump.lsif diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 568ceceeaa2..ec33020205b 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -9,9 +9,8 @@ code_quality: DOCKER_TLS_CERTDIR: "" CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" needs: [] - before_script: - - export SOURCE_CODE=$PWD script: + - export SOURCE_CODE=$PWD - | if ! docker info &>/dev/null; then if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 86f946aafda..77216a6e404 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.5" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 829fd7a722f..32a207a85d1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.5" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 829fd7a722f..8b921305c11 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml new file mode 100644 index 00000000000..ed2172ef7f5 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -0,0 +1,39 @@ +stages: + - review + - production + +.push-and-deploy: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ec2:latest' + script: + - gl-ec2 push-to-s3 + - gl-ec2 deploy-to-ec2 + +review_ec2: + extends: .push-and-deploy + stage: review + environment: + name: review/$CI_COMMIT_REF_NAME + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +production_ec2: + extends: .push-and-deploy + stage: production + environment: + name: production + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index da474f8ac88..317e8bfab0e 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -10,6 +10,7 @@ .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + dependencies: [] script: - ecs update-task-definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 9bde04dff19..5d2c8024524 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -4,7 +4,7 @@ image: php:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest @@ -13,7 +13,7 @@ variables: MYSQL_ROOT_PASSWORD: secret # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - vendor/ diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 7050b41e045..a9638f564f3 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.29.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.33.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index b87178141a1..92379ded77c 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -4,14 +4,14 @@ image: node:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - redis:latest - postgres:latest # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 25ea20e454f..84e8223e69b 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -19,7 +19,7 @@ before_script: - php composer.phar install # Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service -# See http://docs.gitlab.com/ce/ci/services/README.html for examples. +# See http://docs.gitlab.com/ee/ci/services/README.html for examples. services: - mysql:5.7 diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index a683561a455..3a6eac63892 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -1,7 +1,7 @@ image: node:latest # This folder is cached between builds -# http://docs.gitlab.com/ce/ci/yaml/README.html#cache +# http://docs.gitlab.com/ee/ci/yaml/README.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 0c8859dc779..f2f92fe0704 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -1,5 +1,5 @@ # Template project: https://gitlab.com/pages/jekyll -# Docs: https://docs.gitlab.com/ce/pages/ +# Docs: https://docs.gitlab.com/ee/pages/ image: ruby:2.6 variables: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index b3cad8b858a..275364afae4 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -4,7 +4,7 @@ image: "ruby:2.5" # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index f35470367cc..94117a79d1c 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -4,7 +4,7 @@ image: "rust:latest" # Optional: Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service +# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service # services: # - mysql:latest # - redis:latest diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 4b957a8f771..e268b48d133 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -35,4 +35,3 @@ variables: - if: $COVFUZZ_DISABLED when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ - - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux" diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index cc34d23decc..63237e41376 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/ # # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/license-management#settings +# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 77ea11d01d1..4418ff18d73 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SCAN_KUBERNETES_MANIFESTS: "false" @@ -125,6 +125,42 @@ gosec-sast: exists: - '**/*.go' +mobsf-android-sast: + extends: .sast-analyzer + services: + - name: opensecurity/mobile-security-framework-mobsf:latest + alias: mobsf + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + +mobsf-ios-sast: + extends: .sast-analyzer + services: + - name: opensecurity/mobile-security-framework-mobsf:latest + alias: mobsf + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.xcodeproj/*' + nodejs-scan-sast: extends: .sast-analyzer image: @@ -203,6 +239,11 @@ spotbugs-sast: variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: + - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/AndroidManifest.xml' + when: never - if: $SAST_DISABLED when: never - if: $CI_COMMIT_BRANCH && diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index bde6a0fbebb..6ebff102ccb 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -34,8 +34,8 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: - - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME - - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt + - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME + - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml new file mode 100644 index 00000000000..b08ccf18b58 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -0,0 +1,22 @@ +include: + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml + +stages: + - init + - validate + - build + - deploy + +init: + extends: .init + +validate: + extends: .validate + +build: + extends: .build + +deploy: + extends: .deploy + dependencies: + - build diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml new file mode 100644 index 00000000000..000a1a7f580 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -0,0 +1,53 @@ +# Terraform/Base.latest +# +# The purpose of this template is to provide flexibility to the user so +# they are able to only include the jobs that they find interesting. +# +# Therefore, this template is not supposed to run any jobs. The idea is to only +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# +# There is a more opinionated template which we suggest the users to abide, +# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml + +image: + name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + +before_script: + - cd ${TF_ROOT} + +variables: + TF_ROOT: ${CI_PROJECT_DIR} + +cache: + key: "${TF_ROOT}" + paths: + - ${TF_ROOT}/.terraform/ + +.init: &init + stage: init + script: + - gitlab-terraform init + +.validate: &validate + stage: validate + script: + - gitlab-terraform validate + +.build: &build + stage: build + script: + - gitlab-terraform plan + - gitlab-terraform plan-json + artifacts: + paths: + - ${TF_ROOT}/plan.cache + reports: + terraform: ${TF_ROOT}/plan.json + +.deploy: &deploy + stage: deploy + script: + - gitlab-terraform apply + when: manual + only: + - master diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 348e5472cb4..0222ca021b7 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -16,6 +16,7 @@ module Gitlab ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) + LockedError = Class.new(StandardError) attr_reader :job @@ -79,11 +80,9 @@ module Gitlab job.trace_chunks.any? || current_path.present? || old_trace.present? end - def read(should_retry: true, &block) + def read(&block) read_stream(&block) - rescue Errno::ENOENT - raise unless should_retry - + rescue Errno::ENOENT, ChunkedIO::FailedToGetChunkError job.reset read_stream(&block) end @@ -130,6 +129,12 @@ module Gitlab end end + def lock(&block) + in_write_lock(&block) + rescue FailedToObtainLockError + raise LockedError, "build trace `#{job.id}` is locked" + end + private def read_stream diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb new file mode 100644 index 00000000000..62532ef1cd2 --- /dev/null +++ b/lib/gitlab/ci/trace/checksum.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Checksum class is responsible for calculating a CRC32 checksum + # of an entire build trace using partial build trace chunks stored in a + # database. + # + # CRC32 checksum can be easily calculated by combining partial checksums + # in a right order. + # + # Then we compare CRC32 checksum provided by a GitLab Runner and expect + # it to be the same as the CRC32 checksum derived from partial chunks. + # + class Checksum + include Gitlab::Utils::StrongMemoize + + attr_reader :build + + def initialize(build) + @build = build + end + + def valid? + return false unless state_crc32.present? + + state_crc32 == chunks_crc32 + end + + def state_crc32 + strong_memoize(:state_crc32) { build.pending_state&.crc32 } + end + + def chunks_crc32 + strong_memoize(:chunks_crc32) do + trace_chunks.reduce(0) do |crc32, chunk| + Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk)) + end + end + end + + def last_chunk + strong_memoize(:last_chunk) { trace_chunks.max } + end + + ## + # Trace chunks will be persisted in a database if an object store is + # not configured - in that case we do not want to load entire raw data + # of all the chunks into memory. + # + # We ignore `raw_data` attribute instead, and rely on internal build + # trace chunk database adapter to handle + # `ActiveModel::MissingAttributeError` exception. + # + # Alternative solution would be separating chunk data from chunk + # metadata on the database level too. + # + def trace_chunks + strong_memoize(:trace_chunks) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end + end + + def chunks_count + trace_chunks.to_a.size + end + + private + + def chunk_size(chunk) + if chunk == last_chunk + chunk.size + else + ::Ci::BuildTraceChunk::CHUNK_SIZE + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 82a7d5fb83c..097436d84ea 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -6,8 +6,20 @@ module Gitlab class Metrics extend Gitlab::Utils::StrongMemoize - OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, - :accepted, :finalized, :discarded, :conflict].freeze + OPERATIONS = [ + :appended, # new trace data has been written to a chunk + :streamed, # new trace data has been sent by a runner + :chunked, # new trace chunk has been created + :mutated, # trace has been mutated when removing secrets + :overwrite, # runner requested overwritting a build trace + :accepted, # scheduled chunks for migration and responded with 202 + :finalized, # all live build trace chunks have been persisted + :discarded, # failed to persist live chunks before timeout + :conflict, # runner has sent unrecognized build state details + :locked, # build trace has been locked by a different mechanism + :stalled, # failed to migrate chunk due to a worker duplication + :invalid # malformed build trace has been detected using CRC32 + ].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) @@ -18,7 +30,11 @@ module Gitlab end def increment_trace_bytes(size) - self.class.trace_bytes.increment(by: size.to_i) + self.class.trace_bytes.increment({}, size.to_i) + end + + def observe_migration_duration(seconds) + self.class.finalize_histogram.observe({}, seconds.to_f) end def self.trace_operations @@ -38,6 +54,17 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def self.finalize_histogram + strong_memoize(:finalize_histogram) do + name = :gitlab_ci_trace_finalize_duration_seconds + comment = 'Duration of build trace chunks migration to object storage' + buckets = [0.1, 0.5, 1.0, 2.0, 3.0, 5.0, 7.0, 10.0, 20.0, 30.0, 60.0, 300.0] + labels = {} + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index a072036daa8..84a9280e507 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -36,9 +36,9 @@ module Gitlab def self.fabricate(resource) case resource when Hash - self.new(resource.symbolize_keys) + self.new(**resource.symbolize_keys) when ::Ci::HasVariable - self.new(resource.to_runner_variable) + self.new(**resource.to_runner_variable) when self resource.dup else diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 68f61e52df7..52a00e41214 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -95,6 +95,14 @@ module Gitlab }.compact }.compact end + def merged_yaml + @ci_config&.to_hash&.to_yaml + end + + def variables_with_data + @ci_config.variables_with_data + end + private def variables diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index 14eac474e27..a6638b2cbc8 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -19,6 +19,11 @@ module Gitlab def run! log_info("Looking for orphan LFS files for project #{project.name_with_namespace}") + if project.lfs_objects.empty? + log_info("Project #{project.name_with_namespace} is linked to 0 LFS objects. Nothing to do") + return + end + remove_orphan_references end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 92c799875b5..822012e0ed6 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -5,8 +5,8 @@ module Gitlab class PumaWorkerKillerInitializer def self.start( puma_options, - puma_per_worker_max_memory_mb: 850, - puma_master_max_memory_mb: 550, + puma_per_worker_max_memory_mb: 1024, + puma_master_max_memory_mb: 800, additional_puma_dev_max_memory_mb: 200 ) require 'puma_worker_killer' diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index 909d0536b5f..7d36f2f12cf 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -13,7 +13,6 @@ module Gitlab end def full_json_path_for(path) - return unless Feature.enabled?(:code_navigation, project, default_enabled: true) return unless build raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif) diff --git a/lib/gitlab/config/entry/composable_array.rb b/lib/gitlab/config/entry/composable_array.rb new file mode 100644 index 00000000000..e7ad259e826 --- /dev/null +++ b/lib/gitlab/config/entry/composable_array.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Entry that represents a composable array definition + # + class ComposableArray < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include Gitlab::Utils::StrongMemoize + + # TODO: Refactor `Validatable` code so that validations can apply to a child class + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/263231 + validations do + validates :config, type: Array + end + + def compose!(deps = nil) + super do + @entries = Array(@entries) + + # TODO: Isolate handling for a hash via: `[@config].flatten` to the `Needs` entry + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/264376 + [@config].flatten.each_with_index do |value, index| + raise ArgumentError, 'Missing Composable class' unless composable_class + + composable_class_name = composable_class.name.demodulize.underscore + + @entries << ::Gitlab::Config::Entry::Factory.new(composable_class) + .value(value) + .with(key: composable_class_name, parent: self, description: "#{composable_class_name} definition") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each do |entry| + entry.compose!(deps) + end + end + end + + def value + @entries.map(&:value) + end + + def descendants + @entries + end + + def composable_class + strong_memoize(:composable_class) do + opt(:composable_class) + end + end + end + end + end +end diff --git a/lib/gitlab/config/entry/composable_hash.rb b/lib/gitlab/config/entry/composable_hash.rb new file mode 100644 index 00000000000..9531b7e56fd --- /dev/null +++ b/lib/gitlab/config/entry/composable_hash.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + ## + # Entry that represents a composable hash definition + # Where each hash key can be any value written by the user + # + class ComposableHash < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + # TODO: Refactor `Validatable` code so that validations can apply to a child class + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/263231 + validations do + validates :config, type: Hash + end + + def compose!(deps = nil) + super do + @config.each do |name, config| + entry_class = composable_class(name, config) + raise ArgumentError, 'Missing Composable class' unless entry_class + + entry_class_name = entry_class.name.demodulize.underscore + + factory = ::Gitlab::Config::Entry::Factory.new(entry_class) + .value(config || {}) + .with(key: name, parent: self, description: "#{name} #{entry_class_name} definition") # rubocop:disable CodeReuse/ActiveRecord + .metadata(name: name) + + @entries[name] = factory.create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def composable_class(name, config) + opt(:composable_class) + end + end + end + end +end diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 7c5ffaa7621..f76c98f7cbf 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -79,7 +79,7 @@ module Gitlab end def fabricate(entry_class, value = nil) - entry_class.new(value, @metadata) do |node| + entry_class.new(value, **@metadata) do |node| node.key = @attributes[:key] node.parent = @attributes[:parent] node.default = @attributes[:default] diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index 415f6f77214..be7d26fed4e 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -50,6 +50,12 @@ module Gitlab variables.values.flatten(1).all?(&method(:validate_alphanumeric)) end + def validate_string_or_hash_value_variables(variables, allowed_value_data) + variables.is_a?(Hash) && + variables.keys.all?(&method(:validate_alphanumeric)) && + variables.values.all? { |value| validate_string_or_hash_value_variable(value, allowed_value_data) } + end + def validate_alphanumeric(value) validate_string(value) || validate_integer(value) end @@ -62,6 +68,14 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_string_or_hash_value_variable(value, allowed_value_data) + if value.is_a?(Hash) + (value.keys - allowed_value_data).empty? && value.values.all?(&method(:validate_alphanumeric)) + else + validate_alphanumeric(value) + end + end + def validate_regexp(value) Gitlab::UntrustedRegexp::RubySyntax.valid?(value) end diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index 315f1947e2c..ee28891a174 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -19,7 +19,7 @@ module Gitlab entry = self.class.entry_class(strategy) - @subject = entry.new(config, metadata, &blk) + @subject = entry.new(config, **metadata, &blk) super(@subject) end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index a7ec98ace6e..2a386657e0b 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -274,6 +274,8 @@ module Gitlab def validate_each(record, attribute, value) if options[:array_values] validate_key_array_values(record, attribute, value) + elsif options[:allowed_value_data] + validate_key_hash_values(record, attribute, value, options[:allowed_value_data]) else validate_key_values(record, attribute, value) end @@ -290,6 +292,12 @@ module Gitlab record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array') end end + + def validate_key_hash_values(record, attribute, value, allowed_value_data) + unless validate_string_or_hash_value_variables(value, allowed_value_data) + record.errors.add(attribute, 'should be a hash of key value pairs, value can be a hash') + end + end end class ExpressionValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 53406af2c4e..047600af267 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -21,11 +21,13 @@ module Gitlab def resolve(user, commit_message, files) msg = commit_message || default_commit_message resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) - args = { + + resolver.resolve_conflicts( + @source_repo, + resolution, source_branch: merge_request.source_branch, target_branch: merge_request.target_branch - } - resolver.resolve_conflicts(@source_repo, resolution, args) + ) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 954934518d7..7b01db125a9 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -10,7 +10,7 @@ module Gitlab MAX_LINE_LENGTH = 72 MAX_CHANGED_FILES_IN_COMMIT = 3 MAX_CHANGED_LINES_IN_COMMIT = 30 - SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze + SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze DEFAULT_SUBJECT_DESCRIPTION = 'commit subject' WIP_PREFIX = 'WIP: ' PROBLEMS = { @@ -118,7 +118,7 @@ module Gitlab next unless line_too_long?(line) - url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord + url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but # only if the line _without_ the URL does not exceed this limit. diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 3626ec5bf5b..783a5f1715c 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -123,7 +123,8 @@ module Gitlab none: "", qa: "~QA", test: "~test ~Quality for `spec/features/*`", - engineering_productivity: '~"Engineering Productivity" for CI, Danger' + engineering_productivity: '~"Engineering Productivity" for CI, Danger', + ci_template: '~"ci::templates"' }.freeze # First-match win, so be sure to put more specific regex at the top... CATEGORIES = { @@ -176,6 +177,8 @@ module Gitlab %r{(CODEOWNERS)} => :engineering_productivity, %r{(tests.yml)} => :engineering_productivity, + %r{\Alib/gitlab/ci/templates} => :ci_template, + %r{\A(ee/)?spec/features/} => :test, %r{\A(ee/)?spec/support/shared_examples/features/} => :test, %r{\A(ee/)?spec/support/shared_contexts/features/} => :test, @@ -214,6 +217,12 @@ module Gitlab title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') end + def draft_mr? + return false unless gitlab_helper + + DRAFT_REGEX.match?(gitlab_helper.mr_json['title']) + end + def security_mr? return false unless gitlab_helper diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index a6866868e6c..23f877b4e0f 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -52,6 +52,11 @@ module Gitlab # Fetch an already picked backend maintainer, or pick one otherwise spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer end + when :ci_template + if spin.maintainer.nil? + # Fetch an already picked backend maintainer, or pick one otherwise + spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer + end end end @@ -146,13 +151,19 @@ module Gitlab %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) end + hungry_reviewers = reviewers.select { |member| member.hungry } + hungry_traintainers = traintainers.select { |member| member.hungry } # TODO: take CODEOWNERS into account? # https://gitlab.com/gitlab-org/gitlab/issues/26723 - # Make traintainers have triple the chance to be picked as a reviewer random = new_random(mr_source_branch) - reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment) + + # Make hungry traintainers have 4x the chance to be picked as a reviewer + # Make traintainers have 3x the chance to be picked as a reviewer + # Make hungry reviewers have 2x the chance to be picked as a reviewer + weighted_reviewers = reviewers + hungry_reviewers + traintainers + traintainers + traintainers + hungry_traintainers + reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) Spin.new(category, reviewer, maintainer, false, timezone_experiment) diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index ebd96be40d7..4481977db15 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,7 +3,7 @@ module Gitlab module Danger class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) @@ -14,6 +14,7 @@ module Gitlab @role = options['role'] @projects = options['projects'] @available = options['available'] + @hungry = options['hungry'] @tz_offset_hours = options['tz_offset_hours'] end @@ -31,10 +32,8 @@ module Gitlab projects&.has_key?(name) end - # Traintainers also count as reviewers def reviewer?(project, category, labels) - has_capability?(project, category, :reviewer, labels) || - traintainer?(project, category, labels) + has_capability?(project, category, :reviewer, labels) end def traintainer?(project, category, labels) diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 70587b3132a..87ebe832862 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -20,8 +20,8 @@ module Gitlab environment: deployment.environment.name, project: deployment.project.hook_attrs, short_sha: deployment.short_sha, - user: deployment.user.hook_attrs, - user_url: Gitlab::UrlBuilder.build(deployment.user), + user: deployment.deployed_by.hook_attrs, + user_url: Gitlab::UrlBuilder.build(deployment.deployed_by), commit_url: Gitlab::UrlBuilder.build(deployment.commit), commit_title: deployment.commit.title } diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index accc6330253..45d271a2fd4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -92,10 +92,6 @@ module Gitlab @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end - def self.postgresql_9_or_less? - version.to_f < 10 - end - def self.postgresql_minimum_supported_version? version.to_f >= MINIMUM_POSTGRES_VERSION end @@ -127,28 +123,6 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end - # map some of the function names that changed between PostgreSQL 9 and 10 - # https://wiki.postgresql.org/wiki/New_in_postgres_10 - def self.pg_wal_lsn_diff - Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff' - end - - def self.pg_current_wal_insert_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn' - end - - def self.pg_last_wal_receive_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn' - end - - def self.pg_last_wal_replay_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn' - end - - def self.pg_last_xact_replay_timestamp - 'pg_last_xact_replay_timestamp' - end - def self.nulls_last_order(field, direction = 'ASC') Arel.sql("#{field} #{direction} NULLS LAST") end @@ -276,6 +250,20 @@ module Gitlab false end + def self.system_id + row = connection.execute('SELECT system_identifier FROM pg_control_system()').first + + row['system_identifier'] + end + + def self.get_write_location(ar_connection) + row = ar_connection + .select_all("SELECT pg_current_wal_insert_lsn()::text AS location") + .first + + row['location'] if row + end + private_class_method :database_version def self.add_post_migrate_path_to_rails(force: false) diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 1762b81b7d8..11d9881aac2 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -8,15 +8,20 @@ # In order to not use a possible complex time consuming query when calculating min and max for batch_distinct_count # the start and finish can be sent specifically # +# Grouped relations can be used as well. However, the preferred batch count should be around 10K because group by count is more expensive. +# # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 # # Examples: # extend ::Gitlab::Database::BatchCount # batch_count(User.active) # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +# batch_count(Namespace.group(:type)) # batch_distinct_count(::Project, :creator_id) # batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) +# batch_distinct_count(Project.group(:visibility_level), :creator_id) # batch_sum(User, :sign_in_count) +# batch_sum(Issue.group(:state_id), :weight)) module Gitlab module Database module BatchCount @@ -77,34 +82,45 @@ module Gitlab raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 return FALLBACK if unwanted_configuration?(finish, batch_size, start) - counter = 0 + results = nil batch_start = start while batch_start <= finish + batch_relation = build_relation_batch(batch_start, batch_start + batch_size, mode) begin - counter += batch_fetch(batch_start, batch_start + batch_size, mode) + results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend batch_start += batch_size - rescue ActiveRecord::QueryCanceled + rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE batch_size /= 2 else + log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) return FALLBACK end end sleep(SLEEP_TIME_IN_SECONDS) end - counter + results end - def batch_fetch(start, finish, mode) - # rubocop:disable GitlabSecurity/PublicSend - @relation.select(@column).public_send(mode).where(between_condition(start, finish)).send(@operation, *@operation_args) + def merge_results(results, object) + return object unless results + + if object.is_a?(Hash) + results.merge!(object) { |_, a, b| a + b } + else + results + object + end end private + def build_relation_batch(start, finish, mode) + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + end + def batch_size_for_mode_and_operation(mode, operation) return DEFAULT_SUM_BATCH_SIZE if operation == :sum @@ -118,11 +134,11 @@ module Gitlab end def actual_start(start) - start || @relation.minimum(@column) || 0 + start || @relation.unscope(:group, :having).minimum(@column) || 0 end def actual_finish(finish) - finish || @relation.maximum(@column) || 0 + finish || @relation.unscope(:group, :having).maximum(@column) || 0 end def check_mode!(mode) @@ -130,6 +146,20 @@ module Gitlab raise 'Use distinct count for optimized distinct counting' if @relation.limit(1).distinct_value.present? && mode != :distinct raise 'Use distinct count only with non id fields' if @column == :id && mode == :distinct end + + def log_canceled_batch_fetch(batch_start, mode, query, error) + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + start: batch_start, + mode: mode, + query: query, + message: "Query has been canceled with message: #{error.message}" + ) + end end end end diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb new file mode 100644 index 00000000000..1403d561890 --- /dev/null +++ b/lib/gitlab/database/bulk_update.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Constructs queries of the form: + # + # with cte(a, b, c) as ( + # select * from (values (:x, :y, :z), (:q, :r, :s)) as t + # ) + # update table set b = cte.b, c = cte.c where a = cte.a + # + # Which is useful if you want to update a set of records in a single query + # but cannot express the update as a calculation (i.e. you have arbitrary + # updates to perform). + # + # The requirements are that the table must have an ID column used to + # identify the rows to be updated. + # + # Usage: + # + # mapping = { + # issue_a => { title: 'This title', relative_position: 100 }, + # issue_b => { title: 'That title', relative_position: 173 } + # } + # + # ::Gitlab::Database::BulkUpdate.execute(%i[title relative_position], mapping) + # + # Note that this is a very low level tool, and operates on the raw column + # values. Enums/state fields must be translated into their underlying + # representations, for example, and no hooks will be called. + # + module BulkUpdate + LIST_SEPARATOR = ', ' + + class Setter + include Gitlab::Utils::StrongMemoize + + def initialize(model, columns, mapping) + @table_name = model.table_name + @connection = model.connection + @columns = self.class.column_definitions(model, columns) + @mapping = self.class.value_mapping(mapping) + end + + def update! + if without_prepared_statement? + # A workaround for https://github.com/rails/rails/issues/24893 + # When prepared statements are prevented (such as when using the + # query counter or in omnibus by default), we cannot call + # `exec_update`, since that will discard the bindings. + connection.send(:exec_no_cache, sql, log_name, params) # rubocop: disable GitlabSecurity/PublicSend + else + connection.exec_update(sql, log_name, params) + end + end + + def self.column_definitions(model, columns) + raise ArgumentError, 'invalid columns' if columns.blank? || columns.any? { |c| !c.is_a?(Symbol) } + raise ArgumentError, 'cannot set ID' if columns.include?(:id) + + ([:id] | columns).map { |name| column_definition(model, name) } + end + + def self.column_definition(model, name) + definition = model.column_for_attribute(name) + raise ArgumentError, "Unknown column: #{name}" unless definition.type + + definition + end + + def self.value_mapping(mapping) + raise ArgumentError, 'invalid mapping' if mapping.blank? + raise ArgumentError, 'invalid mapping value' if mapping.any? { |_k, v| !v.is_a?(Hash) } + + mapping + end + + private + + attr_reader :table_name, :connection, :columns, :mapping + + def log_name + strong_memoize(:log_name) do + "BulkUpdate #{table_name} #{columns.drop(1).map(&:name)}:#{mapping.size}" + end + end + + def params + mapping.flat_map do |k, v| + obj_id = k.try(:id) || k + v = v.merge(id: obj_id) + columns.map { |c| query_attribute(c, k, v.with_indifferent_access) } + end + end + + # A workaround for https://github.com/rails/rails/issues/24893 + # We need to detect if prepared statements have been disabled. + def without_prepared_statement? + strong_memoize(:without_prepared_statement) do + connection.send(:without_prepared_statement?, [1]) # rubocop: disable GitlabSecurity/PublicSend + end + end + + def query_attribute(column, key, values) + value = values[column.name] + key[column.name] = value if key.try(:id) # optimistic update + ActiveRecord::Relation::QueryAttribute.from_user(nil, value, ActiveModel::Type.lookup(column.type)) + end + + def values + counter = 0 + typed = false + + mapping.map do |k, v| + binds = columns.map do |c| + bind = "$#{counter += 1}" + # PG is not great at inferring types - help it for the first row. + bind += "::#{c.sql_type}" unless typed + bind + end + typed = true + + "(#{list_of(binds)})" + end + end + + def list_of(list) + list.join(LIST_SEPARATOR) + end + + def sql + <<~SQL + WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)}) + UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id + SQL + end + + def column_names + strong_memoize(:column_names) { columns.map(&:name) } + end + + def cte_columns + strong_memoize(:cte_columns) do + column_names.map do |c| + connection.quote_column_name("cte_#{c}") + end + end + end + + def updates + column_names.zip(cte_columns).drop(1).map do |dest, src| + "#{connection.quote_column_name(dest)} = cte.#{src}" + end + end + end + + def self.execute(columns, mapping, &to_class) + raise ArgumentError if mapping.blank? + + entries_by_class = mapping.group_by { |k, v| block_given? ? to_class.call(k) : k.class } + + entries_by_class.each do |model, entries| + Setter.new(model, columns, entries).update! + end + end + end + end +end diff --git a/lib/gitlab/database/concurrent_reindex.rb b/lib/gitlab/database/concurrent_reindex.rb deleted file mode 100644 index 485ab35e55d..00000000000 --- a/lib/gitlab/database/concurrent_reindex.rb +++ /dev/null @@ -1,143 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class ConcurrentReindex - include Gitlab::Utils::StrongMemoize - include MigrationHelpers - - ReindexError = Class.new(StandardError) - - PG_IDENTIFIER_LENGTH = 63 - TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' - REPLACED_INDEX_PREFIX = 'old_reindex_' - - attr_reader :index_name, :logger - - def initialize(index_name, logger:) - @index_name = index_name - @logger = logger - end - - def execute - raise ReindexError, "index #{index_name} does not exist" unless index_exists? - - raise ReindexError, 'UNIQUE indexes are currently not supported' if index_unique? - - logger.debug("dropping dangling index from previous run: #{replacement_index_name}") - remove_replacement_index - - begin - create_replacement_index - - unless replacement_index_valid? - message = 'replacement index was created as INVALID' - logger.error("#{message}, cleaning up") - raise ReindexError, "failed to reindex #{index_name}: #{message}" - end - - swap_replacement_index - rescue Gitlab::Database::WithLockRetries::AttemptsExhaustedError => e - logger.error('failed to obtain the required database locks to swap the indexes, cleaning up') - raise ReindexError, e.message - rescue ActiveRecord::ActiveRecordError, PG::Error => e - logger.error("database error while attempting reindex of #{index_name}: #{e.message}") - raise ReindexError, e.message - ensure - logger.info("dropping unneeded replacement index: #{replacement_index_name}") - remove_replacement_index - end - end - - private - - def connection - @connection ||= ActiveRecord::Base.connection - end - - def replacement_index_name - @replacement_index_name ||= constrained_index_name(TEMPORARY_INDEX_PREFIX) - end - - def index - strong_memoize(:index) do - find_index(index_name) - end - end - - def index_exists? - !index.nil? - end - - def index_unique? - index.indisunique - end - - def constrained_index_name(prefix) - "#{prefix}#{index_name}".slice(0, PG_IDENTIFIER_LENGTH) - end - - def create_replacement_index - create_replacement_index_statement = index.indexdef - .sub(/CREATE INDEX/, 'CREATE INDEX CONCURRENTLY') - .sub(/#{index_name}/, replacement_index_name) - - logger.info("creating replacement index #{replacement_index_name}") - logger.debug("replacement index definition: #{create_replacement_index_statement}") - - disable_statement_timeout do - connection.execute(create_replacement_index_statement) - end - end - - def replacement_index_valid? - find_index(replacement_index_name).indisvalid - end - - def find_index(index_name) - record = connection.select_one(<<~SQL) - SELECT - pg_index.indisunique, - pg_index.indisvalid, - pg_indexes.indexdef - FROM pg_index - INNER JOIN pg_class ON pg_class.oid = pg_index.indexrelid - INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid - INNER JOIN pg_indexes ON pg_class.relname = pg_indexes.indexname - WHERE pg_namespace.nspname = 'public' - AND pg_class.relname = #{connection.quote(index_name)} - SQL - - OpenStruct.new(record) if record - end - - def swap_replacement_index - replaced_index_name = constrained_index_name(REPLACED_INDEX_PREFIX) - - logger.info("swapping replacement index #{replacement_index_name} with #{index_name}") - - with_lock_retries do - rename_index(index_name, replaced_index_name) - rename_index(replacement_index_name, index_name) - rename_index(replaced_index_name, replacement_index_name) - end - end - - def rename_index(old_index_name, new_index_name) - connection.execute("ALTER INDEX #{old_index_name} RENAME TO #{new_index_name}") - end - - def remove_replacement_index - disable_statement_timeout do - connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{replacement_index_name}") - end - end - - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - - Gitlab::Database::WithLockRetries.new(arguments).run(raise_on_exhaustion: true, &block) - end - end - end -end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index e226ed7613a..89190320cf9 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -74,8 +74,9 @@ module Gitlab def get_statistics(table_names, check_statistics: true) time = 6.hours.ago - query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") + query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") .where(relname: table_names) + .where('schemaname = current_schema()') .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') if check_statistics diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 723f0f6a308..66b6ce1ec55 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -176,7 +176,7 @@ module Gitlab name: name.presence || concurrent_foreign_key_name(source, column) } - if foreign_key_exists?(source, target, options) + if foreign_key_exists?(source, target, **options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ @@ -330,13 +330,13 @@ module Gitlab # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` # * +logger+ - [Gitlab::JsonLogger] # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` - def with_lock_retries(**args, &block) + def with_lock_retries(*args, **kwargs, &block) merged_args = { klass: self.class, logger: Gitlab::BackgroundMigration::Logger - }.merge(args) + }.merge(kwargs) - Gitlab::Database::WithLockRetries.new(merged_args).run(&block) + Gitlab::Database::WithLockRetries.new(**merged_args).run(&block) end def true_value @@ -544,6 +544,16 @@ module Gitlab rename_column_concurrently(table, column, temp_column, type: new_type, type_cast_function: type_cast_function, batch_column_name: batch_column_name) end + # Reverses operations performed by change_column_type_concurrently. + # + # table - The table containing the column. + # column - The name of the column to change. + def undo_change_column_type_concurrently(table, column) + temp_column = "#{column}_for_type_change" + + undo_rename_column_concurrently(table, column, temp_column) + end + # Performs cleanup of a concurrent type change. # # table - The table containing the column. @@ -560,6 +570,65 @@ module Gitlab end end + # Reverses operations performed by cleanup_concurrent_column_type_change. + # + # table - The table containing the column. + # column - The name of the column to change. + # old_type - The type of the original column used with change_column_type_concurrently. + # type_cast_function - Required if the conversion back to the original type is not automatic + # batch_column_name - option for tables without a primary key, in this case + # another unique integer column can be used. Example: :user_id + def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id) + temp_column = "#{column}_for_type_change" + + # Using a descriptive name that includes orinal column's name risks + # taking us above the 63 character limit, so we use a hash + identifier = "#{table}_#{column}_for_type_change" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + temp_undo_cleanup_column = "tmp_undo_cleanup_column_#{hashed_identifier}" + + unless column_exists?(table, batch_column_name) + raise "Column #{batch_column_name} does not exist on #{table}" + end + + if transaction_open? + raise 'undo_cleanup_concurrent_column_type_change can not be run inside a transaction' + end + + check_trigger_permissions!(table) + + begin + create_column_from( + table, + column, + temp_undo_cleanup_column, + type: old_type, + batch_column_name: batch_column_name, + type_cast_function: type_cast_function + ) + + transaction do + # This has to be performed in a transaction as otherwise we might + # have inconsistent data. + rename_column(table, column, temp_column) + rename_column(table, temp_undo_cleanup_column, column) + + install_rename_triggers(table, column, temp_column) + end + rescue + # create_column_from can not run inside a transaction, which means + # that there is a risk that if any of the operations that follow it + # fail, we'll be left with an inconsistent schema + # For those reasons, we make sure that we drop temp_undo_cleanup_column + # if an error is caught + if column_exists?(table, temp_undo_cleanup_column) + remove_column(table, temp_undo_cleanup_column) + end + + raise + end + end + # Cleans up a concurrent column name. # # This method takes care of removing previously installed triggers as well @@ -882,7 +951,7 @@ module Gitlab # column. opclasses[new] = opclasses.delete(old) if opclasses[old] - options[:opclasses] = opclasses + options[:opclass] = opclasses end add_concurrent_index(table, new_columns, options) @@ -994,10 +1063,10 @@ into similar problems in the future (e.g. when new tables are created). def postgres_exists_by_name?(table, name) index_sql = <<~SQL SELECT COUNT(*) - FROM pg_index - JOIN pg_class i ON (indexrelid=i.oid) - JOIN pg_class t ON (indrelid=t.oid) - WHERE i.relname = '#{name}' AND t.relname = '#{table}' + FROM pg_catalog.pg_indexes + WHERE schemaname = #{connection.quote(current_schema)} + AND tablename = #{connection.quote(table)} + AND indexname = #{connection.quote(name)} SQL connection.select_value(index_sql).to_i > 0 @@ -1053,11 +1122,15 @@ into similar problems in the future (e.g. when new tables are created). # the table name in addition to using the constraint_name check_sql = <<~SQL SELECT COUNT(*) - FROM pg_constraint - JOIN pg_class ON pg_constraint.conrelid = pg_class.oid - WHERE pg_constraint.contype = 'c' - AND pg_constraint.conname = '#{constraint_name}' - AND pg_class.relname = '#{table}' + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table)} SQL connection.select_value(check_sql) > 0 @@ -1147,6 +1220,64 @@ into similar problems in the future (e.g. when new tables are created). end end + # Copies all check constraints for the old column to the new column. + # + # table - The table containing the columns. + # old - The old column. + # new - The new column. + # schema - The schema the table is defined for + # If it is not provided, then the current_schema is used + def copy_check_constraints(table, old, new, schema: nil) + if transaction_open? + raise 'copy_check_constraints can not be run inside a transaction' + end + + unless column_exists?(table, old) + raise "Column #{old} does not exist on #{table}" + end + + unless column_exists?(table, new) + raise "Column #{new} does not exist on #{table}" + end + + table_with_schema = schema.present? ? "#{schema}.#{table}" : table + + check_constraints_for(table, old, schema: schema).each do |check_c| + validate = !(check_c["constraint_def"].end_with? "NOT VALID") + + # Normalize: + # - Old constraint definitions: + # '(char_length(entity_path) <= 5500)' + # - Definitionss from pg_get_constraintdef(oid): + # 'CHECK ((char_length(entity_path) <= 5500))' + # - Definitions from pg_get_constraintdef(oid, pretty_bool): + # 'CHECK (char_length(entity_path) <= 5500)' + # - Not valid constraints: 'CHECK (...) NOT VALID' + # to a single format that we can use: + # '(char_length(entity_path) <= 5500)' + check_definition = check_c["constraint_def"] + .sub(/^\s*(CHECK)?\s*\({0,2}/, '(') + .sub(/\){0,2}\s*(NOT VALID)?\s*$/, ')') + + constraint_name = begin + if check_definition == "(#{old} IS NOT NULL)" + not_null_constraint_name(table_with_schema, new) + elsif check_definition.start_with? "(char_length(#{old}) <=" + text_limit_name(table_with_schema, new) + else + check_constraint_name(table_with_schema, new, 'copy_check_constraint') + end + end + + add_check_constraint( + table_with_schema, + check_definition.gsub(old.to_s, new.to_s), + constraint_name, + validate: validate + ) + end + end + # Migration Helpers for adding limit to text columns def add_text_limit(table, column, limit, constraint_name: nil, validate: true) add_check_constraint( @@ -1274,6 +1405,37 @@ into similar problems in the future (e.g. when new tables are created). end end + # Returns an ActiveRecord::Result containing the check constraints + # defined for the given column. + # + # If the schema is not provided, then the current_schema is used + def check_constraints_for(table, column, schema: nil) + check_sql = <<~SQL + SELECT + ccu.table_schema as schema_name, + ccu.table_name as table_name, + ccu.column_name as column_name, + con.conname as constraint_name, + pg_get_constraintdef(con.oid) as constraint_def + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + INNER JOIN information_schema.constraint_column_usage ccu + ON con.conname = ccu.constraint_name + AND nsp.nspname = ccu.constraint_schema + AND rel.relname = ccu.table_name + WHERE nsp.nspname = #{connection.quote(schema.presence || current_schema)} + AND rel.relname = #{connection.quote(table)} + AND ccu.column_name = #{connection.quote(column)} + AND con.contype = 'c' + ORDER BY constraint_name + SQL + + connection.exec_query(check_sql) + end + def statement_timeout_disabled? # This is a string of the form "100ms" or "0" when disabled connection.select_value('SHOW statement_timeout') == "0" @@ -1284,8 +1446,9 @@ into similar problems in the future (e.g. when new tables are created). check_sql = <<~SQL SELECT c.is_nullable FROM information_schema.columns c - WHERE c.table_name = '#{table}' - AND c.column_name = '#{column}' + WHERE c.table_schema = #{connection.quote(current_schema)} + AND c.table_name = #{connection.quote(table)} + AND c.column_name = #{connection.quote(column)} SQL connection.select_value(check_sql) == 'YES' @@ -1352,6 +1515,7 @@ into similar problems in the future (e.g. when new tables are created). copy_indexes(table, old, new) copy_foreign_keys(table, old, new) + copy_check_constraints(table, old, new) end def validate_timestamp_column_name!(column_name) diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb index 4c1b13fe3b5..547e0b9b957 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -72,10 +72,10 @@ module Gitlab end def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ + Gitlab::Database::WithLockRetries.new( klass: self.class, logger: Gitlab::AppLogger - }).run(&block) + ).run(&block) end def connection diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index f9ad1e60776..17a42d997e6 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -11,8 +11,6 @@ module Gitlab PAUSE_SECONDS = 0.25 def perform(start_id, stop_id, source_table, partitioned_table, source_column) - return unless Feature.enabled?(:backfill_partitioned_audit_events, default_enabled: true) - if transaction_open? raise "Aborting job to backfill partitioned #{source_table} table! Do not run this job in a transaction block!" end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb new file mode 100644 index 00000000000..2a9f23f0098 --- /dev/null +++ b/lib/gitlab/database/postgres_index.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresIndex < ActiveRecord::Base + self.table_name = 'postgres_indexes' + self.primary_key = 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + # A 'regular' index is a non-unique index, + # that does not serve an exclusion constraint and + # is defined on a table that is not partitioned. + scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)} + + scope :random_few, ->(how_many) do + limit(how_many).order(Arel.sql('RANDOM()')) + end + + scope :not_match, ->(regex) { where("name !~ ?", regex)} + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb new file mode 100644 index 00000000000..074752fe75b --- /dev/null +++ b/lib/gitlab/database/reindexing.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + def self.perform(index_selector) + Coordinator.new(index_selector).perform + end + + def self.candidate_indexes + Gitlab::Database::PostgresIndex + .regular + .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") + .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") + end + end + end +end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb new file mode 100644 index 00000000000..fd3dca88567 --- /dev/null +++ b/lib/gitlab/database/reindexing/concurrent_reindex.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class ConcurrentReindex + include Gitlab::Utils::StrongMemoize + + ReindexError = Class.new(StandardError) + + PG_IDENTIFIER_LENGTH = 63 + TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' + REPLACED_INDEX_PREFIX = 'old_reindex_' + STATEMENT_TIMEOUT = 6.hours + + attr_reader :index, :logger + + def initialize(index, logger: Gitlab::AppLogger) + @index = index + @logger = logger + end + + def perform + raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique? + raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned? + raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX) + + logger.info "Starting reindex of #{index}" + + with_rebuilt_index do |replacement_index| + swap_index(replacement_index) + end + end + + private + + def with_rebuilt_index + if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) + logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}") + remove_index(index.schema, replacement_index_name) + end + + create_replacement_index_statement = index.definition + .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}") + + logger.info("creating replacement index #{replacement_index_name}") + logger.debug("replacement index definition: #{create_replacement_index_statement}") + + set_statement_timeout do + connection.execute(create_replacement_index_statement) + end + + replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) + + unless replacement_index.valid_index? + message = 'replacement index was created as INVALID' + logger.error("#{message}, cleaning up") + raise ReindexError, "failed to reindex #{index}: #{message}" + end + + yield replacement_index + ensure + begin + remove_index(index.schema, replacement_index_name) + rescue => e + logger.error(e) + end + end + + def swap_index(replacement_index) + logger.info("swapping replacement index #{replacement_index} with #{index}") + + with_lock_retries do + rename_index(index.schema, index.name, replaced_index_name) + rename_index(replacement_index.schema, replacement_index.name, index.name) + rename_index(index.schema, replaced_index_name, replacement_index.name) + end + end + + def rename_index(schema, old_index_name, new_index_name) + connection.execute(<<~SQL) + ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)} + RENAME TO #{quote_table_name(new_index_name)} + SQL + end + + def remove_index(schema, name) + logger.info("Removing index #{schema}.#{name}") + + set_statement_timeout do + connection.execute(<<~SQL) + DROP INDEX CONCURRENTLY + IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} + SQL + end + end + + def replacement_index_name + @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}" + end + + def replaced_index_name + @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}" + end + + def with_lock_retries(&block) + arguments = { klass: self.class, logger: logger } + + Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) + end + + def set_statement_timeout + execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) + yield + ensure + execute('RESET statement_timeout') + end + + delegate :execute, :quote_table_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb new file mode 100644 index 00000000000..0957f43e166 --- /dev/null +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class Coordinator + include ExclusiveLeaseGuard + + # Maximum lease time for the global Redis lease + # This should be higher than the maximum time for any + # long running step in the reindexing process (compare with + # statement timeouts). + TIMEOUT_PER_ACTION = 1.day + + attr_reader :indexes + + def initialize(indexes) + @indexes = indexes + end + + def perform + indexes.each do |index| + # This obtains a global lease such that there's + # only one live reindexing process at a time. + try_obtain_lease do + ReindexAction.keep_track_of(index) do + ConcurrentReindex.new(index).perform + end + end + end + end + + private + + def lease_timeout + TIMEOUT_PER_ACTION + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb new file mode 100644 index 00000000000..0928ef90e5d --- /dev/null +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class ReindexAction < ActiveRecord::Base + self.table_name = 'postgres_reindex_actions' + + enum state: { started: 0, finished: 1, failed: 2 } + + def self.keep_track_of(index, &block) + action = create!( + index_identifier: index.identifier, + action_start: Time.zone.now, + ondisk_size_bytes_start: index.ondisk_size_bytes + ) + + yield + + action.state = :finished + rescue + action.state = :failed + raise + ensure + index.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + action.action_end = Time.zone.now + action.ondisk_size_bytes_end = index.ondisk_size_bytes + + action.save! + end + end + end + end +end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index dda4d8eecdb..3d929c62933 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -32,11 +32,14 @@ module Gitlab def trigger_exists?(table_name, name) connection.select_value(<<~SQL) SELECT 1 - FROM pg_trigger - INNER JOIN pg_class - ON pg_trigger.tgrelid = pg_class.oid - WHERE pg_class.relname = '#{table_name}' - AND pg_trigger.tgname = '#{name}' + FROM pg_catalog.pg_trigger trgr + INNER JOIN pg_catalog.pg_class rel + ON trgr.tgrelid = rel.oid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = rel.relnamespace + WHERE nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table_name)} + AND trgr.tgname = #{connection.quote(name)} SQL end @@ -68,10 +71,10 @@ module Gitlab end def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ + Gitlab::Database::WithLockRetries.new( klass: self.class, logger: Gitlab::BackgroundMigration::Logger - }).run(&block) + ).run(&block) end def assert_not_in_transaction_block(scope:) diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index 2633c29438a..ff78fd0218c 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -6,6 +6,11 @@ module Gitlab EMPTY_STRING = Arel.sql("''").freeze EXPRESSION_ON_INVALID_INPUT = Arel::Nodes::NamedFunction.new('CAST', [Arel.sql('0').as('integer')]).freeze DEFAULT_MULTIPLIER = 1 + DISPLAY_NAME = self.name.underscore.freeze + + # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity. + # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE... + SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity. # @@ -74,6 +79,10 @@ module Gitlab end end + def self.order_by_similarity?(arel_query) + arel_query.to_sql.include?(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION) + end + # (SIMILARITY(COALESCE(column, ''), 'search_string') * CAST(multiplier AS numeric)) def self.rule_to_arel(search, rule) Arel::Nodes::Grouping.new( @@ -91,7 +100,7 @@ module Gitlab # SIMILARITY(COALESCE(column, ''), 'search_string') def self.similarity_function_call(search, column) - Arel::Nodes::NamedFunction.new('SIMILARITY', [column, Arel.sql(search)]) + Arel::Nodes::NamedFunction.new(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION, [column, Arel.sql(search)]) end # CAST(multiplier AS numeric) diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index a9c86e4e267..3fb52d786ad 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -95,7 +95,7 @@ module Gitlab run_block_with_transaction rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? - disable_idle_in_transaction_timeout + disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open? wait_until_next_retry reset_db_settings @@ -149,7 +149,7 @@ module Gitlab log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) log(message: "Executing the migration without lock timeout", current_iteration: current_iteration) - execute("SET LOCAL lock_timeout TO '0'") + disable_lock_timeout if ActiveRecord::Base.connection.transaction_open? run_block @@ -184,6 +184,10 @@ module Gitlab execute("SET LOCAL idle_in_transaction_session_timeout TO '0'") end + def disable_lock_timeout + execute("SET LOCAL lock_timeout TO '0'") + end + def reset_db_settings execute('RESET idle_in_transaction_session_timeout; RESET lock_timeout') end diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml new file mode 100644 index 00000000000..1d341e6520e --- /dev/null +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -0,0 +1,43 @@ +# This file exists to lock the attributes of Design Management models +# that get copied in `DesignManagement::CopyDesignCollection::CopyService` +# to specific schemas. +# +# This allows us to perform sanity checks and alert when there are changes +# to the schema by running expectations against the lists in this file +# and the actual schema of the models in `copy_designs_service_spec.rb`. +# +# If you are here because you received a failed test in +# `copy_designs_service_spec.rb`, you need to decide how to handle the +# changes and whether the new attribute(s) should be included in the copy +# or ignored. + +# COPY. +# Add attributes that should be copied to the `{model}_attributes` lists: +design_attributes: + - filename + - relative_position + +version_attributes: + - author_id + - created_at + +action_attributes: # (None) + +# IGNORE. +# Add attributes that should not be copied to the `ignore_{model}_attributes` lists: +ignore_design_attributes: + - id + - issue_id + - project_id + +ignore_version_attributes: + - id + - issue_id + - sha + +ignore_action_attributes: + - id + - design_id + - event + - image_v432x230 + - version_id diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index d54e1aad19a..16257bb5ff5 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -11,7 +11,7 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, - diff_options: diff_options, + diff_options: merged_diff_options(diff_options), diff_refs: merge_request_diff.diff_refs, fallback_diff_refs: merge_request_diff.fallback_diff_refs) end @@ -64,6 +64,13 @@ module Gitlab diff_stats_cache.read || super end end + + def merged_diff_options(diff_options) + project = @merge_request_diff.project + max_diff_options = ::Commit.max_diff_options(project: project).merge(project: project) + + diff_options.present? ? diff_options.merge(max_diff_options) : max_diff_options + end end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 0eb22e6b3cb..90cb9c8638a 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -20,10 +20,27 @@ module Gitlab # - Assigns DiffFile#highlighted_diff_lines for cached files # def decorate(diff_file) - if content = read_file(diff_file) - diff_file.highlighted_diff_lines = content.map do |line| - Gitlab::Diff::Line.safe_init_from_hash(line) - end + content = read_file(diff_file) + + return [] unless content + + # TODO: We could add some kind of flag to #initialize that would allow + # us to force re-caching + # https://gitlab.com/gitlab-org/gitlab/-/issues/263508 + # + if content.empty? && recache_due_to_size?(diff_file) + # If the file is missing from the cache and there's reason to believe + # it is uncached due to a size issue around changing the values for + # max patch size, manually populate the hash and then set the value. + # + new_cache_content = {} + new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + + write_to_redis_hash(new_cache_content) + + set_highlighted_diff_lines(diff_file, read_file(diff_file)) + else + set_highlighted_diff_lines(diff_file, content) end end @@ -58,6 +75,28 @@ module Gitlab private + def set_highlighted_diff_lines(diff_file, content) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.safe_init_from_hash(line) + end + end + + def recache_due_to_size?(diff_file) + diff_file_class = diff_file.diff.class + + current_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes + default_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes(diff_file_class::DEFAULT_MAX_PATCH_BYTES) + + # If the diff is >= than the default limit, but less than the current + # limit, it is likely uncached due to having hit the default limit, + # making it eligible for recalculating. + # + diff_file.diff.diff_bytesize.between?( + default_patch_safe_limit_bytes, + current_patch_safe_limit_bytes + ) + end + def cacheable_files strong_memoize(:cacheable_files) do diff_files.select { |file| cacheable?(file) && read_file(file).nil? } diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 10762d83588..da5b0afad38 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -35,7 +35,7 @@ module Gitlab lease.obtain(1 + retries) - yield(lease.retried?) + yield(lease.retried?, lease) ensure lease&.cancel end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index dca60c93fb2..1ce3ffe4c86 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -48,12 +48,12 @@ module Gitlab invite_members_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA' }, + invite_members_version_b: { + tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB' + }, new_create_project_ui: { tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi' }, - terms_opt_in: { - tracking_category: 'Growth::Acquisition::Experiment::TermsOptIn' - }, contact_sales_btn_in_app: { tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp' }, @@ -62,6 +62,15 @@ module Gitlab }, invite_email: { tracking_category: 'Growth::Acquisition::Experiment::InviteEmail' + }, + invitation_reminders: { + tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders' + }, + group_only_trials: { + tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials' + }, + default_to_issues_board: { + tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard' } }.freeze @@ -91,28 +100,40 @@ module Gitlab } end + def push_frontend_experiment(experiment_key) + var_name = experiment_key.to_s.camelize(:lower) + enabled = experiment_enabled?(experiment_key) + + gon.push({ experiments: { var_name => enabled } }, true) + end + def experiment_enabled?(experiment_key) return false if dnt_enabled? - return true if Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index) + return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index) return true if forced_enabled?(experiment_key) false end def track_experiment_event(experiment_key, action, value = nil) + return if dnt_enabled? + track_experiment_event_for(experiment_key, action, value) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), tracking_data) + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) end end def frontend_experimentation_tracking_data(experiment_key, action, value = nil) + return if dnt_enabled? + track_experiment_event_for(experiment_key, action, value) do |tracking_data| gon.push(tracking_data: tracking_data) end end def record_experiment_user(experiment_key) + return if dnt_enabled? return unless Experimentation.enabled?(experiment_key) && current_user ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) @@ -183,9 +204,14 @@ module Gitlab experiment.enabled? && experiment.enabled_for_environment? end - def enabled_for_user?(experiment_key, experimentation_subject_index) + def enabled_for_attribute?(experiment_key, attribute) + index = Digest::SHA1.hexdigest(attribute).hex % 100 + enabled_for_value?(experiment_key, index) + end + + def enabled_for_value?(experiment_key, experimentation_subject_index) enabled?(experiment_key) && - experiment(experiment_key).enabled_for_experimentation_subject?(experimentation_subject_index) + experiment(experiment_key).enabled_for_index?(experimentation_subject_index) end end @@ -200,10 +226,10 @@ module Gitlab environment end - def enabled_for_experimentation_subject?(experimentation_subject_index) - return false if experimentation_subject_index.blank? + def enabled_for_index?(index) + return false if index.blank? - experimentation_subject_index <= experiment_percentage + index <= experiment_percentage end private diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 09a49b6c1ca..78c47023c08 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -120,8 +120,8 @@ module Gitlab # default. # # Patches surpassing this limit should still be persisted in the database. - def patch_safe_limit_bytes - patch_hard_limit_bytes / 10 + def patch_safe_limit_bytes(limit = patch_hard_limit_bytes) + limit / 10 end # Returns the limit for a single diff file (patch). @@ -174,9 +174,13 @@ module Gitlab @line_count ||= Util.count_lines(@diff) end + def diff_bytesize + @diff_bytesize ||= @diff.bytesize + end + def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes + @too_large = diff_bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -194,7 +198,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes + @collapsed = !expanded && diff_bytesize >= self.class.patch_safe_limit_bytes end def collapse! diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index e6121d688ba..6090d1b9f69 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -7,19 +7,27 @@ module Gitlab class DiffCollection include Enumerable - DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze - attr_reader :limits delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits + def self.default_limits(project: nil) + if Feature.enabled?(:increased_diff_limits, project) + { max_files: 200, max_lines: 7500 } + else + { max_files: 100, max_lines: 5000 } + end + end + def self.limits(options = {}) limits = {} - limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) - limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + defaults = default_limits(project: options[:project]) + limits[:max_files] = options.fetch(:max_files, defaults[:max_files]) + limits[:max_lines] = options.fetch(:max_lines, defaults[:max_lines]) limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file - limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min - limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min + + limits[:safe_max_files] = [limits[:max_files], defaults[:max_files]].min + limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes @@ -110,11 +118,17 @@ module Gitlab files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes end + def expand_diff? + # Force single-entry diff collections to always present as expanded + # + @iterator.size == 1 || !@enforce_limits || @expanded + end + def each_gitaly_patch i = @array.length @iterator.each do |raw| - diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded) + diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?) if raw.overflow_marker @overflow = true @@ -137,11 +151,9 @@ module Gitlab break end - expanded = !@enforce_limits || @expanded - - diff = Gitlab::Git::Diff.new(raw, expanded: expanded) + diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?) - if !expanded && over_safe_limits?(i) && diff.line_count > 0 + if !expand_diff? && over_safe_limits?(i) && diff.line_count > 0 diff.collapse! end diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb index 7e49d79676e..e30ec836a49 100644 --- a/lib/gitlab/git/diff_stats_collection.rb +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -22,8 +22,8 @@ module Gitlab @collection.map(&:path) end - def real_size - max_files = ::Commit.max_diff_options[:max_files] + def real_size(project: nil) + max_files = ::Commit.max_diff_options(project: project)[:max_files] if paths.size > max_files "#{max_files}+" else diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 8ace4157ad7..1a3409c1f84 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -297,10 +297,17 @@ module Gitlab end file_name = "#{name}.#{extension}" - File.join(storage_path, self.gl_repository, sha, file_name) + File.join(storage_path, self.gl_repository, sha, archive_version_path, file_name) end private :archive_file_path + def archive_version_path + return '' unless Feature.enabled?(:include_lfs_blobs_in_archive) + + '@v2' + end + private :archive_version_path + # Return repo size in megabytes def size size = gitaly_repository_client.repository_size @@ -580,9 +587,9 @@ module Gitlab tags.find { |tag| tag.name == name } end - def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) + def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) + gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) end end @@ -610,7 +617,7 @@ module Gitlab } wrapped_gitaly_errors do - gitaly_operation_client.user_revert(args) + gitaly_operation_client.user_revert(**args) end end @@ -626,7 +633,7 @@ module Gitlab } wrapped_gitaly_errors do - gitaly_operation_client.user_cherry_pick(args) + gitaly_operation_client.user_cherry_pick(**args) end end @@ -640,7 +647,7 @@ module Gitlab } wrapped_gitaly_errors do - gitaly_operation_client.user_update_submodule(args) + gitaly_operation_client.user_update_submodule(**args) end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index da2d015ca4a..11919be594d 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -100,10 +100,6 @@ module Gitlab wrapped_gitaly_errors do gitaly_find_page(title: title, version: version, dir: dir) end - rescue Gitlab::Git::CommandError - # Return nil for invalid versions. - # This can be removed with https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2323 in place. - nil end def file(name, version) @@ -159,6 +155,8 @@ module Gitlab return unless wiki_page Gitlab::Git::WikiPage.new(wiki_page, version) + rescue GRPC::InvalidArgument + nil end def gitaly_find_file(name, version) @@ -173,9 +171,9 @@ module Gitlab gitaly_pages = if load_content - gitaly_wiki_client.load_all_pages(params) + gitaly_wiki_client.load_all_pages(**params) else - gitaly_wiki_client.list_all_pages(params) + gitaly_wiki_client.list_all_pages(**params) end gitaly_pages.map do |wiki_page, version| diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b67b3a37440..0576d1dd9db 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -47,6 +47,16 @@ module Gitlab :cmd, :changes attr_accessor :container + def self.error_message(key) + self.ancestors.each do |cls| + return cls.const_get('ERROR_MESSAGES', false).fetch(key) + rescue NameError, KeyError + next + end + + raise ArgumentError, "No error message defined for #{key}" + end + def initialize(actor, container, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @container = container @@ -137,6 +147,10 @@ module Gitlab private def check_container! + # Strict nil check, to avoid any surprises with Object#present? + # which can delegate to #empty? + raise NotFoundError, not_found_message if container.nil? + check_project! if project? end @@ -204,9 +218,7 @@ module Gitlab end def check_project_accessibility! - if project.blank? || !can_read_project? - raise NotFoundError, not_found_message - end + raise NotFoundError, not_found_message unless can_read_project? end def not_found_message @@ -279,10 +291,10 @@ module Gitlab error_message(:download) end - # We assume that all git-access classes are in project context by default. - # Override this method to be more specific. def project? - true + # Strict nil check, to avoid any surprises with Object#present? + # which can delegate to #empty? + !project.nil? end def project @@ -290,7 +302,7 @@ module Gitlab end def check_push_access! - if container.repository_read_only? + if project&.repository_read_only? raise ForbiddenError, error_message(:read_only) end @@ -411,13 +423,7 @@ module Gitlab protected def error_message(key) - self.class.ancestors.each do |cls| - return cls.const_get('ERROR_MESSAGES', false).fetch(key) - rescue NameError, KeyError - next - end - - raise ArgumentError, "No error message defined for #{key}" + self.class.error_message(key) end def success_result @@ -504,7 +510,7 @@ module Gitlab changes_size = 0 changes_list.each do |change| - changes_size += repository.new_blobs(change[:newrev]).sum(&:size) # rubocop: disable CodeReuse/ActiveRecord + changes_size += repository.new_blobs(change[:newrev]).sum(&:size) check_size_against_limit(changes_size) end diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb index e03459ea7a1..224c6b9eb52 100644 --- a/lib/gitlab/git_access_result/custom_action.rb +++ b/lib/gitlab/git_access_result/custom_action.rb @@ -12,7 +12,7 @@ module Gitlab # 'data' => { # 'api_endpoints' => %w{geo/proxy_git_ssh/info_refs_receive_pack geo/proxy_git_ssh/receive_pack}, # 'gl_username' => user.username, - # 'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki) + # 'primary_repo' => geo_primary_http_url_to_repo(container) # } # } # diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index ae83e45f2b3..710e2ce90ec 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -21,6 +21,11 @@ module Gitlab @authentication_abilities &= [:download_code, :push_code] end + override :project + def project + container.project if container.is_a?(ProjectSnippet) + end + override :check def check(cmd, changes) check_snippet_accessibility! @@ -46,29 +51,19 @@ module Gitlab # snippets never return custom actions, such as geo replication. end - override :project? - def project? - project_snippet? - end - - override :project - def project - snippet&.project - end - override :check_valid_actor! def check_valid_actor! # TODO: Investigate if expanding actor/authentication types are needed. # https://gitlab.com/gitlab-org/gitlab/issues/202190 - if actor && !actor.is_a?(User) && !actor.instance_of?(Key) + if actor && !allowed_actor? raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] end super end - def project_snippet? - snippet.is_a?(ProjectSnippet) + def allowed_actor? + actor.is_a?(User) || actor.instance_of?(Key) end override :check_push_access! @@ -120,7 +115,7 @@ module Gitlab override :check_single_change_access def check_single_change_access(change, _skip_lfs_integrity_check: false) Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, logger: logger).validate! - Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate! + Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit, logger: logger).validate! rescue Checks::TimedLogger::TimeoutError raise TimeoutError, logger.full_message end @@ -138,3 +133,5 @@ module Gitlab end end end + +Gitlab::GitAccessSnippet.prepend_if_ee('EE::Gitlab::GitAccessSnippet') diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index a941282e713..3011b794b8f 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -12,6 +12,11 @@ module Gitlab write_to_wiki: "You are not allowed to write to this project's wiki." }.freeze + override :project + def project + container.project if container.is_a?(ProjectWiki) + end + override :download_ability def download_ability :download_wiki_code @@ -40,11 +45,6 @@ module Gitlab def not_found_message error_message(:not_found) end - - override :repository - def repository - container.wiki.repository - end end end diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb index 98d327a7329..e98ae75590d 100644 --- a/lib/gitlab/gitaly_client/diff_stitcher.rb +++ b/lib/gitlab/gitaly_client/diff_stitcher.rb @@ -5,8 +5,10 @@ module Gitlab class DiffStitcher include Enumerable - def initialize(rpc_response) - @rpc_response = rpc_response + delegate :size, to: :rpc_response + + def initialize(rpc_response_param) + @rpc_response = rpc_response_param end def each @@ -31,6 +33,10 @@ module Gitlab end end end + + private + + attr_reader :rpc_response end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 786eb3ca4ae..4850d646de4 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -102,7 +102,7 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref) + def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, @@ -110,7 +110,8 @@ module Gitlab target_ref: encode_binary(target_ref), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, message: encode_binary(message), - first_parent_ref: encode_binary(first_parent_ref) + first_parent_ref: encode_binary(first_parent_ref), + allow_conflicts: allow_conflicts ) response = GitalyClient.call(@repository.storage, :operation_service, diff --git a/lib/gitlab/gitpod.rb b/lib/gitlab/gitpod.rb index 11b54db72ea..e35fb8fed02 100644 --- a/lib/gitlab/gitpod.rb +++ b/lib/gitlab/gitpod.rb @@ -3,17 +3,13 @@ module Gitlab class Gitpod class << self - def feature_conditional? - feature.conditional? - end - def feature_available? # The gitpod_bundle feature could be conditionally applied, so check if `!off?` - !feature.off? + !feature.off? || feature_enabled? end def feature_enabled?(actor = nil) - feature.enabled?(actor) + Feature.enabled?(:gitpod, actor, default_enabled: true) end def feature_and_settings_enabled?(actor = nil) diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 54dca93a891..352a93817be 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -4,6 +4,8 @@ module Gitlab class GlRepository include Singleton + # TODO: Refactor these constants into proper classes + # https://gitlab.com/gitlab-org/gitlab/-/issues/259008 PROJECT = RepoType.new( name: :project, access_checker_class: Gitlab::GitAccessProject, @@ -12,8 +14,12 @@ module Gitlab WIKI = RepoType.new( name: :wiki, access_checker_class: Gitlab::GitAccessWiki, - repository_resolver: -> (container) { ::Repository.new(container.wiki.full_path, container, shard: container.wiki.repository_storage, disk_path: container.wiki.disk_path, repo_type: WIKI) }, - project_resolver: -> (container) { container.is_a?(Project) ? container : nil }, + repository_resolver: -> (container) do + wiki = container.is_a?(Wiki) ? container : container.wiki # Also allow passing a Project, Group, or Geo::DeletedProject + ::Repository.new(wiki.full_path, wiki, shard: wiki.repository_storage, disk_path: wiki.disk_path, repo_type: WIKI) + end, + container_class: ProjectWiki, + project_resolver: -> (wiki) { wiki.try(:project) }, suffix: :wiki ).freeze SNIPPET = RepoType.new( diff --git a/lib/gitlab/gl_repository/identifier.rb b/lib/gitlab/gl_repository/identifier.rb index 57350b1edb0..f521a14ea19 100644 --- a/lib/gitlab/gl_repository/identifier.rb +++ b/lib/gitlab/gl_repository/identifier.rb @@ -53,12 +53,13 @@ module Gitlab private def container_class - case @container_type - when 'project' - Project - when 'group' - Group - end + # NOTE: This is currently only used and supported for group wikis + # https://gitlab.com/gitlab-org/gitlab/-/issues/219192 + return unless @repo_type_name == 'wiki' + + "#{@container_type}_#{@repo_type_name}".classify.constantize + rescue NameError + nil end end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 346f6be0d98..4b1f4fcc2a2 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -29,10 +29,6 @@ module Gitlab end def identifier_for_container(container) - if container.is_a?(Group) - return "#{container.class.name.underscore}-#{container.id}-#{name}" - end - "#{name}-#{container.id}" end @@ -84,3 +80,5 @@ module Gitlab end end end + +Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 66517ecd743..10660649623 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -43,12 +43,10 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:snippets_vue, default_enabled: true) - push_frontend_feature_flag(:monaco_blobs, default_enabled: true) - push_frontend_feature_flag(:monaco_ci, default_enabled: false) - push_frontend_feature_flag(:snippets_edit_vue, default_enabled: true) push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) + push_frontend_feature_flag(:usage_data_api, default_enabled: true) + push_frontend_feature_flag(:security_auto_fix, default_enabled: false) # Startup CSS feature is a special one as it can be enabled by means of cookies and params gon.push({ features: { 'startupCss' => use_startup_css? } }, true) @@ -59,9 +57,9 @@ module Gitlab # name - The name of the feature flag, e.g. `my_feature`. # args - Any additional arguments to pass to `Feature.enabled?`. This allows # you to check if a flag is enabled for a particular user. - def push_frontend_feature_flag(name, *args) + def push_frontend_feature_flag(name, *args, **kwargs) var_name = name.to_s.camelize(:lower) - enabled = Feature.enabled?(name, *args) + enabled = Feature.enabled?(name, *args, **kwargs) # Here the `true` argument signals gon that the value should be merged # into any existing ones, instead of overwriting them. This allows you to diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 27673e5c27a..c70127553fd 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -29,8 +29,8 @@ module Gitlab raise NotImplementedError, "Implement #find_object in #{self.class.name}" end - def authorized_find!(*args) - object = Graphql::Lazy.force(find_object(*args)) + def authorized_find!(*args, **kwargs) + object = Graphql::Lazy.force(find_object(*args, **kwargs)) authorize!(object) diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb new file mode 100644 index 00000000000..a96e4c4b976 --- /dev/null +++ b/lib/gitlab/graphql/global_id_compatibility.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module GlobalIDCompatibility + # TODO: remove this module once the compatibility layer is no longer needed. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883 + def coerce_global_id_arguments!(args) + global_id_arguments = self.class.arguments.values.select do |arg| + arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType + end + + global_id_arguments.each do |arg| + k = arg.keyword + args[k] &&= arg.type.coerce_isolated_input(args[k]) + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 7be6810f7ba..0b5bde8d8d9 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -12,13 +12,19 @@ module Gitlab end method_name = kwargs.delete(:method) || name.to_s.sub(/_html$/, '') - kwargs[:resolve] = Gitlab::Graphql::MarkdownField::Resolver.new(method_name.to_sym).proc + resolver_method = "#{name}_resolver".to_sym + kwargs[:resolver_method] = resolver_method kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`" # Adding complexity to rendered notes since that could cause queries. kwargs[:complexity] ||= 5 field name, GraphQL::STRING_TYPE, **kwargs + + define_method resolver_method do + # We need to `dup` the context so the MarkdownHelper doesn't modify it + ::MarkupHelper.markdown_field(object, method_name.to_sym, context.to_h.dup) + end end end end diff --git a/lib/gitlab/graphql/markdown_field/resolver.rb b/lib/gitlab/graphql/markdown_field/resolver.rb deleted file mode 100644 index 11a01b95ad1..00000000000 --- a/lib/gitlab/graphql/markdown_field/resolver.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module MarkdownField - class Resolver - attr_reader :method_name - - def initialize(method_name) - @method_name = method_name - end - - def proc - -> (object, _args, ctx) do - # We need to `dup` the context so the MarkdownHelper doesn't modify it - ::MarkupHelper.markdown_field(object, method_name, ctx.to_h.dup) - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 17cd22d38ad..252f6371765 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -110,8 +110,7 @@ module Gitlab end if last - # grab one more than we need - paginated_nodes = sliced_nodes.last(limit_value + 1) + paginated_nodes = LastItems.take_items(sliced_nodes, limit_value + 1) # there is an extra node, so there is a previous page @has_previous_page = paginated_nodes.count > limit_value diff --git a/lib/gitlab/graphql/pagination/keyset/last_items.rb b/lib/gitlab/graphql/pagination/keyset/last_items.rb new file mode 100644 index 00000000000..45bf15236c1 --- /dev/null +++ b/lib/gitlab/graphql/pagination/keyset/last_items.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Pagination + module Keyset + # This class handles the last(N) ActiveRecord call even if a special ORDER BY configuration is present. + # For the last(N) call, ActiveRecord calls reverse_order, however for some cases it raises + # ActiveRecord::IrreversibleOrderError error. + class LastItems + # rubocop: disable CodeReuse/ActiveRecord + def self.take_items(scope, count) + if custom_order = lookup_custom_reverse_order(scope.order_values) + items = scope.reorder(*custom_order).first(count) # returns a single record when count is nil + items.is_a?(Array) ? items.reverse : items + else + scope.last(count) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # Detect special ordering and provide the reversed order + def self.lookup_custom_reverse_order(order_values) + if ordering_by_merged_at_and_mr_id_desc?(order_values) + [ + Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'ASC'), # reversing the order + MergeRequest.arel_table[:id].asc + ] + elsif ordering_by_merged_at_and_mr_id_asc?(order_values) + [ + Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'DESC'), + MergeRequest.arel_table[:id].asc + ] + end + end + + def self.ordering_by_merged_at_and_mr_id_desc?(order_values) + order_values.size == 2 && + order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'DESC') && + order_values.last.is_a?(Arel::Nodes::Descending) && + order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql + end + + def self.ordering_by_merged_at_and_mr_id_asc?(order_values) + order_values.size == 2 && + order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'ASC') && + order_values.last.is_a?(Arel::Nodes::Descending) && + order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql + end + + private_class_method :ordering_by_merged_at_and_mr_id_desc? + private_class_method :ordering_by_merged_at_and_mr_id_asc? + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index f54695ddb9a..f3ce3a10703 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -94,6 +94,10 @@ module Gitlab [order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr] elsif ordering_by_similarity?(order_value) ['similarity', order_value.direction, order_value.expr] + elsif ordering_by_case?(order_value) + ['case_order_value', order_value.direction, order_value.expr] + elsif ordering_by_array_position?(order_value) + ['array_position', order_value.direction, order_value.expr] else [order_value.expr.name, order_value.direction, nil] end @@ -104,9 +108,19 @@ module Gitlab order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'lower' end + # determine if ordering using ARRAY_POSITION, eg. "ORDER BY ARRAY_POSITION(Array[4,3,1,2]::smallint, state)" + def ordering_by_array_position?(order_value) + order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'array_position' + end + # determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore def ordering_by_similarity?(order_value) - order_value.to_sql.match?(/SIMILARITY\(.+\*/) + Gitlab::Database::SimilarityScore.order_by_similarity?(order_value) + end + + # determine if ordering using CASE + def ordering_by_case?(order_value) + order_value.expr.is_a?(Arel::Nodes::Case) end end end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 6b6bb72eb31..1285365376f 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -6,6 +6,8 @@ module Gitlab class LoggerAnalyzer COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value } DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value } + FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] } + ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze def analyze?(query) Feature.enabled?(:graphql_logging, default_enabled: true) @@ -23,18 +25,21 @@ module Gitlab end def call(memo, visit_type, irep_node) - memo + RequestStore.store[:graphql_logs] = memo end def final_value(memo) return if memo.nil? - analyzers = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER] - complexity, depth = GraphQL::Analysis.analyze_query(memo[:query], analyzers) + complexity, depth, field_usages = GraphQL::Analysis.analyze_query(memo[:query], ALL_ANALYZERS) memo[:depth] = depth memo[:complexity] = complexity + # This duration is not the execution time of the + # query but the execution time of the analyzer. memo[:duration_s] = duration(memo[:time_started]).round(1) + memo[:used_fields] = field_usages.first + memo[:used_deprecated_fields] = field_usages.second GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 0cc3de297ba..5fec50eecd2 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -4,10 +4,10 @@ module Gitlab class GroupSearchResults < SearchResults attr_reader :group - def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, filters: {}) + def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, sort: nil, filters: {}) @group = group - super(current_user, query, limit_projects, default_project_filter: default_project_filter, filters: filters) + super(current_user, query, limit_projects, default_project_filter: default_project_filter, sort: sort, filters: filters) end # rubocop:disable CodeReuse/ActiveRecord @@ -38,3 +38,5 @@ module Gitlab end end end + +Gitlab::GroupSearchResults.prepend_if_ee('EE::Gitlab::GroupSearchResults') diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb index cdc6d2a7519..f0c6fdab600 100644 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ b/lib/gitlab/health_checks/unicorn_check.rb @@ -22,7 +22,7 @@ module Gitlab def check return unless http_servers - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 018cb36fc58..379a734b19c 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -20,8 +20,8 @@ module Gitlab /\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads ).freeze - def self.clean(*args) - new(*args).clean + def self.clean(*args, **kwargs) + new(*args, **kwargs).clean end def initialize(relation_hash:, relation_class:, excluded_keys: []) diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05b69362976..d60bc79df4c 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -31,8 +31,8 @@ module Gitlab TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze - def self.create(*args) - new(*args).create + def self.create(*args, **kwargs) + new(*args, **kwargs).create end def self.relation_class(relation_name) @@ -53,6 +53,7 @@ module Gitlab @importable = importable @imported_object_retries = 0 @relation_hash[importable_column_name] = @importable.id + @original_user = {} # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' @@ -112,6 +113,7 @@ module Gitlab def update_user_references self.class::USER_REFERENCES.each do |reference| if @relation_hash[reference] + @original_user[reference] = @relation_hash[reference] @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end end @@ -243,28 +245,20 @@ module Gitlab # will be used. Otherwise, a note stating the original author name # is left. def set_note_author - old_author_id = @relation_hash['author_id'] + old_author_id = @original_user['author_id'] author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) + unless @members_mapper.include?(old_author_id) + @relation_hash['note'] = "%{note}\n\n %{missing_author_note}" % { + note: @relation_hash['note'].presence || '*Blank note*', + missing_author_note: missing_author_note(@relation_hash['updated_at'], author['name']) + } + end end def missing_author_note(updated_at, author_name) timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? + "*By #{author_name} on #{timestamp} (imported from GitLab)*" end def existing_object? diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 081745a49f4..5a6f6e017d2 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -10,8 +10,8 @@ module Gitlab MAX_RETRIES = 8 IGNORED_FILENAMES = %w(. ..).freeze - def self.import(*args) - new(*args).import + def self.import(*args, **kwargs) + new(*args, **kwargs).import end def initialize(importable:, archive_file:, shared:) diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index e9b05afc7d4..0d9839b86cf 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -35,6 +35,7 @@ module Gitlab # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") + next unless File.exist?(path) File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num| @@ -43,6 +44,11 @@ module Gitlab end end + # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) + def clear_consumed_relations + @consumed_relations.clear + end + private def json_decode(string) diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 515fd98630c..4964b8b16f4 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -21,10 +21,10 @@ module Gitlab save_lfs_object(lfs_object) end - append_lfs_json_for_batch(batch) if write_lfs_json_enabled? + append_lfs_json_for_batch(batch) end - write_lfs_json if write_lfs_json_enabled? + write_lfs_json true rescue => e @@ -35,10 +35,6 @@ module Gitlab private - def write_lfs_json_enabled? - ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) - end - def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 31d1f7b48bd..6b37683ea68 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -34,8 +34,8 @@ module Gitlab @user.id end - def include?(old_author_id) - map.has_key?(old_author_id) && map[old_author_id] != default_user_id + def include?(old_user_id) + map.has_key?(old_user_id) end private @@ -63,6 +63,8 @@ module Gitlab end def add_team_member(member, existing_user = nil) + return true if existing_user && @importable.members.exists?(user_id: existing_user.id) + member['user'] = existing_user member_hash = member_hash(member) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 9ec5df8cde9..a0526ba0414 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -403,9 +403,15 @@ ee: - issues: - epic_issue: - :epic + - :issuable_sla - protected_branches: - :unprotect_access_levels - protected_environments: - :deploy_access_levels - :service_desk_setting - :security_setting + + included_attributes: + issuable_sla: + - :issue + - :due_at diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb new file mode 100644 index 00000000000..2d989d21166 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class DateCalculator + include Gitlab::Utils::StrongMemoize + + def initialize(dates) + @dates = dates.dup + @dates.flatten! + @dates.compact! + @dates.sort! + @dates.map! { |date| date.to_time.to_f } + end + + def closest_date_to_average + strong_memoize(:closest_date_to_average) do + next if @dates.empty? + + average_date = (@dates.first + @dates.last) / 2.0 + closest_date = @dates.min_by { |date| (date - average_date).abs } + Time.zone.at(closest_date) + end + end + + def calculate_by_closest_date_to_average(date) + return date unless closest_date_to_average && closest_date_to_average < Time.current + + date + (Time.current - closest_date_to_average).seconds + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb new file mode 100644 index 00000000000..b0c3940b5f9 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class SampleDataRelationTreeRestorer < RelationTreeRestorer + DATE_MODELS = %i[issues milestones].freeze + + def initialize(*args) + super + + date_calculator + end + + private + + def build_relation(relation_key, relation_definition, data_hash) + # Override due date attributes in data hash for Sample Data templates + # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import + # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333) + override_date_attributes!(relation_key, data_hash) + super + end + + def override_date_attributes!(relation_key, data_hash) + return unless DATE_MODELS.include?(relation_key.to_sym) + + data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil? + data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil? + end + + # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) + def dates + unless relation_reader.legacy? + DATE_MODELS.map do |tag| + relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do + relation_reader.clear_consumed_relations + end + end + end + end + + def date_calculator + @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index a16ffe36054..b1d647281ab 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -70,7 +70,7 @@ module Gitlab end def relation_tree_restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( + @relation_tree_restorer ||= relation_tree_restorer_class.new( user: @user, shared: @shared, relation_reader: relation_reader, @@ -84,6 +84,14 @@ module Gitlab ) end + def relation_tree_restorer_class + sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer + end + + def sample_data_template? + @project&.import_data&.data&.dig('sample_data') + end + def members_mapper @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, user: @user, diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 7cca3596da6..80dacf2eb20 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -36,7 +36,7 @@ module Gitlab end def exportable - @project.present(exportable_params) + @project.present(**exportable_params) end def exportable_params diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 9e10e7aea13..f808e30bd6e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -14,10 +14,10 @@ module Gitlab def restore return true unless File.exist?(path_to_bundle) + ensure_repository_does_not_exist! + repository.create_from_bundle(path_to_bundle) rescue => e - Repositories::DestroyService.new(repository).execute - shared.error(e) false end @@ -25,6 +25,16 @@ module Gitlab private attr_accessor :repository, :path_to_bundle, :shared + + def ensure_repository_does_not_exist! + if repository.exists? + shared.logger.info( + message: %Q{Deleting existing "#{repository.path}" to re-import it.} + ) + + Repositories::DestroyService.new(repository).execute + end + end end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index e4724659eff..045ba2495bf 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -5,8 +5,8 @@ module Gitlab class Saver include Gitlab::ImportExport::CommandLineUtil - def self.save(*args) - new(*args).save + def self.save(*args, **kwargs) + new(*args, **kwargs).save end def initialize(exportable:, shared:) diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index dca8e3a7449..26e7d2cf765 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -40,7 +40,7 @@ module Gitlab def add_upload(upload) uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys - UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h + UploadService.new(@project, File.open(upload, 'r'), FileUploader, **uploader_context).execute.to_h end def copy_project_uploads diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 4154d4fe775..48f5b558e52 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -3,8 +3,8 @@ module Gitlab module ImportExport class VersionChecker - def self.check!(*args) - new(*args).check! + def self.check!(*args, **kwargs) + new(*args, **kwargs).check! end def initialize(shared:) diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 4a85a313fd7..d1ac6a55fb7 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -37,7 +37,7 @@ module Gitlab %i[get_request_count query_time read_bytes write_bytes].each do |method| define_method method do - STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord + STORAGES.sum(&method) end end end diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 7be54a214dd..945ab7f40c2 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -16,9 +16,12 @@ module Gitlab end # finder - The finder class to use for retrieving the issuables. - def initialize(finder, project = nil) + # fast_fail - restrict counting to a shorter period, degrading gracefully on + # failure + def initialize(finder, project = nil, fast_fail: false) @finder = finder @project = project + @fast_fail = fast_fail @cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache end @@ -26,6 +29,10 @@ module Gitlab self[state || :opened] end + def fast_fail? + !!@fast_fail + end + # Define method for each state STATES.each do |state| define_method(state) { self[state] } @@ -53,7 +60,53 @@ module Gitlab end def initialize_cache - Hash.new { |hash, finder| hash[finder] = finder.count_by_state } + Hash.new { |hash, finder| hash[finder] = perform_count(finder) } + end + + def perform_count(finder) + return finder.count_by_state unless fast_fail? + + fast_count_by_state_attempt! + + # Determining counts when referring to issuable titles or descriptions can + # be very expensive, and involve the database reading gigabytes of data + # for a relatively minor piece of functionality. This may slow index pages + # by seconds in the best case, or lead to a statement timeout in the worst + # case. + # + # In time, we may be able to use elasticsearch or postgresql tsv columns + # to perform the calculation more efficiently. Until then, use a shorter + # timeout and return -1 as a sentinel value if it is triggered + begin + ApplicationRecord.with_fast_statement_timeout do + finder.count_by_state + end + rescue ActiveRecord::QueryCanceled => err + fast_count_by_state_failure! + + Gitlab::ErrorTracking.track_exception( + err, + params: finder.params, + current_user_id: finder.current_user&.id, + issue_url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/249180' + ) + + Hash.new(-1) + end + end + + def fast_count_by_state_attempt! + Gitlab::Metrics.counter( + :gitlab_issuable_fast_count_by_state_total, + "Count of total calls to IssuableFinder#count_by_state with fast failure" + ).increment + end + + def fast_count_by_state_failure! + Gitlab::Metrics.counter( + :gitlab_issuable_fast_count_by_state_failures_total, + "Count of failed calls to IssuableFinder#count_by_state with fast failure" + ).increment end end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index e7a8cc6305a..2cede524cac 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -23,7 +23,15 @@ module Gitlab TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total def self.notify(key, jid) - Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + Gitlab::Redis::SharedState.with do |redis| + # Use a Redis MULTI transaction to ensure we always set an expiry + redis.multi do |multi| + multi.lpush(key, jid) + # This TTL needs to be long enough to allow whichever Sidekiq job calls + # JobWaiter#wait to reach BLPOP. + multi.expire(key, 6.hours.to_i) + end + end end def self.key?(key) @@ -52,10 +60,6 @@ module Gitlab increment_counter(STARTED_METRIC) Gitlab::Redis::SharedState.with do |redis| - # Fallback key expiry: allow a long grace period to reduce the chance of - # a job pushing to an expired key and recreating it - redis.expire(key, [timeout * 2, 10.minutes.to_i].max) - while jobs_remaining > 0 # Redis will not take fractional seconds. Prefer waiting too long over # not waiting long enough @@ -75,9 +79,6 @@ module Gitlab @finished << jid @jobs_remaining -= 1 end - - # All jobs have finished, so expire the key immediately - redis.expire(key, 0) if jobs_remaining == 0 end finished diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index fa68afd39f5..13cd6dcad3f 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -167,6 +167,21 @@ module Gitlab end end + # Ingresses resource is currently on the apis/extensions api group + # until Kubernetes 1.21. Kubernetest 1.22+ has ingresses resources in + # the networking.k8s.io/v1 api group. + # + # As we still support Kubernetes 1.12+, we will need to support both. + def get_ingresses(**args) + extensions_client.discover unless extensions_client.discovered + + if extensions_client.respond_to?(:get_ingresses) + extensions_client.get_ingresses(**args) + else + networking_client.get_ingresses(**args) + end + end + def create_or_update_cluster_role_binding(resource) update_cluster_role_binding(resource) end diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb index d247662dc3b..a5651f2f184 100644 --- a/lib/gitlab/kubernetes/pod.rb +++ b/lib/gitlab/kubernetes/pod.rb @@ -2,13 +2,47 @@ module Gitlab module Kubernetes - module Pod + class Pod PENDING = 'Pending' RUNNING = 'Running' SUCCEEDED = 'Succeeded' FAILED = 'Failed' UNKNOWN = 'Unknown' PHASES = [PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN].freeze + + STABLE_TRACK_VALUE = 'stable' + + def initialize(attributes = {}) + @attributes = attributes + end + + def track + attributes.dig('metadata', 'labels', 'track') || STABLE_TRACK_VALUE + end + + def name + metadata['name'] || metadata['generateName'] + end + + def stable? + track == STABLE_TRACK_VALUE + end + + def status + attributes.dig('status', 'phase') + end + + def order + stable? ? 1 : 0 + end + + private + + attr_reader :attributes + + def metadata + attributes.fetch('metadata', {}) + end end end end diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index e4d600694c2..825d7399190 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -6,6 +6,14 @@ module Gitlab # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md class Client + GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs+json' + GIT_LFS_USER_AGENT = "GitLab #{Gitlab::VERSION} LFS client" + DEFAULT_HEADERS = { + 'Accept' => GIT_LFS_CONTENT_TYPE, + 'Content-Type' => GIT_LFS_CONTENT_TYPE, + 'User-Agent' => GIT_LFS_USER_AGENT + }.freeze + attr_reader :base_url def initialize(base_url, credentials:) @@ -13,19 +21,19 @@ module Gitlab @credentials = credentials end - def batch(operation, objects) + def batch!(operation, objects) body = { operation: operation, transfers: ['basic'], # We don't know `ref`, so can't send it - objects: objects.map { |object| { oid: object.oid, size: object.size } } + objects: objects.as_json(only: [:oid, :size]) } rsp = Gitlab::HTTP.post( batch_url, basic_auth: basic_auth, body: body.to_json, - headers: { 'Content-Type' => 'application/vnd.git-lfs+json' } + headers: build_request_headers ) raise BatchSubmitError unless rsp.success? @@ -40,17 +48,19 @@ module Gitlab body end - def upload(object, upload_action, authenticated:) + def upload!(object, upload_action, authenticated:) file = object.file.open params = { body_stream: file, headers: { 'Content-Length' => object.size.to_s, - 'Content-Type' => 'application/octet-stream' + 'Content-Type' => 'application/octet-stream', + 'User-Agent' => GIT_LFS_USER_AGENT }.merge(upload_action['header'] || {}) } + authenticated = true if params[:headers].key?('Authorization') params[:basic_auth] = basic_auth unless authenticated rsp = Gitlab::HTTP.put(upload_action['href'], params) @@ -60,8 +70,26 @@ module Gitlab file&.close end + def verify!(object, verify_action, authenticated:) + params = { + body: object.to_json(only: [:oid, :size]), + headers: build_request_headers(verify_action['header']) + } + + authenticated = true if params[:headers].key?('Authorization') + params[:basic_auth] = basic_auth unless authenticated + + rsp = Gitlab::HTTP.post(verify_action['href'], params) + + raise ObjectVerifyError unless rsp.success? + end + private + def build_request_headers(extra_headers = nil) + DEFAULT_HEADERS.merge(extra_headers || {}) + end + attr_reader :credentials def batch_url @@ -96,6 +124,12 @@ module Gitlab "Failed to upload object" end end + + class ObjectVerifyError < StandardError + def message + "Failed to verify object" + end + end end end end diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb new file mode 100644 index 00000000000..80dff075391 --- /dev/null +++ b/lib/gitlab/manifest_import/metadata.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module ManifestImport + class Metadata + EXPIRY_TIME = 1.week + + attr_reader :user, :fallback + + def initialize(user, fallback: {}) + @user = user + @fallback = fallback + end + + def save(repositories, group_id) + Gitlab::Redis::SharedState.with do |redis| + redis.multi do + redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + end + end + end + + def repositories + redis_get('repositories').then do |repositories| + next unless repositories + + Gitlab::Json.parse(repositories).map(&:symbolize_keys) + end || fallback[:manifest_import_repositories] + end + + def group_id + redis_get('group_id')&.to_i || fallback[:manifest_import_group_id] + end + + private + + def key_for(field) + "manifest_import:metadata:user:#{user.id}:#{field}" + end + + def redis_get(field) + Gitlab::Redis::SharedState.with do |redis| + redis.get(key_for(field)) + end + end + end + end +end diff --git a/lib/gitlab/marginalia.rb b/lib/gitlab/marginalia.rb index 24e21a1d512..325a8c5c325 100644 --- a/lib/gitlab/marginalia.rb +++ b/lib/gitlab/marginalia.rb @@ -4,8 +4,6 @@ module Gitlab module Marginalia cattr_accessor :enabled, default: false - MARGINALIA_FEATURE_FLAG = :marginalia - def self.set_application_name ::Marginalia.application_name = Gitlab.process_name end @@ -16,15 +14,11 @@ module Gitlab end end - def self.cached_feature_enabled? - enabled - end - - def self.set_feature_cache + def self.set_enabled_from_feature_flag # During db:create and db:bootstrap skip feature query as DB is not available yet. return false unless Gitlab::Database.cached_table_exists?('features') - self.enabled = Feature.enabled?(MARGINALIA_FEATURE_FLAG) + self.enabled = Feature.enabled?(:marginalia, type: :ops) end end end diff --git a/lib/gitlab/marginalia/active_record_instrumentation.rb b/lib/gitlab/marginalia/active_record_instrumentation.rb index 3266b9f8336..452f472bf6a 100644 --- a/lib/gitlab/marginalia/active_record_instrumentation.rb +++ b/lib/gitlab/marginalia/active_record_instrumentation.rb @@ -5,7 +5,7 @@ module Gitlab module Marginalia module ActiveRecordInstrumentation def annotate_sql(sql) - Gitlab::Marginalia.cached_feature_enabled? ? super(sql) : sql + Gitlab::Marginalia.enabled ? super(sql) : sql end end end diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index da3b597a74e..36e9a6ccef6 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -3,7 +3,7 @@ module Gitlab module MarkdownCache # Increment this number every time the renderer changes its output - CACHE_COMMONMARK_VERSION = 25 + CACHE_COMMONMARK_VERSION = 26 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index d1490d5d9b6..8a176be30a2 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -13,11 +13,12 @@ module Gitlab @dashboard_hash = dashboard_hash @project = project @dashboard_path = dashboard_path + @affected_environment_ids = [] end def execute import - rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError + rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError false end @@ -32,28 +33,51 @@ module Gitlab def import delete_stale_metrics create_or_update_metrics + update_prometheus_environments end # rubocop: disable CodeReuse/ActiveRecord def create_or_update_metrics # TODO: use upsert and worker for callbacks? + + affected_metric_ids = [] prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project)) + prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) + + affected_metric_ids << prometheus_metric.id end + + @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id end # rubocop: enable CodeReuse/ActiveRecord def delete_stale_metrics - identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } + identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } stale_metrics = PrometheusMetric.for_project(project) .for_dashboard_path(dashboard_path) .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers) + .not_identifier(identifiers_from_yml) + + return unless stale_metrics.exists? + + delete_stale_alerts(stale_metrics) + stale_metrics.each_batch { |batch| batch.delete_all } + end + + def delete_stale_alerts(stale_metrics) + stale_alerts = find_alerts(stale_metrics) + + affected_environment_ids = stale_alerts.get_environment_id + return unless affected_environment_ids.present? - # TODO: use destroy_all and worker for callbacks? - stale_metrics.each(&:destroy) + @affected_environment_ids += affected_environment_ids + stale_alerts.each_batch { |batch| batch.delete_all } + end + + def find_alerts(metrics) + Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute end def prometheus_metrics_attributes @@ -65,6 +89,19 @@ module Gitlab ).execute end end + + def update_prometheus_environments + affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project) + + return unless affected_environments.exists? + + affected_environments.each do |affected_environment| + ::Clusters::Applications::ScheduleUpdateService.new( + affected_environment.cluster_prometheus_adapter, + project + ).execute + end + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb new file mode 100644 index 00000000000..5ed4466f440 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + # Acts on metrics which have been ingested from source controlled dashboards + class CustomDashboardMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, includes + # the record's id in the dashboard config. + def transform! + database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute + + for_metrics do |metric| + metric_record = database_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb index 9e2bb0d1a70..ad9d78133af 100644 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -46,7 +46,7 @@ module Gitlab links&.each do |link| next unless link.is_a? Hash - Gitlab::UrlBlocker.validate!(link[:url], blocker_args) + Gitlab::UrlBlocker.validate!(link[:url], **blocker_args) rescue Gitlab::UrlBlocker::BlockedUrlError link[:url] = '' end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb index 4d94ab098ae..bc85dc4e131 100644 --- a/lib/gitlab/metrics/dashboard/transformers/errors.rb +++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb @@ -4,10 +4,10 @@ module Gitlab module Metrics module Dashboard module Transformers - TransformerError = Class.new(StandardError) - module Errors - class MissingAttribute < TransformerError + BaseError = Class.new(StandardError) + + class MissingAttribute < BaseError def initialize(attribute_name) super("Missing attribute: '#{attribute_name}'") end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 15db3999fa4..f6bda0dbea4 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -3,20 +3,13 @@ module Gitlab module Metrics class RequestsRackMiddleware - HTTP_METHODS = { - "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503), - "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503), - "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503), - "options" => %w(200 404), - "patch" => %w(200 202 204 400 403 404 409 416 422 500), - "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503), - "propfind" => %w(404), - "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500), - "report" => %w(404) - }.freeze + HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze + FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category' + FEATURE_CATEGORY_DEFAULT = 'unknown' + def initialize(app) @app = app end @@ -39,33 +32,39 @@ module Gitlab end def self.initialize_http_request_duration_seconds - HTTP_METHODS.each do |method, statuses| - statuses.each do |status| - http_request_duration_seconds.get({ method: method, status: status.to_s }) - end + HTTP_METHODS.each do |method| + http_request_duration_seconds.get({ method: method }) end end def call(env) method = env['REQUEST_METHOD'].downcase + method = 'INVALID' unless HTTP_METHODS.include?(method) started = Time.now.to_f + health_endpoint = health_endpoint?(env['PATH_INFO']) + status = 'undefined' + feature_category = nil begin - if health_endpoint?(env['PATH_INFO']) - RequestsRackMiddleware.http_health_requests_total.increment(method: method) - else - RequestsRackMiddleware.http_request_total.increment(method: method) - end - status, headers, body = @app.call(env) elapsed = Time.now.to_f - started - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) + feature_category = headers&.fetch(FEATURE_CATEGORY_HEADER, nil) + + unless health_endpoint + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed) + end [status, headers, body] rescue RequestsRackMiddleware.rack_uncaught_errors_count.increment raise + ensure + if health_endpoint + RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method) + else + RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT) + end end end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 8c4d150adad..d7935d65e12 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -54,7 +54,7 @@ module Gitlab end def unicorn_workers_count - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index e53ac00e77f..f9ba0a69b0e 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -61,7 +61,7 @@ module Gitlab end def current_transaction - Transaction.current + ::Gitlab::Metrics::Transaction.current end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 47d0b9ba8cb..4b65bbcc791 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -18,6 +18,15 @@ module Gitlab request = ActionDispatch::Request.new(env) render_go_doc(request) || @app.call(env) + rescue Gitlab::Auth::IpBlacklisted + Gitlab::AuthLogger.error( + message: 'Rack_Attack', + env: :blocklist, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath + ) + Rack::Response.new('', 403).finish end private diff --git a/lib/gitlab/middleware/handle_null_bytes.rb b/lib/gitlab/middleware/handle_null_bytes.rb new file mode 100644 index 00000000000..c88dfb6ee0b --- /dev/null +++ b/lib/gitlab/middleware/handle_null_bytes.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + # There is no valid reason for a request to contain a null byte (U+0000) + # so just return HTTP 400 (Bad Request) if we receive one + class HandleNullBytes + NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze + + attr_reader :app + + def initialize(app) + @app = app + end + + def call(env) + return [400, {}, ["Bad Request"]] if request_has_null_byte?(env) + + app.call(env) + end + + private + + def request_has_null_byte?(request) + return false if ENV['REJECT_NULL_BYTES'] == "1" + + request = Rack::Request.new(request) + + request.params.values.any? do |value| + param_has_null_byte?(value) + end + end + + def param_has_null_byte?(value, depth = 0) + # Guard against possible attack sending large amounts of nested params + # Should be safe as deeply nested params are highly uncommon. + return false if depth > 2 + + depth += 1 + + if value.respond_to?(:match) + string_contains_null_byte?(value) + elsif value.respond_to?(:values) + value.values.any? do |hash_value| + param_has_null_byte?(hash_value, depth) + end + elsif value.is_a?(Array) + value.any? do |array_value| + param_has_null_byte?(array_value, depth) + end + else + false + end + end + + def string_contains_null_byte?(string) + string.match?(NULL_BYTE_REGEX) + end + end + end +end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 8e6ac7610f2..7e98f1fc1f7 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -137,6 +137,7 @@ module Gitlab # TODO this class is meant to replace Handler when the feature flag # upload_middleware_jwt_params_handler is removed + # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps class HandlerForJWTParams < Handler def with_open_files @rewritten_fields.keys.each do |field| @@ -228,7 +229,7 @@ module Gitlab private def handler_class - if Feature.enabled?(:upload_middleware_jwt_params_handler) + if Feature.enabled?(:upload_middleware_jwt_params_handler, default_enabled: true) ::Gitlab::Middleware::Multipart::HandlerForJWTParams else ::Gitlab::Middleware::Multipart::Handler diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 8796dd4d7ec..46c74b8fe3c 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -10,9 +10,9 @@ module Gitlab @request_context = request_context end - def paginate(relation) + def paginate(relation, exclude_total_headers: false) paginate_with_limit_optimization(add_default_order(relation)).tap do |data| - add_pagination_headers(data) + add_pagination_headers(data, exclude_total_headers) end end @@ -27,7 +27,7 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT @@ -47,14 +47,14 @@ module Gitlab relation end - def add_pagination_headers(paginated_data) + def add_pagination_headers(paginated_data, exclude_total_headers) header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) - return if data_without_counts?(paginated_data) + return if exclude_total_headers || data_without_counts?(paginated_data) header 'X-Total', paginated_data.total_count.to_s header 'X-Total-Pages', total_pages(paginated_data).to_s diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index ded8d4ade3f..333564bee01 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -4,11 +4,11 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, query, project:, repository_ref: nil, filters: {}) + def initialize(current_user, query, project:, repository_ref: nil, sort: nil, filters: {}) @project = project @repository_ref = repository_ref.presence - super(current_user, query, [project], filters: filters) + super(current_user, query, [project], sort: sort, filters: filters) end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index e6e599e079d..a830f949b21 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -36,35 +36,37 @@ module Gitlab name == other.name && title == other.title end - def self.localized_templates_table - [ - ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), - ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), - ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), - ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'), - ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), - ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), - ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), - ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'), - ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), - ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), - ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), - ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), - ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), - ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'), - ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'), - ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), - ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'), - ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), - ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab.'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management') - ].freeze - end - class << self + # TODO: Review child inheritance of this table (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430928221) + def localized_templates_table + [ + ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), + ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'), + ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'), + ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'), + ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), + ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), + ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), + ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'), + ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), + ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), + ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), + ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), + ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), + ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman', 'illustrations/logos/middleman.svg'), + ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), + ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'), + ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), + ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'), + ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), + ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab.'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management') + ].freeze + end + def all localized_templates_table end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 6aa3f515ef0..efe07aa8ab2 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -145,9 +145,9 @@ module Gitlab run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids) end - desc _('Add a To Do') - explanation _('Adds a To Do.') - execution_message _('Added a To Do.') + desc _('Add a to do') + explanation _('Adds a to do.') + execution_message _('Added a to do.') types Issuable condition do quick_action_target.persisted? && @@ -157,9 +157,9 @@ module Gitlab @updates[:todo_event] = 'add' end - desc _('Mark To Do as done') - explanation _('Marks To Do as done.') - execution_message _('Marked To Do as done.') + desc _('Mark to do as done') + explanation _('Marks to do as done.') + execution_message _('Marked to do as done.') types Issuable condition do quick_action_target.persisted? && diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 98db8ff761e..c8c949a9363 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -121,6 +121,20 @@ module Gitlab result[:message] end end + + desc _('Approve a merge request') + explanation _('Approve the current merge request.') + types MergeRequest + condition do + quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) + end + command :approve do + success = MergeRequests::ApprovalService.new(quick_action_target.project, current_user).execute(quick_action_target) + + next unless success + + @execution_message[:approve] = _('Approved the current merge request.') + end end def merge_orchestration_service diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb index ff5754675e2..010a6b59da5 100644 --- a/lib/gitlab/redis/hll.rb +++ b/lib/gitlab/redis/hll.rb @@ -3,15 +3,16 @@ module Gitlab module Redis class HLL + BATCH_SIZE = 300 KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze KeyFormatError = Class.new(StandardError) def self.count(params) - self.new.count(params) + self.new.count(**params) end def self.add(params) - self.new.add(params) + self.new.add(**params) end def count(keys:) @@ -29,17 +30,24 @@ module Gitlab # 2020-216-{project_action} # i_{analytics}_dev_ops_score-2020-32 def add(key:, value:, expiry:) - unless KEY_REGEX.match?(key) - raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") - end + validate_key!(key) Gitlab::Redis::SharedState.with do |redis| redis.multi do |multi| - multi.pfadd(key, value) + Array.wrap(value).each_slice(BATCH_SIZE) { |batch| multi.pfadd(key, batch) } + multi.expire(key, expiry) end end end + + private + + def validate_key!(key) + return if KEY_REGEX.match?(key) + + raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") + end end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 8e23ac6aca5..693a10a9de3 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -46,6 +46,12 @@ module Gitlab maven_app_name_regex end + def nuget_version_regex + @nuget_version_regex ||= / + \A#{_semver_major_minor_patch_regex}(\.\d*)?#{_semver_prerelease_build_regex}\z + /x.freeze + end + def pypi_version_regex # See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159 @@ -61,6 +67,39 @@ module Gitlab )\z}xi.freeze end + def debian_package_name_regex + # See official parser + # https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n122 + # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze + # But we prefer a more strict version from Lintian + # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116 + @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze + end + + def debian_version_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205 + @debian_version_regex ||= %r{ + \A(?: + (?:([0-9]{1,9}):)? (?# epoch) + ([0-9][0-9a-z\.+~-]*) (?# version) + (?:(-[0-0a-z\.+~]+))? (?# revision) + )\z}xi.freeze + end + + def debian_architecture_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 + # But we limit to lower case + @debian_architecture_regex ||= %r{\A[a-z0-9][-a-z0-9]*\z}.freeze + end + + def debian_distribution_regex + @debian_distribution_regex ||= %r{\A[a-z0-9][a-z0-9\.-]*\z}i.freeze + end + + def debian_component_regex + @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze + end + def unbounded_semver_regex # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -68,16 +107,34 @@ module Gitlab # reordered to be greedy. Without this change, the unbounded regex would # only partially match "v0.0.0-20201230123456-abcdefabcdef". @unbounded_semver_regex ||= / + #{_semver_major_minor_patch_regex}#{_semver_prerelease_build_regex} + /x.freeze + end + + def semver_regex + @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + + # These partial semver regexes are intended for use in composing other + # regexes rather than being used alone. + def _semver_major_minor_patch_regex + @_semver_major_minor_patch_regex ||= / (?<major>0|[1-9]\d*) \.(?<minor>0|[1-9]\d*) \.(?<patch>0|[1-9]\d*) + /x.freeze + end + + def _semver_prerelease_build_regex + @_semver_prerelease_build_regex ||= / (?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))? (?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))? /x.freeze end - def semver_regex - @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + def prefixed_semver_regex + # identical to semver_regex, except starting with 'v' + @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) end def go_package_regex @@ -103,6 +160,14 @@ module Gitlab def generic_package_version_regex /\A\d+\.\d+\.\d+\z/ end + + def generic_package_name_regex + maven_file_name_regex + end + + def generic_package_file_name_regex + generic_package_name_regex + end end extend self @@ -211,8 +276,27 @@ module Gitlab "Must start with a letter, and cannot end with '-'" end + # The section start, e.g. section_start:12345678:NAME + def logs_section_prefix_regex + /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)/ + end + + # The optional section options, e.g. [collapsed=true] + def logs_section_options_regex + /(\[(?:\w+=\w+)(?:, ?(?:\w+=\w+))*\])?/ + end + + # The region end, always: \r\e\[0K + def logs_section_suffix_regex + /\r\033\[0K/ + end + def build_trace_section_regex - @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze + @build_trace_section_regexp ||= %r{ + #{logs_section_prefix_regex} + #{logs_section_options_regex} + #{logs_section_suffix_regex} + }x.freeze end def markdown_code_or_html_blocks diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index cd03a347355..8f5495ece5e 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -131,12 +131,12 @@ module Gitlab def shift_left move_sequence_before(true) - object.reset + object.reset_relative_position end def shift_right move_sequence_after(true) - object.reset + object.reset_relative_position end def create_space_left diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 67e23624045..9ee6f67e455 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -35,6 +35,10 @@ module Gitlab snippet, redirected_path = find_snippet(full_path) [snippet, snippet&.project, redirected_path] + elsif type.wiki? + wiki, redirected_path = find_wiki(full_path) + + [wiki, wiki.try(:project), redirected_path] else project, redirected_path = find_project(full_path) @@ -67,6 +71,17 @@ module Gitlab [Snippet.find_by_id_and_project(id: snippet_id, project: project), redirected_path] end + # Wiki path can be either: + # - namespace/project + # - group/subgroup/project + def self.find_wiki(wiki_path) + return [nil, nil] if wiki_path.blank? + + project, redirected_path = find_project(wiki_path) + + [project&.wiki, redirected_path] + end + def self.extract_snippet_info(snippet_path) path_segments = snippet_path.split('/') snippet_id = path_segments.pop diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index 82c57f7a07d..03d9f961dd9 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -6,9 +6,10 @@ module Gitlab attr_reader :limit # @param current_size_proc [Proc] returns repository size in bytes - def initialize(current_size_proc:, limit:, enabled: true) + def initialize(current_size_proc:, limit:, namespace:, enabled: true) @current_size_proc = current_size_proc @limit = limit + @namespace = namespace @enabled = enabled && limit != 0 end @@ -40,7 +41,13 @@ module Gitlab end def error_message - @error_message_object ||= Gitlab::RepositorySizeErrorMessage.new(self) + @error_message_object ||= ::Gitlab::RepositorySizeErrorMessage.new(self) end + + private + + attr_reader :namespace end end + +Gitlab::RepositorySizeChecker.prepend_if_ee('EE::Gitlab::RepositorySizeChecker') diff --git a/lib/gitlab/sample_data_template.rb b/lib/gitlab/sample_data_template.rb new file mode 100644 index 00000000000..ae74dc710b7 --- /dev/null +++ b/lib/gitlab/sample_data_template.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + class SampleDataTemplate < ProjectTemplate + class << self + def localized_templates_table + [ + SampleDataTemplate.new('basic', 'Basic', _('Basic Sample Data template with Issues, Merge Requests and Milestones.'), 'https://gitlab.com/gitlab-org/sample-data-templates/basic'), + SampleDataTemplate.new('serenity_valley', 'Serenity Valley', _('Serenity Valley Sample Data template.'), 'https://gitlab.com/gitlab-org/sample-data-templates/serenity-valley') + ].freeze + end + + def all + localized_templates_table + end + + def archive_directory + Rails.root.join("vendor/sample_data_templates") + end + end + end +end diff --git a/lib/gitlab/search/recent_items.rb b/lib/gitlab/search/recent_items.rb index 40d96ded275..593148025e1 100644 --- a/lib/gitlab/search/recent_items.rb +++ b/lib/gitlab/search/recent_items.rb @@ -6,9 +6,12 @@ module Gitlab # items. The #type and #finder methods are the only ones needed to be # implemented by classes inheriting from this. class RecentItems - ITEMS_LIMIT = 100 + ITEMS_LIMIT = 100 # How much history to remember from the user + SEARCH_LIMIT = 5 # How many matching items to return from search EXPIRES_AFTER = 7.days + attr_reader :user + def initialize(user:, items_limit: ITEMS_LIMIT, expires_after: EXPIRES_AFTER) @user = user @items_limit = items_limit @@ -30,21 +33,25 @@ module Gitlab end def search(term) - ids = with_redis do |redis| - redis.zrevrange(key, 0, @items_limit - 1) - end.map(&:to_i) - - finder.new(@user, search: term, in: 'title').execute.reorder(nil).id_in_ordered(ids) # rubocop: disable CodeReuse/ActiveRecord + finder.new(user, search: term, in: 'title') + .execute + .limit(SEARCH_LIMIT).reorder(nil).id_in_ordered(latest_ids) # rubocop: disable CodeReuse/ActiveRecord end private + def latest_ids + with_redis do |redis| + redis.zrevrange(key, 0, @items_limit - 1) + end.map(&:to_i) + end + def with_redis(&blk) Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord end def key - "recent_items:#{type.name.downcase}:#{@user.id}" + "recent_items:#{type.name.downcase}:#{user.id}" end def type diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 06d8dca2f70..b81264c5d0c 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query, :filters + attr_reader :current_user, :query, :sort, :filters # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -19,31 +19,23 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, query, limit_projects = nil, default_project_filter: false, filters: {}) + def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {}) @current_user = current_user @query = query @limit_projects = limit_projects || Project.all @default_project_filter = default_project_filter + @sort = sort @filters = filters end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil) should_preload = preload_method.present? - collection = case scope - when 'projects' - projects - when 'issues' - issues - when 'merge_requests' - merge_requests - when 'milestones' - milestones - when 'users' - users - else - should_preload = false - Kaminari.paginate_array([]) - end + collection = collection_for(scope) + + if collection.nil? + should_preload = false + collection = Kaminari.paginate_array([]) + end collection = collection.public_send(preload_method) if should_preload # rubocop:disable GitlabSecurity/PublicSend collection = collection.page(page).per(per_page) @@ -116,8 +108,41 @@ module Gitlab UsersFinder.new(current_user, search: query).execute end + # highlighting is only performed by Elasticsearch backed results + def highlight_map(scope) + {} + end + private + def collection_for(scope) + case scope + when 'projects' + projects + when 'issues' + issues + when 'merge_requests' + merge_requests + when 'milestones' + milestones + when 'users' + users + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_sort(scope) + case sort + when 'oldest' + scope.reorder('created_at ASC') + when 'newest' + scope.reorder('created_at DESC') + else + scope + end + end + # rubocop: enable CodeReuse/ActiveRecord + def projects limit_projects.search(query) end @@ -129,7 +154,7 @@ module Gitlab issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - issues + apply_sort(issues) end # rubocop: disable CodeReuse/ActiveRecord @@ -149,7 +174,7 @@ module Gitlab merge_requests = merge_requests.in_projects(project_ids_relation) end - merge_requests + apply_sort(merge_requests) end def default_scope @@ -193,6 +218,10 @@ module Gitlab end params[:state] = filters[:state] if filters.key?(:state) + + if [true, false].include?(filters[:confidential]) && Feature.enabled?(:search_filter_by_confidential) + params[:confidential] = filters[:confidential] + end end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index e26d45e1b33..4158cec9b09 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -87,7 +87,9 @@ module Gitlab SeedFu.quiet = true - yield + without_statement_timeout do + yield + end SeedFu.quiet = false ActiveRecord::Base.logger = old_logger @@ -114,6 +116,13 @@ module Gitlab def self.mute_mailer ActionMailer::MessageDelivery.prepend(DeliverNever) end + + def self.without_statement_timeout + ActiveRecord::Base.connection.execute('SET statement_timeout=0') + yield + ensure + ActiveRecord::Base.connection.execute('RESET statement_timeout') + end end end # :nocov: diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index e1a87a77f04..8793a672693 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -231,7 +231,7 @@ module Gitlab def rss_increase_by_jobs Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord + Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| rss_increase_by_job(job) end end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/server.rb b/lib/gitlab/sidekiq_middleware/worker_context/server.rb index d2d84742c17..2d8fd8002d2 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/server.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/server.rb @@ -12,8 +12,10 @@ module Gitlab # This is not a worker we know about, perhaps from a gem return yield unless worker_class.respond_to?(:get_worker_context) - # Use the context defined on the class level as a base context - wrap_in_optional_context(worker_class.get_worker_context, &block) + Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s) do + # Use the context defined on the class level as the more specific context + wrap_in_optional_context(worker_class.get_worker_context, &block) + end end end end diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 2d8df2ca204..714ca77c3e5 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -48,7 +48,7 @@ module Gitlab *Documentation* For more information about GitLab chatops, refer to its - documentation: https://docs.gitlab.com/ce/ci/chatops/README.html. + documentation: https://docs.gitlab.com/ee/ci/chatops/README.html. MESSAGE message diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb index f647c85e1c8..315c603c1dd 100644 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -3,11 +3,38 @@ module Gitlab module StaticSiteEditor module Config + # + # Base GitLab Static Site Editor Configuration facade + # class FileConfig - def data - { - static_site_generator: 'middleman' - } + ConfigError = Class.new(StandardError) + + def initialize(yaml) + content_hash = content_hash(yaml) + @global = Entry::Global.new(content_hash) + @global.compose! + rescue Gitlab::Config::Loader::FormatError => e + raise FileConfig::ConfigError, e.message + end + + def valid? + @global.valid? + end + + def errors + @global.errors + end + + def to_hash_with_defaults + # NOTE: The current approach of simply mapping all the descendents' keys and values ('config') + # into a flat hash may need to be enhanced as we add more complex, non-scalar entries. + @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h + end + + private + + def content_hash(yaml) + Gitlab::Config::Loader::Yaml.new(yaml).load! end end end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/global.rb b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb new file mode 100644 index 00000000000..c295ccf1d11 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # This class represents a global entry - root Entry for entire + # GitLab StaticSiteEditor Configuration file. + # + class Global < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[ + image_upload_path + mounts + static_site_generator + ].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :image_upload_path, Entry::ImageUploadPath, + description: 'Configuration of the Static Site Editor image upload path.' + entry :mounts, Entry::Mounts, + description: 'Configuration of the Static Site Editor mounts.' + entry :static_site_generator, Entry::StaticSiteGenerator, + description: 'Configuration of the Static Site Editor static site generator.' + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb new file mode 100644 index 00000000000..6a2b9e10d33 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the path to which images will be uploaded + # + class ImageUploadPath < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + + def self.default + 'source/images' + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb new file mode 100644 index 00000000000..b10956e17a5 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the mappings of mounted source directories to target paths + # + class Mount < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[source target].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :source, type: String, presence: true + validates :target, type: String, presence: true, allow_blank: true + end + + def self.default + # NOTE: This is the default for middleman projects. Ideally, this would be determined + # based on the defaults for whatever `static_site_generator` is configured. + { + source: 'source', + target: '' + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb new file mode 100644 index 00000000000..10bd377e419 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the mappings of mounted source directories to target paths + # + class Mounts < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + + entry :mount, Entry::Mount, description: 'Configuration of a Static Site Editor mount.' + + validations do + validates :config, type: Array, presence: true + end + + def skip_config_hash_validation? + true + end + + def self.default + [Entry::Mount.default] + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb new file mode 100644 index 00000000000..593c0951f93 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the static site generator tool/framework. + # + class StaticSiteGenerator < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String, inclusion: { in: %w[middleman], message: "should be 'middleman'" } + end + + def self.default + 'middleman' + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index f3dce74a32f..ff24ec69ab0 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -4,8 +4,6 @@ module Gitlab module StaticSiteEditor module Config class GeneratedConfig - SUPPORTED_EXTENSIONS = %w[.md].freeze - def initialize(repository, ref, path, return_url) @repository = repository @ref = ref @@ -23,7 +21,7 @@ module Gitlab project: project.path, namespace: project.namespace.full_path, return_url: sanitize_url(return_url), - is_supported_content: supported_content?.to_s, + is_supported_content: supported_content?, base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path), merge_requests_illustration_path: merge_requests_illustration_path } @@ -35,8 +33,12 @@ module Gitlab delegate :project, to: :repository + def supported_extensions + %w[.md].freeze + end + def commit_id - repository.commit(ref)&.id if ref + repository.commit(ref)&.id end def supported_content? @@ -50,7 +52,7 @@ module Gitlab def extension_supported? return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) - SUPPORTED_EXTENSIONS.any? { |ext| path.end_with?(ext) } + supported_extensions.any? { |ext| path.end_with?(ext) } end def file_exists? diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb new file mode 100644 index 00000000000..a918e7bec80 --- /dev/null +++ b/lib/gitlab/subscription_portal.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module SubscriptionPortal + def self.default_subscriptions_url + ::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com' + end + + SUBSCRIPTIONS_URL = ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url).freeze + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index bb1e9db55fa..e12af6bf0a4 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -3,7 +3,7 @@ module Gitlab module Template class GitlabCiYmlTemplate < BaseTemplate - BASE_EXCLUDED_PATTERNS = [%r{\.latest$}].freeze + BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze def content explanation = "# This file is a template, and might need editing before it works on your project." diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 72783a2d682..16e7b8a7eca 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -10,21 +10,21 @@ module Gitlab APPLICATION_DEFAULT = 1 # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class) + Theme = Struct.new(:id, :name, :css_class, :css_filename) # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo'), - Theme.new(4, 'Blue', 'ui-blue'), - Theme.new(7, 'Light Blue', 'ui-light-blue'), - Theme.new(5, 'Green', 'ui-green'), - Theme.new(8, 'Light Green', 'ui-light-green'), - Theme.new(9, 'Red', 'ui-red'), - Theme.new(10, 'Light Red', 'ui-light-red'), - Theme.new(2, 'Dark', 'ui-dark'), - Theme.new(3, 'Light', 'ui-light'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark') + Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo'), + Theme.new(4, 'Blue', 'ui-blue', 'theme_blue'), + Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue'), + Theme.new(5, 'Green', 'ui-green', 'theme_green'), + Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green'), + Theme.new(9, 'Red', 'ui-red', 'theme_red'), + Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red'), + Theme.new(2, 'Dark', 'ui-dark', 'theme_dark'), + Theme.new(3, 'Light', 'ui-light', 'theme_light'), + Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil) ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 89605ce5d07..68f24559b1f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -12,6 +12,19 @@ # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } module Gitlab class UsageData + CE_MEMOIZED_VALUES = %i( + issue_minimum_id + issue_maximum_id + project_minimum_id + project_maximum_id + user_minimum_id + user_maximum_id + unique_visit_service + deployment_minimum_id + deployment_maximum_id + auth_providers + ).freeze + class << self include Gitlab::Utils::UsageData include Gitlab::Utils::StrongMemoize @@ -138,8 +151,10 @@ module Gitlab pages_domains: count(PagesDomain), pool_repositories: count(PoolRepository), projects: count(Project), + projects_creating_incidents: distinct_count(Issue.incident, :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), + projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_service_enabled: count(AlertsService.active), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -166,8 +181,7 @@ module Gitlab user_preferences_usage, ingress_modsecurity_usage, container_expiration_policies_usage, - service_desk_counts, - snowplow_event_counts + service_desk_counts ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -175,7 +189,7 @@ module Gitlab end # rubocop: enable Metrics/AbcSize - def snowplow_event_counts(time_period: {}) + def snowplow_event_counts(time_period) return {} unless report_snowplow_events? { @@ -200,7 +214,7 @@ module Gitlab personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), project_snippets: count(ProjectSnippet.where(last_28_days_time_period)) }.merge( - snowplow_event_counts(time_period: last_28_days_time_period(column: :collector_tstamp)) + snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -242,7 +256,8 @@ module Gitlab signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), - grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? } + grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, + gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } end @@ -264,7 +279,8 @@ module Gitlab Gitlab::UsageDataCounters::SourceCodeCounter, Gitlab::UsageDataCounters::MergeRequestCounter, Gitlab::UsageDataCounters::DesignsCounter, - Gitlab::UsageDataCounters::KubernetesAgentCounter + Gitlab::UsageDataCounters::KubernetesAgentCounter, + Gitlab::UsageDataCounters::StaticSiteEditorCounter ] end @@ -287,7 +303,8 @@ module Gitlab }, database: { adapter: alt_usage_data { Gitlab::Database.adapter_name }, - version: alt_usage_data { Gitlab::Database.version } + version: alt_usage_data { Gitlab::Database.version }, + pg_system_id: alt_usage_data { Gitlab::Database.system_id } }, mail: { smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] } @@ -385,10 +402,12 @@ module Gitlab def services_usage # rubocop: disable UsageData/LargeTable: Service.available_services_names.each_with_object({}) do |service_name, response| - response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, instance: false, type: "#{service_name}_service".camelize)) + response["projects_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where(type: "#{service_name}_service".camelize)) + response["groups_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where(type: "#{service_name}_service".camelize)) response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize)) response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize)) - response["projects_inheriting_instance_#{service_name}_active".to_sym] = count(Service.active.where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) + response["projects_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) + response["groups_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize)) end.merge(jira_usage, jira_import_usage) # rubocop: enable UsageData/LargeTable: end @@ -444,8 +463,11 @@ module Gitlab # rubocop: enable UsageData/LargeTable # rubocop: enable CodeReuse/ActiveRecord + # augmented in EE def user_preferences_usage - {} # augmented in EE + { + user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active)) + } end def merge_requests_users(time_period) @@ -469,7 +491,7 @@ module Gitlab end def last_28_days_time_period(column: :created_at) - { column => 28.days.ago..Time.current } + { column => 30.days.ago..2.days.ago } end # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv @@ -541,6 +563,7 @@ module Gitlab groups: distinct_count(::GroupMember.where(time_period), :user_id), users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, + user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), projects_imported: { gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), @@ -555,7 +578,8 @@ module Gitlab jira: distinct_count(::JiraImportState.where(time_period), :user_id), fogbugz: projects_imported_count('fogbugz', time_period), phabricator: projects_imported_count('phabricator', time_period) - } + }, + groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -567,7 +591,8 @@ module Gitlab clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: user_minimum_id, - finish: user_maximum_id) + finish: user_maximum_id), + projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -696,10 +721,10 @@ module Gitlab counter = Gitlab::UsageDataCounters::EditorUniqueCounter { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(date_range) } + action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, + action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, + action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } } end @@ -801,17 +826,7 @@ module Gitlab end def clear_memoized - clear_memoization(:issue_minimum_id) - clear_memoization(:issue_maximum_id) - clear_memoization(:user_minimum_id) - clear_memoization(:user_maximum_id) - clear_memoization(:unique_visit_service) - clear_memoization(:deployment_minimum_id) - clear_memoization(:deployment_maximum_id) - clear_memoization(:approval_merge_request_rule_minimum_id) - clear_memoization(:approval_merge_request_rule_maximum_id) - clear_memoization(:project_minimum_id) - clear_memoization(:project_maximum_id) + CE_MEMOIZED_VALUES.each { |v| clear_memoization(v) } end # rubocop: disable CodeReuse/ActiveRecord @@ -843,6 +858,39 @@ module Gitlab def projects_imported_count(from, time_period) distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord end + + # rubocop:disable CodeReuse/ActiveRecord + def distinct_count_user_auth_by_provider(time_period) + counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| + hash[provider] = distinct_count( + ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id) + end + + if any_ldap_auth_providers? + counts['ldap'] = distinct_count( + ::AuthenticationEvent.success.ldap.where(time_period), :user_id + ) + end + + counts + end + # rubocop:enable CodeReuse/ActiveRecord + + # rubocop:disable UsageData/LargeTable + def auth_providers + strong_memoize(:auth_providers) do + ::AuthenticationEvent.providers + end + end + # rubocop:enable UsageData/LargeTable + + def auth_providers_except_ldap + auth_providers.reject { |provider| provider.starts_with?('ldap') } + end + + def any_ldap_auth_providers? + auth_providers.any? { |provider| provider.starts_with?('ldap') } + end end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 53bf6daea4c..eb132ef0967 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -72,7 +72,8 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current) end if eligible_for_totals?(events_names) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index fc1b5a59487..e8839875109 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -3,14 +3,42 @@ module Gitlab module UsageDataCounters module IssueActivityUniqueCounter - ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' - ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_CATEGORY = 'issues_edit' + ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed' + ISSUE_CREATED = 'g_project_management_issue_created' + ISSUE_CLOSED = 'g_project_management_issue_closed' + ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed' + ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed' ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' - ISSUE_CATEGORY = 'issues_edit' + ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed' + ISSUE_REOPENED = 'g_project_management_issue_reopened' + ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' + ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed' + ISSUE_CROSS_REFERENCED = 'g_project_management_issue_cross_referenced' + ISSUE_MOVED = 'g_project_management_issue_moved' + ISSUE_RELATED = 'g_project_management_issue_related' + ISSUE_UNRELATED = 'g_project_management_issue_unrelated' + ISSUE_MARKED_AS_DUPLICATE = 'g_project_management_issue_marked_as_duplicate' + ISSUE_LOCKED = 'g_project_management_issue_locked' + ISSUE_UNLOCKED = 'g_project_management_issue_unlocked' + ISSUE_ADDED_TO_EPIC = 'g_project_management_issue_added_to_epic' + ISSUE_REMOVED_FROM_EPIC = 'g_project_management_issue_removed_from_epic' + ISSUE_CHANGED_EPIC = 'g_project_management_issue_changed_epic' + ISSUE_DESIGNS_ADDED = 'g_project_management_issue_designs_added' + ISSUE_DESIGNS_MODIFIED = 'g_project_management_issue_designs_modified' + ISSUE_DESIGNS_REMOVED = 'g_project_management_issue_designs_removed' + ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed' + ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed' + ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed' class << self + def track_issue_created_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CREATED, author, time) + end + def track_issue_title_changed_action(author:, time: Time.zone.now) track_unique_action(ISSUE_TITLE_CHANGED, author, time) end @@ -31,10 +59,98 @@ module Gitlab track_unique_action(ISSUE_MADE_VISIBLE, author, time) end + def track_issue_closed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CLOSED, author, time) + end + + def track_issue_reopened_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_REOPENED, author, time) + end + + def track_issue_label_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_LABEL_CHANGED, author, time) + end + + def track_issue_milestone_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) + end + + def track_issue_iteration_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_ITERATION_CHANGED, author, time) + end + + def track_issue_weight_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_WEIGHT_CHANGED, author, time) + end + + def track_issue_cross_referenced_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CROSS_REFERENCED, author, time) + end + + def track_issue_moved_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MOVED, author, time) + end + + def track_issue_related_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_RELATED, author, time) + end + + def track_issue_unrelated_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_UNRELATED, author, time) + end + + def track_issue_marked_as_duplicate_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author, time) + end + + def track_issue_locked_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_LOCKED, author, time) + end + + def track_issue_unlocked_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_UNLOCKED, author, time) + end + + def track_issue_added_to_epic_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_ADDED_TO_EPIC, author, time) + end + + def track_issue_removed_from_epic_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_REMOVED_FROM_EPIC, author, time) + end + + def track_issue_changed_epic_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CHANGED_EPIC, author, time) + end + + def track_issue_designs_added_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_DESIGNS_ADDED, author, time) + end + + def track_issue_designs_modified_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_DESIGNS_MODIFIED, author, time) + end + + def track_issue_designs_removed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_DESIGNS_REMOVED, author, time) + end + + def track_issue_due_date_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_DUE_DATE_CHANGED, author, time) + end + + def track_issue_time_estimate_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author, time) + end + + def track_issue_time_spent_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) + end + private def track_unique_action(action, author, time) - return unless Feature.enabled?(:track_issue_activity_actions) + return unless Feature.enabled?(:track_issue_activity_actions, default_enabled: true) return unless author Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index 25e7f858bb1..bc56c5d6d9b 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -185,6 +185,11 @@ redis_slot: incident_management category: incident_management aggregation: weekly +# Testing category +- name: i_testing_test_case_parsed + category: testing + redis_slot: testing + aggregation: weekly # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -206,3 +211,95 @@ category: issues_edit redis_slot: project_management aggregation: daily +- name: g_project_management_issue_created + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_closed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_reopened + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_label_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_milestone_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_iteration_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_weight_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_cross_referenced + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_moved + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_related + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_unrelated + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_marked_as_duplicate + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_locked + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_unlocked + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_added_to_epic + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_removed_from_epic + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_changed_epic + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_designs_added + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_designs_modified + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_designs_removed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_due_date_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_time_estimate_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_time_spent_changed + category: issues_edit + redis_slot: project_management + aggregation: daily diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb new file mode 100644 index 00000000000..8886a106da8 --- /dev/null +++ b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class StaticSiteEditorCounter < BaseCounter + KNOWN_EVENTS = %w[views].freeze + PREFIX = 'static_site_editor' + + class << self + def increment_views_count + count(:views) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index bacd63ab282..c54e766230e 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -22,7 +22,7 @@ module Gitlab end def sum(relation, column, *rest) - relation.select(relation.all.table[column].sum).to_sql # rubocop:disable CodeReuse/ActiveRecord + relation.select(relation.all.table[column].sum).to_sql end private diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index ca6a36c9cea..5267733d220 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -39,9 +39,9 @@ module Gitlab FALLBACK = -1 - def count(relation, column = nil, batch: true, start: nil, finish: nil) + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch - Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish) + Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) else relation.count end @@ -106,7 +106,6 @@ module Gitlab # @param values [Array|String] the values counted def track_usage_event(event_name, values) return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - return unless Gitlab::CurrentSettings.usage_ping_enabled? Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s) end diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb index 302697ff8eb..5768bab03c6 100644 --- a/lib/gitlab/view/presenter/factory.rb +++ b/lib/gitlab/view/presenter/factory.rb @@ -10,7 +10,7 @@ module Gitlab end def fabricate! - presenter_class.new(subject, attributes) + presenter_class.new(subject, **attributes) end private diff --git a/lib/gitlab/visibility_level_checker.rb b/lib/gitlab/visibility_level_checker.rb index f15f1486a4e..3ffd86c4f8c 100644 --- a/lib/gitlab/visibility_level_checker.rb +++ b/lib/gitlab/visibility_level_checker.rb @@ -3,7 +3,7 @@ # Gitlab::VisibilityLevelChecker verifies that: # - Current @project.visibility_level is not restricted # - Override visibility param is not restricted -# - @see https://docs.gitlab.com/ce/api/project_import_export.html#import-a-file +# - @see https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file # # @param current_user [User] Current user object to verify visibility level against # @param project [Project] Current project that is being created/imported diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb index fda41da5a94..069e68e8d29 100644 --- a/lib/gitlab/webpack/dev_server_middleware.rb +++ b/lib/gitlab/webpack/dev_server_middleware.rb @@ -11,8 +11,15 @@ module Gitlab @proxy_host = opts.fetch(:proxy_host, 'localhost') @proxy_port = opts.fetch(:proxy_port, 3808) @proxy_path = opts[:proxy_path] if opts[:proxy_path] + @proxy_scheme = opts[:proxy_https] ? 'https' : 'http' - super(app, backend: "http://#{@proxy_host}:#{@proxy_port}", **opts) + super(app, backend: "#{@proxy_scheme}://#{@proxy_host}:#{@proxy_port}", **opts) + end + + # disable SSL check since any cert used here will likely be self-signed + def rewrite_env(env) + env["rack.ssl_verify_none"] = true + env end def perform_request(env) diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index d2c01bbd55e..5873d9c2b99 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -1,16 +1,33 @@ # frozen_string_literal: true -require 'webpack/rails/manifest' +require 'net/http' +require 'uri' module Gitlab module Webpack - class Manifest < ::Webpack::Rails::Manifest - # Raised if a supplied asset does not exist in the webpack manifest + class Manifest + # Raised if we can't read our webpack manifest for whatever reason + class ManifestLoadError < StandardError + def initialize(message, orig) + super "#{message}\n\n(original error #{orig.class.name}: #{orig})" + end + end + + # Raised if webpack couldn't build one of your entry points + class WebpackError < StandardError + def initialize(errors) + super "Error in webpack compile, details follow below:\n#{errors.join("\n\n")}" + end + end + + # Raised if a supplied entry point does not exist in the webpack manifest AssetMissingError = Class.new(StandardError) class << self + include Gitlab::Utils::StrongMemoize + def entrypoint_paths(source) - raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled? + raise WebpackError, manifest["errors"] unless manifest_bundled? dll_assets = manifest.fetch("dllAssets", []) entrypoint = manifest["entrypoints"][source] @@ -18,12 +35,93 @@ module Gitlab # Can be either a string or an array of strings. # Do not include source maps as they are not javascript [dll_assets, entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| - "/#{::Rails.configuration.webpack.public_path}/#{p}" + "/#{Gitlab.config.webpack.public_path}/#{p}" + end + else + raise AssetMissingError, "Can't find asset '#{source}' in webpack manifest" + end + end + + def asset_paths(source) + raise WebpackError, manifest["errors"] unless manifest_bundled? + + paths = manifest["assetsByChunkName"][source] + if paths + # Can be either a string or an array of strings. + # Do not include source maps as they are not javascript + [paths].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| + "/#{Gitlab.config.webpack.public_path}/#{p}" end else raise AssetMissingError, "Can't find entry point '#{source}' in webpack manifest" end end + + def clear_manifest! + clear_memoization(:manifest) + end + + private + + def manifest_bundled? + !manifest["errors"].any? { |error| error.include? "Module build failed" } + end + + def manifest + if Gitlab.config.webpack.dev_server.enabled + # Don't cache if we're in dev server mode, manifest may change ... + load_manifest + else + # ... otherwise cache at class level, as JSON loading/parsing can be expensive + strong_memoize(:manifest) { load_manifest } + end + end + + def load_manifest + data = if Gitlab.config.webpack.dev_server.enabled + load_dev_server_manifest + else + load_static_manifest + end + + Gitlab::Json.parse(data) + end + + def load_dev_server_manifest + host = Gitlab.config.webpack.dev_server.host + port = Gitlab.config.webpack.dev_server.port + scheme = Gitlab.config.webpack.dev_server.https ? 'https' : 'http' + uri = Addressable::URI.new(scheme: scheme, host: host, port: port, path: dev_server_path) + + # localhost could be blocked via Gitlab::HTTP + response = HTTParty.get(uri.to_s, verify: false) # rubocop:disable Gitlab/HTTParty + + return response.body if response.code == 200 + + raise "HTTP error #{response.code}" + rescue OpenSSL::SSL::SSLError, EOFError => e + ssl_status = Gitlab.config.webpack.dev_server.https ? ' over SSL' : '' + raise ManifestLoadError.new("Could not connect to webpack-dev-server at #{uri}#{ssl_status}.\n\nIs SSL enabled? Check that settings in `gitlab.yml` and webpack-dev-server match.", e) + rescue => e + raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri}.\n\nIs webpack-dev-server running? Try running `gdk status webpack` or `gdk tail webpack`.", e) + end + + def load_static_manifest + File.read(static_manifest_path) + rescue => e + raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path}.\n\nHave you run `rake gitlab:assets:compile`?", e) + end + + def static_manifest_path + ::Rails.root.join( + Gitlab.config.webpack.output_dir, + Gitlab.config.webpack.manifest_filename + ) + end + + def dev_server_path + "/#{Gitlab.config.webpack.public_path}/#{Gitlab.config.webpack.manifest_filename}" + end end end end diff --git a/lib/gitlab/whats_new.rb b/lib/gitlab/whats_new.rb new file mode 100644 index 00000000000..e95ace2c475 --- /dev/null +++ b/lib/gitlab/whats_new.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module WhatsNew + CACHE_DURATION = 1.day + WHATS_NEW_FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml') + + private + + def whats_new_most_recent_release_items + Rails.cache.fetch('whats_new:release_items', expires_in: CACHE_DURATION) do + file = File.read(most_recent_release_file_path) + + items = YAML.safe_load(file, permitted_classes: [Date]) + + items if items.is_a?(Array) + end + rescue => e + Gitlab::ErrorTracking.track_exception(e, yaml_file_path: most_recent_release_file_path) + + nil + end + + def most_recent_release_file_path + @most_recent_release_file_path ||= Dir.glob(WHATS_NEW_FILES_PATH).max + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 8a5acd242d9..eb780a2f7f6 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -269,7 +269,8 @@ module Gitlab commit_id: metadata['CommitId'], prefix: metadata['ArchivePrefix'], format: format, - path: path.presence || "" + path: path.presence || "", + include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive) ).to_proto ) } diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb index a906beda80e..bbb64e0d5da 100644 --- a/lib/gitlab_danger.rb +++ b/lib/gitlab_danger.rb @@ -11,7 +11,7 @@ class GitlabDanger karma database commit_messages - telemetry + product_analytics utility_css pajamas ].freeze @@ -24,6 +24,7 @@ class GitlabDanger ce_ee_vue_templates sidekiq_queues specialization_labels + ci_templates ].freeze MESSAGE_PREFIX = '==>'.freeze diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index 9668badc757..f16bd7c735b 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -66,7 +66,7 @@ module GoogleApi cluster_options = make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons) - request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(cluster_options) + request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(**cluster_options) service.create_cluster(project_id, zone, request_body, options: user_agent_header) end diff --git a/lib/grafana/client.rb b/lib/grafana/client.rb index b419f79bace..7c0e56b61c8 100644 --- a/lib/grafana/client.rb +++ b/lib/grafana/client.rb @@ -19,8 +19,8 @@ module Grafana # @param name [String] Unique identifier for a Grafana datasource def get_datasource(name:) # CGI#escape formats strings such that the Grafana endpoint - # will not recognize the dashboard name. Preferring URI#escape. - http_get("#{@api_url}/api/datasources/name/#{URI.escape(name)}") # rubocop:disable Lint/UriEscapeUnescape + # will not recognize the dashboard name. Prefer Addressable::URI#encode_component. + http_get("#{@api_url}/api/datasources/name/#{Addressable::URI.encode_component(name)}") end # @param datasource_id [String] Grafana ID for the datasource diff --git a/lib/pager_duty/validator/schemas/message.json b/lib/pager_duty/validator/schemas/message.json new file mode 100644 index 00000000000..b1a3185cd1a --- /dev/null +++ b/lib/pager_duty/validator/schemas/message.json @@ -0,0 +1,47 @@ +{ + "type": "object", + "required": ["event", "incident"], + "properties": { + "event": { "type": "string" }, + "incident": { + "type": "object", + "required": [ + "html_url", + "incident_number", + "title", + "status", + "created_at", + "urgency", + "incident_key" + ], + "properties": { + "html_url": { "type": "string" }, + "incindent_number": { "type": "integer" }, + "title": { "type": "string" }, + "status": { "type": "string" }, + "created_at": { "type": "string" }, + "urgency": { "type": "string", "enum": ["high", "low"] }, + "incident_key": { "type": ["string", "null"] }, + "assignments": { + "type": "array", + "items": { + "assignee": { + "type": "array", + "items": { + "summary": { "type": "string" }, + "html_url": { "type": "string" } + } + } + } + }, + "impacted_services": { + "type": "array", + "items": { + "summary": { "type": "string" }, + "html_url": { "type": "string" } + } + } + } + } + } +} diff --git a/lib/pager_duty/webhook_payload_parser.rb b/lib/pager_duty/webhook_payload_parser.rb index 573fb36f0ca..11071926cf2 100644 --- a/lib/pager_duty/webhook_payload_parser.rb +++ b/lib/pager_duty/webhook_payload_parser.rb @@ -2,6 +2,8 @@ module PagerDuty class WebhookPayloadParser + SCHEMA_PATH = File.join('lib', 'pager_duty', 'validator', 'schemas', 'message.json') + def initialize(payload) @payload = payload end @@ -11,7 +13,7 @@ module PagerDuty end def call - Array(payload['messages']).map { |msg| parse_message(msg) } + Array(payload['messages']).map { |msg| parse_message(msg) }.reject(&:empty?) end private @@ -19,6 +21,8 @@ module PagerDuty attr_reader :payload def parse_message(message) + return {} unless valid_message?(message) + { 'event' => message['event'], 'incident' => parse_incident(message['incident']) @@ -26,8 +30,6 @@ module PagerDuty end def parse_incident(incident) - return {} if incident.blank? - { 'url' => incident['html_url'], 'incident_number' => incident['incident_number'], @@ -62,5 +64,9 @@ module PagerDuty def reject_empty(entities) Array(entities).reject { |e| e['summary'].blank? && e['url'].blank? } end + + def valid_message?(message) + ::JSONSchemer.schema(Pathname.new(SCHEMA_PATH)).valid?(message) + end end end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index ed3470f81f4..77108bb81ca 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -17,24 +17,35 @@ module Peek } }.freeze + def results + super.merge(calls: detailed_calls) + end + def self.thresholds @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS) end private + def detailed_calls + "#{calls} (#{cached_calls} cached)" + end + + def cached_calls + detail_store.count { |item| item[:cached] == 'cached' } + end + def setup_subscribers super subscribe('sql.active_record') do |_, start, finish, _, data| if Gitlab::PerformanceBar.enabled_for_request? - unless data[:cached] - detail_store << { - duration: finish - start, - sql: data[:sql].strip, - backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) - } - end + detail_store << { + duration: finish - start, + sql: data[:sql].strip, + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller), + cached: data[:cached] ? 'cached' : '' + } end end end diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb index 389f5301079..4cc2e85c7bb 100644 --- a/lib/peek/views/detailed_view.rb +++ b/lib/peek/views/detailed_view.rb @@ -23,7 +23,7 @@ module Peek private def duration - detail_store.map { |entry| entry[:duration] }.sum * 1000 # rubocop:disable CodeReuse/ActiveRecord + detail_store.map { |entry| entry[:duration] }.sum * 1000 end def calls diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb index 679c021c730..ac33f0b3c2c 100644 --- a/lib/safe_zip/extract.rb +++ b/lib/safe_zip/extract.rb @@ -19,11 +19,7 @@ module SafeZip def extract(opts = {}) params = SafeZip::ExtractParams.new(**opts) - if Feature.enabled?(:safezip_use_rubyzip, default_enabled: true) - extract_with_ruby_zip(params) - else - legacy_unsafe_extract_with_system_zip(params) - end + extract_with_ruby_zip(params) end private @@ -53,21 +49,5 @@ module SafeZip .extract end end - - def legacy_unsafe_extract_with_system_zip(params) - # Requires UnZip at least 6.00 Info-ZIP. - # -n never overwrite existing files - args = %W(unzip -n -qq #{archive_path}) - - # We add * to end of directory, because we want to extract directory and all subdirectories - args += params.directories_wildcard - - # Target directory where we extract - args += %W(-d #{params.extract_path}) - - unless system(*args) - raise Error, 'archive failed to extract' - end - end end end diff --git a/lib/sentry/client/issue.rb b/lib/sentry/client/issue.rb index c5e9df9cd21..f714bda49fd 100644 --- a/lib/sentry/client/issue.rb +++ b/lib/sentry/client/issue.rb @@ -14,7 +14,7 @@ module Sentry }.freeze def list_issues(**keyword_args) - response = get_issues(keyword_args) + response = get_issues(**keyword_args) issues = response[:issues] pagination = response[:pagination] @@ -44,7 +44,7 @@ module Sentry def get_issues(**keyword_args) response = http_get( api_urls.issues_url, - query: list_issue_sentry_query(keyword_args) + query: list_issue_sentry_query(**keyword_args) ) { diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb index 697ced3590b..a205861b9a9 100644 --- a/lib/system_check/app/redis_version_check.rb +++ b/lib/system_check/app/redis_version_check.rb @@ -37,7 +37,7 @@ module SystemCheck @custom_error_message ) for_more_information( - 'doc/administration/high_availability/redis.md#provide-your-own-redis-instance' + 'doc/administration/redis/index.html#redis-replication-and-failover-using-the-non-bundled-redis' ) fix_and_rerun end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 1e28d15f75e..e2c92054d62 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -38,7 +38,7 @@ namespace :gettext do Rake::Task['gettext:find'].invoke # leave only the required changes. - unless system(*%w(git checkout -- locale/*/gitlab.po)) + unless system(*%w(git -c core.hooksPath=/dev/null checkout -- locale/*/gitlab.po)) raise 'failed to cleanup generated locale/*/gitlab.po files' end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake index caa583fb3a9..ab2d77eeaf0 100644 --- a/lib/tasks/gitlab/assets.rake +++ b/lib/tasks/gitlab/assets.rake @@ -81,7 +81,7 @@ namespace :gitlab do if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists - Rake::Task['webpack:compile'].invoke + system('yarn webpack') end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2a3713ed85c..de2dfca8c1b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -107,7 +107,7 @@ namespace :gitlab do puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) exit 1 else - Backup::Repository.new(progress).dump( + Backup::Repositories.new(progress).dump( max_concurrency: max_concurrency, max_storage_concurrency: max_storage_concurrency ) @@ -117,7 +117,7 @@ namespace :gitlab do task restore: :gitlab_environment do puts_time "Restoring repositories ...".color(:blue) - Backup::Repository.new(progress).restore + Backup::Repositories.new(progress).restore puts_time "done".color(:green) end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 425f66918b0..a3f20f31f64 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -35,6 +35,11 @@ namespace :gitlab do # Truncate schema_migrations to ensure migrations re-run connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations' + # Drop any views + connection.views.each do |view| + connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE") + end + # Drop tables with cascade to avoid dependent table errors # PG: http://www.postgresql.org/docs/current/static/ddl-depend.html # Add `IF EXISTS` because cascade could have already deleted a table. @@ -60,6 +65,19 @@ namespace :gitlab do end end + desc 'GitLab | DB | Run database migrations and print `unattended_migrations_completed` if action taken' + task unattended: :environment do + no_database = !ActiveRecord::Base.connection.schema_migration.table_exists? + needs_migrations = ActiveRecord::Base.connection.migration_context.needs_migration? + + if no_database || needs_migrations + Rake::Task['gitlab:db:configure'].invoke + puts "unattended_migrations_completed" + else + puts "unattended_migrations_static" + end + end + desc 'GitLab | DB | Checks if migrations require downtime or not' task :downtime_check, [:ref] => :environment do |_, args| abort 'You must specify a Git reference to compare with' unless args[:ref] @@ -169,9 +187,21 @@ namespace :gitlab do desc 'reindex a regular (non-unique) index without downtime to eliminate bloat' task :reindex, [:index_name] => :environment do |_, args| - raise ArgumentError, 'must give the index name to reindex' unless args[:index_name] + unless Feature.enabled?(:database_reindexing, type: :ops) + puts "This feature (database_reindexing) is currently disabled.".color(:yellow) + exit + end + + indexes = if args[:index_name] + [Gitlab::Database::PostgresIndex.by_identifier(args[:index_name])] + else + Gitlab::Database::Reindexing.candidate_indexes.random_few(2) + end - Gitlab::Database::ConcurrentReindex.new(args[:index_name], logger: Logger.new(STDOUT)).execute + Gitlab::Database::Reindexing.perform(indexes) + rescue => e + Gitlab::AppLogger.error(e) + raise end end end diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 0b98755a77c..b242329d720 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -37,7 +37,10 @@ namespace :gitlab do web_hooks.find_each do |hook| next unless hook.url == web_hook_url - hook.destroy! + result = WebHooks::DestroyService.new(nil).sync_destroy(hook) + + raise "Unable to destroy Web hook" unless result[:status] == :success + count += 1 end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index cd5943b552e..9b034d1c6c2 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -42,6 +42,9 @@ class UploadedFile @remote_id = remote_id end + # TODO this function is meant to replace .from_params when the feature flag + # upload_middleware_jwt_params_handler is removed + # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps def self.from_params_without_field(params, upload_paths) path = params['path'] remote_id = params['remote_id'] @@ -68,6 +71,10 @@ class UploadedFile ) end + # Deprecated. Don't use it. + # .from_params_without_field will replace this one + # See .from_params_without_field and + # https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps def self.from_params(params, field, upload_paths, path_override = nil) path = path_override || params["#{field}.path"] remote_id = params["#{field}.remote_id"] |