diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 08:27:35 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 08:27:35 +0000 |
| commit | 7e9c479f7de77702622631cff2628a9c8dcbc627 (patch) | |
| tree | c8f718a08e110ad7e1894510980d2155a6549197 /lib | |
| parent | e852b0ae16db4052c1c567d9efa4facc81146e88 (diff) | |
| download | gitlab-ce-7e9c479f7de77702622631cff2628a9c8dcbc627.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42
Diffstat (limited to 'lib')
382 files changed, 6389 insertions, 1912 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 7e3d70a210a..e6ce62a1c6e 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -8,6 +8,8 @@ module API helpers ::API::Helpers::MembersHelpers + feature_category :authentication_and_authorization + %w[group project].each do |source_type| params do requires :id, type: String, desc: "The #{source_type} ID" diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb index 44c389d6f94..654d3a48162 100644 --- a/lib/api/admin/ci/variables.rb +++ b/lib/api/admin/ci/variables.rb @@ -8,6 +8,8 @@ module API before { authenticated_as_admin! } + feature_category :continuous_integration + namespace 'admin' do namespace 'ci' do namespace 'variables' do diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index ce1bdd65eff..679e231b283 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -5,6 +5,8 @@ module API class InstanceClusters < ::API::Base include PaginationParams + feature_category :kubernetes_management + before do authenticated_as_admin! end diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb index c2e9de5fb4e..7e561783685 100644 --- a/lib/api/admin/sidekiq.rb +++ b/lib/api/admin/sidekiq.rb @@ -5,6 +5,8 @@ module API class Sidekiq < ::API::Base before { authenticated_as_admin! } + feature_category :not_owned + namespace 'admin' do namespace 'sidekiq' do namespace 'queues' do diff --git a/lib/api/api.rb b/lib/api/api.rb index 84b4d5a5835..ea149f25584 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -25,7 +25,8 @@ module API Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new, Gitlab::GrapeLogging::Loggers::PerfLogger.new, Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new, - Gitlab::GrapeLogging::Loggers::ContextLogger.new + Gitlab::GrapeLogging::Loggers::ContextLogger.new, + Gitlab::GrapeLogging::Loggers::ContentLogger.new ] allow_access_with_scope :api @@ -48,11 +49,17 @@ module API before do coerce_nil_params_to_array! + api_endpoint = env['api.endpoint'] + feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s + + header[Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER] = feature_category + Gitlab::ApplicationContext.push( user: -> { @current_user }, project: -> { @project }, namespace: -> { @group }, - caller_id: route.origin + caller_id: route.origin, + feature_category: feature_category ) end @@ -115,7 +122,14 @@ module API format :json formatter :json, Gitlab::Json::GrapeFormatter - content_type :txt, "text/plain" + + # There is a small chance some users depend on the old behavior. + # We this change under a feature flag to see if affects GitLab.com users. + if Gitlab::Database.cached_table_exists?('features') && Feature.enabled?(:api_json_content_type) + content_type :json, 'application/json' + else + content_type :txt, 'text/plain' + end # Ensure the namespace is right, otherwise we might load Grape::API::Helpers helpers ::API::Helpers @@ -147,6 +161,8 @@ module API mount ::API::Commits mount ::API::CommitStatuses mount ::API::ContainerRegistryEvent + mount ::API::ContainerRepositories + mount ::API::DependencyProxy mount ::API::DeployKeys mount ::API::DeployTokens mount ::API::Deployments @@ -171,6 +187,7 @@ module API mount ::API::ImportBitbucketServer mount ::API::ImportGithub mount ::API::IssueLinks + mount ::API::Invitations mount ::API::Issues mount ::API::JobArtifacts mount ::API::Jobs @@ -202,7 +219,8 @@ module API mount ::API::DebianGroupPackages mount ::API::DebianProjectPackages mount ::API::MavenPackages - mount ::API::NpmPackages + mount ::API::NpmProjectPackages + mount ::API::NpmInstancePackages mount ::API::GenericPackages mount ::API::GoProxy mount ::API::Pages @@ -222,6 +240,7 @@ module API mount ::API::ProjectTemplates mount ::API::Terraform::State mount ::API::Terraform::StateVersion + mount ::API::PersonalAccessTokens mount ::API::ProtectedBranches mount ::API::ProtectedTags mount ::API::Releases @@ -266,7 +285,7 @@ module API end end - route :any, '*path' do + route :any, '*path', feature_category: :not_owned do error!('404 Not Found', 404) end end diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index 00b495bbc1e..fe498bf611b 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -4,6 +4,8 @@ module API class Appearance < ::API::Base before { authenticated_as_admin! } + feature_category :navigation + helpers do def current_appearance @current_appearance ||= (::Appearance.current || ::Appearance.new) diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 2afe8763d9d..8b14e16b495 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -5,6 +5,8 @@ module API class Applications < ::API::Base before { authenticated_as_admin! } + feature_category :authentication_and_authorization + resource :applications do helpers do def validate_redirect_uri(value) diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index 5a9b9940fcf..a42d89ddf83 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -2,6 +2,8 @@ module API class Avatar < ::API::Base + feature_category :users + resource :avatar do desc 'Return avatar url for a user' do success Entities::Avatar diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 6d40ae8f5ff..8ea4f32d3eb 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -6,9 +6,9 @@ module API before { authenticate! } AWARDABLES = [ - { type: 'issue', find_by: :iid }, - { type: 'merge_request', find_by: :iid }, - { type: 'snippet', find_by: :id } + { type: 'issue', find_by: :iid, feature_category: :issue_tracking }, + { type: 'merge_request', find_by: :iid, feature_category: :code_review }, + { type: 'snippet', find_by: :id, feature_category: :snippets } ].freeze params do @@ -34,7 +34,7 @@ module API params do use :pagination end - get endpoint do + get endpoint, feature_category: awardable_params[:feature_category] do if can_read_awardable? awards = awardable.award_emoji present paginate(awards), with: Entities::AwardEmoji @@ -50,7 +50,7 @@ module API params do requires :award_id, type: Integer, desc: 'The ID of the award' end - get "#{endpoint}/:award_id" do + get "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do if can_read_awardable? present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji else @@ -65,7 +65,7 @@ module API params do requires :name, type: String, desc: 'The name of a award_emoji (without colons)' end - post endpoint do + post endpoint, feature_category: awardable_params[:feature_category] do not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? service = AwardEmojis::AddService.new(awardable, params[:name], current_user).execute @@ -84,7 +84,7 @@ module API params do requires :award_id, type: Integer, desc: 'The ID of an award emoji' end - delete "#{endpoint}/:award_id" do + delete "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do award = awardable.award_emoji.find(params[:award_id]) unauthorized! unless award.user == current_user || current_user.admin? diff --git a/lib/api/badges.rb b/lib/api/badges.rb index fc00594c9ec..04f155be4e1 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -8,6 +8,8 @@ module API helpers ::API::Helpers::BadgesHelpers + feature_category :continuous_integration + helpers do def find_source_if_admin(source_type) source = find_source(source_type, params[:id]) diff --git a/lib/api/base.rb b/lib/api/base.rb index e174cef3bad..33e47c18fcd 100644 --- a/lib/api/base.rb +++ b/lib/api/base.rb @@ -2,5 +2,30 @@ module API class Base < Grape::API::Instance # rubocop:disable API/Base + include ::Gitlab::WithFeatureCategory + + class << self + def feature_category_for_app(app) + feature_category_for_action(path_for_app(app)) + end + + def path_for_app(app) + normalize_path(app.namespace, app.options[:path].first) + end + + def route(methods, paths = ['/'], route_options = {}, &block) + if category = route_options.delete(:feature_category) + feature_category(category, Array(paths).map { |path| normalize_path(namespace, path) }) + end + + super + end + + private + + def normalize_path(namespace, path) + [namespace.presence, path.to_s.chomp('/').presence].compact.join('/') + end + end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index d2d1628aff4..f4b23c507f4 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -9,6 +9,8 @@ module API before { authenticate! } + feature_category :boards + helpers do def board_parent user_project @@ -40,6 +42,43 @@ module API authorize!(:read_board, user_project) present board, with: Entities::Board end + + desc 'Create a project board' do + detail 'This feature was introduced in 10.4' + success Entities::Board + end + params do + requires :name, type: String, desc: 'The board name' + end + post '/' do + authorize!(:admin_board, board_parent) + + create_board + end + + desc 'Update a project board' do + detail 'This feature was introduced in 11.0' + success Entities::Board + end + params do + use :update_params + end + put '/:board_id' do + authorize!(:admin_board, board_parent) + + update_board + end + + desc 'Delete a project board' do + detail 'This feature was introduced in 10.4' + success Entities::Board + end + + delete '/:board_id' do + authorize!(:admin_board, board_parent) + + delete_board + end end params do diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb index 6a86c02bf4a..2ae82f78e01 100644 --- a/lib/api/boards_responses.rb +++ b/lib/api/boards_responses.rb @@ -10,6 +10,35 @@ module API board_parent.boards.find(params[:board_id]) end + def create_board + forbidden! unless board_parent.multiple_issue_boards_available? + + response = + ::Boards::CreateService.new(board_parent, current_user, { name: params[:name] }).execute + + present response.payload, with: Entities::Board + end + + def update_board + service = ::Boards::UpdateService.new(board_parent, current_user, declared_params(include_missing: false)) + service.execute(board) + + if board.valid? + present board, with: Entities::Board + else + bad_request!("Failed to save board #{board.errors.messages}") + end + end + + def delete_board + forbidden! unless board_parent.multiple_issue_boards_available? + + destroy_conditionally!(board) do |board| + service = ::Boards::DestroyService.new(board_parent, current_user) + service.execute(board) + end + end + def board_lists board.destroyable_lists end @@ -62,6 +91,12 @@ module API params :list_creation_params do requires :label_id, type: Integer, desc: 'The ID of an existing label' end + + params :update_params do + # Configurable issue boards are not available in CE/EE Core. + # https://docs.gitlab.com/ee/user/project/issue_board.html#configurable-issue-boards + optional :name, type: String, desc: 'The board name' + end end end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 37cce6eafba..6842e93a4de 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -10,6 +10,8 @@ module API after_validation { content_type "application/json" } + feature_category :source_code_management + before do require_repository_enabled! authorize! :download_code, user_project diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 8ce7694bbfd..0762c276aad 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -4,6 +4,8 @@ module API class BroadcastMessages < ::API::Base include PaginationParams + feature_category :navigation + resource :broadcast_messages do helpers do def find_message diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb index b669acf668c..8a9ba2cbe0f 100644 --- a/lib/api/ci/pipeline_schedules.rb +++ b/lib/api/ci/pipeline_schedules.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :continuous_integration + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 61e03ed1a95..1b36e27f6c9 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -7,6 +7,8 @@ module API before { authenticate_non_get! } + feature_category :continuous_integration + params do requires :id, type: String, desc: 'The project ID' end @@ -128,15 +130,9 @@ module API pipeline = user_project.all_pipelines.find(params[:pipeline_id]) - if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) - builds = ::Ci::JobsFinder - .new(current_user: current_user, pipeline: pipeline, params: params) - .execute - else - authorize!(:read_build, pipeline) - builds = pipeline.builds - builds = filter_builds(builds, params[:scope]) - end + builds = ::Ci::JobsFinder + .new(current_user: current_user, pipeline: pipeline, params: params) + .execute builds = builds.with_preloads @@ -157,16 +153,9 @@ module API pipeline = user_project.all_pipelines.find(params[:pipeline_id]) - if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) - bridges = ::Ci::JobsFinder - .new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge) - .execute - else - authorize!(:read_pipeline, pipeline) - bridges = pipeline.bridges - bridges = filter_builds(bridges, params[:scope]) - end - + bridges = ::Ci::JobsFinder + .new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge) + .execute bridges = bridges.with_preloads present paginate(bridges), with: Entities::Ci::Bridge @@ -246,21 +235,6 @@ module API end helpers do - # NOTE: This method should be removed once the ci_jobs_finder_refactor FF is - # removed. https://gitlab.com/gitlab-org/gitlab/-/issues/245183 - # rubocop: disable CodeReuse/ActiveRecord - def filter_builds(builds, scope) - return builds if scope.nil? || scope.empty? - - available_statuses = ::CommitStatus::AVAILABLE_STATUSES - - unknown = scope - available_statuses - render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? - - builds.where(status: scope) - end - # rubocop: enable CodeReuse/ActiveRecord - def pipeline strong_memoize(:pipeline) do user_project.all_pipelines.find(params[:pipeline_id]) diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index ef679147c9f..85232b4ae1b 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -5,6 +5,10 @@ module API class Runner < ::API::Base helpers ::API::Helpers::Runner + content_type :txt, 'text/plain' + + feature_category :continuous_integration + resource :runners do desc 'Registers a new Runner' do success Entities::RunnerRegistrationDetails @@ -203,27 +207,18 @@ module API error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] - content_range = content_range.split('-') - - # TODO: - # it seems that `Content-Range` as formatted by runner is wrong, - # the `byte_end` should point to final byte, but it points byte+1 - # that means that we have to calculate end of body, - # as we cannot use `content_length[1]` - # Issue: https://gitlab.com/gitlab-org/gitlab-runner/issues/3275 - - body_data = request.body.read - body_start = content_range[0].to_i - body_end = body_start + body_data.bytesize - - stream_size = job.trace.append(body_data, body_start) - unless stream_size == body_end - break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{stream_size}" }) + + result = ::Ci::AppendBuildTraceService + .new(job, content_range: content_range) + .execute(request.body.read) + + if result.status == 416 + break error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{result.stream_size}" }) end - status 202 + status result.status header 'Job-Status', job.status - header 'Range', "0-#{stream_size}" + header 'Range', "0-#{result.stream_size}" header 'X-GitLab-Trace-Update-Interval', job.trace.update_interval.to_s end diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb index d37f10fe631..44ffc941cfa 100644 --- a/lib/api/ci/runners.rb +++ b/lib/api/ci/runners.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :continuous_integration + resource :runners do desc 'Get runners available for user' do success Entities::Runner diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index af103b8c1f8..26af921432c 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -4,6 +4,8 @@ require 'mime/types' module API class CommitStatuses < ::API::Base + feature_category :continuous_integration + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 582ccd41847..a24848082a9 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -6,6 +6,8 @@ module API class Commits < ::API::Base include PaginationParams + feature_category :source_code_management + before do require_repository_enabled! authorize! :download_code, user_project diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 1becbd668a3..0ac5cc45ccf 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -9,6 +9,8 @@ module API include ::API::Helpers::Packages::BasicAuthHelpers::Constants include ::Gitlab::Utils::StrongMemoize + feature_category :package_registry + content_type :json, 'application/json' default_format :json diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb index 9b6867a328b..188a42f26f8 100644 --- a/lib/api/conan_package_endpoints.rb +++ b/lib/api/conan_package_endpoints.rb @@ -29,6 +29,8 @@ module API CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze included do + feature_category :package_registry + helpers ::API::Helpers::PackagesManagerClientsHelpers helpers ::API::Helpers::Packages::Conan::ApiHelpers helpers ::API::Helpers::RelatedResourcesHelpers diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb new file mode 100644 index 00000000000..a91db93b182 --- /dev/null +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# NPM Package Manager Client API +# +# These API endpoints are not consumed directly by users, so there is no documentation for the +# individual endpoints. They are called by the NPM package manager client when users run commands +# like `npm install` or `npm publish`. The usage of the GitLab NPM registry is documented here: +# https://docs.gitlab.com/ee/user/packages/npm_registry/ +# +# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 +# +# Caution: This Concern has to be included at the end of the API class +# The last route of this Concern has a globbing wildcard that will match all urls. +# As such, routes declared after the last route of this Concern will not match any url. +module API + module Concerns + module Packages + module NpmEndpoints + extend ActiveSupport::Concern + + included do + helpers ::API::Helpers::Packages::DependencyProxyHelpers + + before do + require_packages_enabled! + authenticate_non_get! + end + + params do + requires :package_name, type: String, desc: 'Package name' + end + namespace '-/package/*package_name' do + desc 'Get all tags for a given an NPM package' do + detail 'This feature was introduced in GitLab 12.7' + success ::API::Entities::NpmPackageTag + end + get 'dist-tags', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do + package_name = params[:package_name] + + bad_request!('Package Name') if package_name.blank? + + authorize_read_package!(project) + + packages = ::Packages::Npm::PackageFinder.new(project, package_name) + .execute + + not_found! if packages.empty? + + present ::Packages::Npm::PackagePresenter.new(package_name, packages), + with: ::API::Entities::NpmPackageTag + end + + params do + requires :tag, type: String, desc: "Package dist-tag" + end + namespace 'dist-tags/:tag', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do + desc 'Create or Update the given tag for the given NPM package and version' do + detail 'This feature was introduced in GitLab 12.7' + end + put format: false do + package_name = params[:package_name] + version = env['api.request.body'] + tag = params[:tag] + + bad_request!('Package Name') if package_name.blank? + bad_request!('Version') if version.blank? + bad_request!('Tag') if tag.blank? + + authorize_create_package!(project) + + package = ::Packages::Npm::PackageFinder + .new(project, package_name) + .find_by_version(version) + not_found!('Package') unless package + + ::Packages::Npm::CreateTagService.new(package, tag).execute + + no_content! + end + + desc 'Deletes the given tag' do + detail 'This feature was introduced in GitLab 12.7' + end + delete format: false do + package_name = params[:package_name] + tag = params[:tag] + + bad_request!('Package Name') if package_name.blank? + bad_request!('Tag') if tag.blank? + + authorize_destroy_package!(project) + + package_tag = ::Packages::TagsFinder + .new(project, package_name, package_type: :npm) + .find_by_name(tag) + + not_found!('Package tag') unless package_tag + + ::Packages::RemoveTagService.new(package_tag).execute + + no_content! + end + end + end + + desc 'NPM registry metadata endpoint' do + detail 'This feature was introduced in GitLab 11.8' + end + params do + requires :package_name, type: String, desc: 'Package name' + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do + package_name = params[:package_name] + + packages = ::Packages::Npm::PackageFinder.new(project_or_nil, package_name) + .execute + + redirect_request = project_or_nil.blank? || packages.empty? + + redirect_registry_request(redirect_request, :npm, package_name: package_name) do + authorize_read_package!(project) + + not_found!('Packages') if packages.empty? + + present ::Packages::Npm::PackagePresenter.new(package_name, packages), + with: ::API::Entities::NpmPackage + end + end + end + end + end + end +end diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb index 6c4b80b612a..9bad31f6661 100644 --- a/lib/api/container_registry_event.rb +++ b/lib/api/container_registry_event.rb @@ -4,6 +4,8 @@ module API class ContainerRegistryEvent < ::API::Base DOCKER_DISTRIBUTION_EVENTS_V1_JSON = 'application/vnd.docker.distribution.events.v1+json' + feature_category :package_registry + before { authenticate_registry_notification! } resource :container_registry_event do diff --git a/lib/api/container_repositories.rb b/lib/api/container_repositories.rb new file mode 100644 index 00000000000..c84527f26e7 --- /dev/null +++ b/lib/api/container_repositories.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module API + class ContainerRepositories < ::API::Base + include Gitlab::Utils::StrongMemoize + helpers ::API::Helpers::PackagesHelpers + + before { authenticate! } + + feature_category :container_registry + + namespace 'registry' do + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :repositories, requirements: { id: /[0-9]*/ } do + desc 'Get a container repository' do + detail 'This feature was introduced in GitLab 13.6.' + success Entities::ContainerRegistry::Repository + end + params do + optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included' + optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included' + end + get ':id' do + authorize!(:read_container_image, repository) + + present repository, with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count], user: current_user + end + end + end + + helpers do + def repository + strong_memoize(:repository) do + ContainerRepository.find(params[:id]) + end + end + end + end +end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb index 168b3ca7a4f..c95c75b7e5c 100644 --- a/lib/api/debian_package_endpoints.rb +++ b/lib/api/debian_package_endpoints.rb @@ -26,6 +26,8 @@ module API }.freeze included do + feature_category :package_registry + helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers diff --git a/lib/api/dependency_proxy.rb b/lib/api/dependency_proxy.rb new file mode 100644 index 00000000000..3379bb2f029 --- /dev/null +++ b/lib/api/dependency_proxy.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module API + class DependencyProxy < ::API::Base + helpers ::API::Helpers::PackagesHelpers + + feature_category :dependency_proxy + + helpers do + def obtain_new_purge_cache_lease + Gitlab::ExclusiveLease + .new("dependency_proxy:delete_group_blobs:#{user_group.id}", + timeout: 1.hour) + .try_obtain + end + end + + before do + authorize! :admin_group, user_group + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Deletes all dependency_proxy_blobs for a group' do + detail 'This feature was introduced in GitLab 12.10' + end + delete ':id/dependency_proxy/cache' do + not_found! unless user_group.dependency_proxy_feature_available? + + message = 'This request has already been made. It may take some time to purge the cache. You can run this at most once an hour for a given group' + render_api_error!(message, 409) unless obtain_new_purge_cache_lease + + # rubocop:disable CodeReuse/Worker + PurgeDependencyProxyCacheWorker.perform_async(current_user.id, user_group.id) + # rubocop:enable CodeReuse/Worker + end + end + end +end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 314f5b6ee1d..0a541620c3a 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :continuous_delivery + helpers do def add_deploy_keys_project(project, attrs = {}) project.deploy_keys_projects.create(attrs) diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 1c156b8b3bb..5fab590eb4e 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -4,6 +4,8 @@ module API class DeployTokens < ::API::Base include PaginationParams + feature_category :continuous_delivery + helpers do def scope_params scopes = params.delete(:scopes) diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index ff06bdbae16..5346fcf03c9 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :continuous_delivery + params do requires :id, type: String, desc: 'The project ID' end diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb index 3d2608c8c5a..4c4ec200060 100644 --- a/lib/api/discussions.rb +++ b/lib/api/discussions.rb @@ -8,7 +8,7 @@ module API before { authenticate! } - Helpers::DiscussionsHelpers.noteable_types.each do |noteable_type| + Helpers::DiscussionsHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category| parent_type = noteable_type.parent_class.to_s.underscore noteables_str = noteable_type.to_s.underscore.pluralize noteables_path = noteable_type == Commit ? "repository/#{noteables_str}" : noteables_str @@ -25,7 +25,7 @@ module API use :pagination end - get ":id/#{noteables_path}/:noteable_id/discussions" do + get ":id/#{noteables_path}/:noteable_id/discussions", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) discussion_ids = paginate(noteable.discussion_ids_relation) @@ -41,7 +41,7 @@ module API requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' end - get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do + get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) @@ -91,7 +91,7 @@ module API end end end - post ":id/#{noteables_path}/:noteable_id/discussions" do + post ":id/#{noteables_path}/:noteable_id/discussions", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) type = params[:position] ? 'DiffNote' : 'DiscussionNote' id_key = noteable.is_a?(Commit) ? :commit_id : :noteable_id @@ -121,7 +121,7 @@ module API requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :noteable_id, types: [Integer, String], desc: 'The ID of the noteable' end - get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do + get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) @@ -141,7 +141,7 @@ module API requires :body, type: String, desc: 'The content of a note' optional :created_at, type: String, desc: 'The creation date of the note' end - post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes" do + post ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) notes = readable_discussion_notes(noteable, params[:discussion_id]) first_note = notes.first @@ -175,7 +175,7 @@ module API requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' end - get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + get ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) get_note(noteable, params[:note_id]) @@ -192,7 +192,7 @@ module API optional :resolved, type: Boolean, desc: 'Mark note resolved/unresolved' exactly_one_of :body, :resolved end - put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) if params[:resolved].nil? @@ -210,7 +210,7 @@ module API requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :note_id, type: Integer, desc: 'The ID of a note' end - delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id" do + delete ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id/notes/:note_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) delete_note(noteable, params[:note_id]) @@ -225,7 +225,7 @@ module API requires :discussion_id, type: String, desc: 'The ID of a discussion' requires :resolved, type: Boolean, desc: 'Mark discussion resolved/unresolved' end - put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id" do + put ":id/#{noteables_path}/:noteable_id/discussions/:discussion_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) resolve_discussion(noteable, params[:discussion_id], params[:resolved]) diff --git a/lib/api/entities/board.rb b/lib/api/entities/board.rb index 5bb1cde0fa9..b7a50408313 100644 --- a/lib/api/entities/board.rb +++ b/lib/api/entities/board.rb @@ -4,6 +4,7 @@ module API module Entities class Board < Grape::Entity expose :id + expose :name expose :project, using: Entities::BasicProjectDetails expose :lists, using: Entities::List do |board| diff --git a/lib/api/entities/commit_signature.rb b/lib/api/entities/commit_signature.rb index b5232273521..505ce462edf 100644 --- a/lib/api/entities/commit_signature.rb +++ b/lib/api/entities/commit_signature.rb @@ -4,13 +4,28 @@ module API module Entities class CommitSignature < Grape::Entity expose :signature_type + expose :signature, merge: true do |commit, options| - if commit.signature.is_a?(GpgSignature) - ::API::Entities::GpgCommitSignature.represent commit.signature, options + if commit.signature.is_a?(GpgSignature) || commit.raw_commit_from_rugged? + ::API::Entities::GpgCommitSignature.represent commit_signature(commit), options elsif commit.signature.is_a?(X509CommitSignature) ::API::Entities::X509Signature.represent commit.signature, options end end + + expose :commit_source do |commit, _| + commit.raw_commit_from_rugged? ? "rugged" : "gitaly" + end + + private + + def commit_signature(commit) + if commit.raw_commit_from_rugged? + commit.gpg_commit.signature + else + commit.signature + end + end end end end diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index c430b73580b..c9c2c5156cc 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -10,6 +10,8 @@ module API end class Repository < Grape::Entity + include ::API::Helpers::RelatedResourcesHelpers + expose :id expose :name expose :path @@ -19,6 +21,13 @@ module API expose :expiration_policy_started_at, as: :cleanup_policy_started_at expose :tags_count, if: -> (_, options) { options[:tags_count] } expose :tags, using: Tag, if: -> (_, options) { options[:tags] } + expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) } + + private + + def delete_api_path + expose_url api_v4_projects_registry_repositories_path(repository_id: object.id, id: object.project_id) + end end class TagDetails < Tag diff --git a/lib/api/entities/invitation.rb b/lib/api/entities/invitation.rb new file mode 100644 index 00000000000..342f4804cf3 --- /dev/null +++ b/lib/api/entities/invitation.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Entities + class Invitation < Grape::Entity + expose :access_level + expose :requested_at + expose :expires_at + expose :invite_email + expose :invite_token + expose :user_name, if: -> (member, _) { member.user.present? } + expose :created_by_name + end + end +end diff --git a/lib/api/entities/merge_request_changes.rb b/lib/api/entities/merge_request_changes.rb index a835d119736..488f33dfb93 100644 --- a/lib/api/entities/merge_request_changes.rb +++ b/lib/api/entities/merge_request_changes.rb @@ -4,7 +4,27 @@ module API module Entities class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::Diff do |compare, _| - compare.raw_diffs(limits: false).to_a + Array(diff_collection(compare)) + end + + expose :overflow?, as: :overflow + + private + + def overflow? + expose_raw_diffs? ? false : diff_collection(object).overflow? + end + + def diff_collection(compare) + @diffs ||= if expose_raw_diffs? + compare.raw_diffs(limits: false) + else + compare.diffs.diffs + end + end + + def expose_raw_diffs? + options[:access_raw_diffs] || ::Feature.enabled?(:mrc_api_use_raw_diffs_from_gitaly, options[:project]) end end end diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb index b54f0e04a9d..e7153f9bebb 100644 --- a/lib/api/entities/package.rb +++ b/lib/api/entities/package.rb @@ -38,7 +38,8 @@ module API expose :project_path, if: ->(obj, opts) { opts[:group] && Ability.allowed?(opts[:user], :read_project, obj.project) } expose :tags - expose :pipeline, if: ->(package) { package.build_info }, using: Package::Pipeline + expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline + expose :pipelines, if: ->(package) { package.pipelines.present? }, using: Package::Pipeline expose :versions, using: ::API::Entities::PackageVersion, unless: ->(_, opts) { opts[:collection] } diff --git a/lib/api/entities/package_file.rb b/lib/api/entities/package_file.rb index 8be4e5a4316..2cc2f62a948 100644 --- a/lib/api/entities/package_file.rb +++ b/lib/api/entities/package_file.rb @@ -6,6 +6,7 @@ module API expose :id, :package_id, :created_at expose :file_name, :size expose :file_md5, :file_sha1 + expose :pipelines, if: ->(package_file) { package_file.pipelines.present? }, using: Package::Pipeline end end end diff --git a/lib/api/entities/package_version.rb b/lib/api/entities/package_version.rb index 5f3e86c3229..82522d3f423 100644 --- a/lib/api/entities/package_version.rb +++ b/lib/api/entities/package_version.rb @@ -8,7 +8,7 @@ module API expose :created_at expose :tags - expose :pipeline, if: ->(package) { package.build_info }, using: Package::Pipeline + expose :pipeline, if: ->(package) { package.original_build_info }, using: Package::Pipeline end end end diff --git a/lib/api/entities/project_hook.rb b/lib/api/entities/project_hook.rb index 751f9500252..6c71e5d317c 100644 --- a/lib/api/entities/project_hook.rb +++ b/lib/api/entities/project_hook.rb @@ -5,7 +5,7 @@ module API class ProjectHook < Hook expose :project_id, :issues_events, :confidential_issues_events expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events, :deployment_events - expose :job_events + expose :job_events, :releases_events expose :push_events_branch_filter end end diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb index afe14cf33cf..44a46c5861e 100644 --- a/lib/api/entities/release.rb +++ b/lib/api/entities/release.rb @@ -30,8 +30,6 @@ module API expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_download_code? } expose :_links do expose :self_url, as: :self, expose_nil: false - expose :merge_requests_url, expose_nil: false - expose :issues_url, expose_nil: false expose :edit_url, expose_nil: false end diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb index 654df2e2caf..c1d83a8924f 100644 --- a/lib/api/entities/releases/link.rb +++ b/lib/api/entities/releases/link.rb @@ -14,10 +14,8 @@ module API def direct_asset_url return object.url unless object.filepath - release = object.release - project = release.project - - Gitlab::Routing.url_helpers.project_release_url(project, release) << object.filepath + release = object.release.present + release.download_url(object.filepath) end end end diff --git a/lib/api/entities/user_status.rb b/lib/api/entities/user_status.rb index 9bc4cbf240f..1d5cc27e5ef 100644 --- a/lib/api/entities/user_status.rb +++ b/lib/api/entities/user_status.rb @@ -5,6 +5,7 @@ module API class UserStatus < Grape::Entity expose :emoji expose :message + expose :availability expose :message_html do |entity| MarkupHelper.markdown_field(entity, :message) end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 0e780d4ef36..5dd2fa22690 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :continuous_delivery + params do requires :id, type: String, desc: 'The project ID' end diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb index 03f83477954..0e44c8b1081 100644 --- a/lib/api/error_tracking.rb +++ b/lib/api/error_tracking.rb @@ -4,6 +4,8 @@ module API class ErrorTracking < ::API::Base before { authenticate! } + feature_category :error_tracking + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/events.rb b/lib/api/events.rb index 43efacf9c0b..233c62b5389 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -8,6 +8,8 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? } + feature_category :users + resource :events do desc "List currently authenticated user's events" do detail 'This feature was introduced in GitLab 9.3.' diff --git a/lib/api/feature_flag_scopes.rb b/lib/api/feature_flag_scopes.rb index d77e243aa88..3f3bf4d9f42 100644 --- a/lib/api/feature_flag_scopes.rb +++ b/lib/api/feature_flag_scopes.rb @@ -7,6 +7,8 @@ module API ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS = FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS .merge(environment_scope: API::NO_SLASH_URL_PART_REGEX) + feature_category :feature_flags + before do authorize_read_feature_flags! end diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb index 613c3fb0f5b..67168ba9be6 100644 --- a/lib/api/feature_flags.rb +++ b/lib/api/feature_flags.rb @@ -7,6 +7,8 @@ module API FEATURE_FLAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS .merge(name: API::NO_SLASH_URL_PART_REGEX) + feature_category :feature_flags + before do authorize_read_feature_flags! end diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb index e5218cfd7f1..086bcbcdc89 100644 --- a/lib/api/feature_flags_user_lists.rb +++ b/lib/api/feature_flags_user_lists.rb @@ -8,6 +8,8 @@ module API message.is_a?(String) ? { message: message }.to_json : message.to_json } + feature_category :feature_flags + before do authorize_admin_feature_flags_user_lists! end @@ -22,10 +24,13 @@ module API success ::API::Entities::FeatureFlag::UserList end params do + optional :search, type: String, desc: 'Returns the list of user lists matching the search critiera' + use :pagination end get do - present paginate(user_project.operations_feature_flags_user_lists), + user_lists = ::FeatureFlagsUserListsFinder.new(user_project, current_user, params).execute + present paginate(user_lists), with: ::API::Entities::FeatureFlag::UserList end diff --git a/lib/api/features.rb b/lib/api/features.rb index 5d2e545abd6..2c2e3e3d0c9 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -4,6 +4,8 @@ module API class Features < ::API::Base before { authenticated_as_admin! } + feature_category :feature_flags + helpers do def gate_value(params) case params[:value] @@ -61,6 +63,8 @@ module API mutually_exclusive :key, :project end post ':name' do + validate_feature_flag_name!(params[:name]) + feature = Feature.get(params[:name]) # rubocop:disable Gitlab/AvoidFeatureGet targets = gate_targets(params) value = gate_value(params) @@ -97,5 +101,13 @@ module API no_content! end end + + helpers do + def validate_feature_flag_name!(name) + # no-op + end + end end end + +API::Features.prepend_if_ee('EE::API::Features') diff --git a/lib/api/files.rb b/lib/api/files.rb index 6833fc429e2..cb73bde73f5 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -9,6 +9,8 @@ module API # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } + feature_category :source_code_management + helpers ::API::Helpers::HeadersHelpers helpers do diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb index a83e36165a2..d001ced8581 100644 --- a/lib/api/freeze_periods.rb +++ b/lib/api/freeze_periods.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :continuous_delivery + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index a0c33ab65b9..3e1dd044c8d 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -7,6 +7,8 @@ module API file_name: API::NO_SLASH_URL_PART_REGEX }.freeze + feature_category :package_registry + before do require_packages_enabled! authenticate! diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index 30f0cfb4dfd..8fb4c561c40 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -4,11 +4,15 @@ module API helpers Gitlab::Golang helpers ::API::Helpers::PackagesHelpers + feature_category :package_registry + # basic semver, except case encoded (A => !a) MODULE_VERSION_REGEX = /v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([-.!a-z0-9]+))?(?:\+([-.!a-z0-9]+))?/.freeze MODULE_VERSION_REQUIREMENTS = { module_version: MODULE_VERSION_REGEX }.freeze + content_type :txt, 'text/plain' + before { require_packages_enabled! } helpers do diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index d4574b22d99..ac5a1a2ce94 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -7,6 +7,8 @@ module API prepend_if_ee('EE::API::BoardsResponses') # rubocop: disable Cop/InjectEnterpriseEditionModule + feature_category :boards + before do authenticate! end diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index 75429cf7a5c..a435b050042 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :kubernetes_management + params do requires :id, type: String, desc: 'The ID of the group' end diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index 1bb26b3931c..4fede0ad583 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -8,6 +8,8 @@ module API before { authorize_read_group_container_images! } + feature_category :package_registry + REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( tag_name: API::NO_SLASH_URL_PART_REGEX) diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 6ebaa8de185..29ffbea687a 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -10,6 +10,8 @@ module API authorize! :admin_group, user_group end + feature_category :importers + params do requires :id, type: String, desc: 'The ID of a group' end diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb index e703a217fd5..4a752732652 100644 --- a/lib/api/group_import.rb +++ b/lib/api/group_import.rb @@ -2,6 +2,8 @@ module API class GroupImport < ::API::Base + feature_category :importers + helpers Helpers::FileUploadHelpers helpers do diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 8443ddf10ce..bf3ac8800b7 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :issue_tracking + params do requires :id, type: String, desc: 'The ID of a group' end @@ -20,10 +22,16 @@ module API desc: 'Include issue and merge request counts' optional :include_ancestor_groups, type: Boolean, default: true, desc: 'Include ancestor groups' + optional :include_descendant_groups, type: Boolean, default: false, + desc: 'Include descendant groups. This feature was added in GitLab 13.6' + optional :only_group_labels, type: Boolean, default: true, + desc: 'Toggle to include only group labels or also project labels. This feature was added in GitLab 13.6' + optional :search, type: String, + desc: 'Keyword to filter labels by. This feature was added in GitLab 13.6' use :pagination end get ':id/labels' do - get_labels(user_group, Entities::GroupLabel, include_ancestor_groups: params[:include_ancestor_groups]) + get_labels(user_group, Entities::GroupLabel, declared_params) end desc 'Get a single label' do @@ -33,9 +41,13 @@ module API params do optional :include_ancestor_groups, type: Boolean, default: true, desc: 'Include ancestor groups' + optional :include_descendant_groups, type: Boolean, default: false, + desc: 'Include descendant groups. This feature was added in GitLab 13.6' + optional :only_group_labels, type: Boolean, default: true, + desc: 'Toggle to include only group labels or also project labels. This feature was added in GitLab 13.6' end get ':id/labels/:name' do - get_label(user_group, Entities::GroupLabel, include_ancestor_groups: params[:include_ancestor_groups]) + get_label(user_group, Entities::GroupLabel, declared_params) end desc 'Create a new label' do diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index aef9877b84c..dfffd3b1209 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :issue_tracking + params do requires :id, type: String, desc: 'The ID of a group' end diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index 5b6290df0dd..31b28c3990f 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -8,6 +8,8 @@ module API authorize_packages_access!(user_group) end + feature_category :package_registry + helpers ::API::Helpers::PackagesHelpers params do diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index ee110d67fa5..0c40db02eb5 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -7,6 +7,8 @@ module API before { authenticate! } before { authorize! :admin_build, user_group } + feature_category :continuous_integration + params do requires :id, type: String, desc: 'The ID of a group' end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index bf3d6c3c7e0..a8b1cdab021 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,6 +7,8 @@ module API before { authenticate_non_get! } + feature_category :subgroups + helpers Helpers::GroupsHelpers helpers do @@ -46,7 +48,7 @@ module API find_params.fetch(:all_available, current_user&.can_read_all_resources?) groups = GroupsFinder.new(current_user, find_params).execute - groups = groups.search(params[:search]) if params[:search].present? + groups = groups.search(params[:search], include_parents: true) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? order_options = { params[:order_by] => params[:sort] } order_options["id"] ||= "asc" diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index c8aee1f3479..147d8407142 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -89,16 +89,15 @@ module API @project ||= find_project!(params[:id]) end - def available_labels_for(label_parent, include_ancestor_groups: true) - search_params = { include_ancestor_groups: include_ancestor_groups } - + def available_labels_for(label_parent, params = { include_ancestor_groups: true, only_group_labels: true }) if label_parent.is_a?(Project) - search_params[:project_id] = label_parent.id + params.delete(:only_group_labels) + params[:project_id] = label_parent.id else - search_params.merge!(group_id: label_parent.id, only_group_labels: true) + params[:group_id] = label_parent.id end - LabelsFinder.new(current_user, search_params).execute + LabelsFinder.new(current_user, params).execute end def find_user(id) @@ -388,8 +387,8 @@ module API render_api_error!('401 Unauthorized', 401) end - def not_allowed! - render_api_error!('405 Method Not Allowed', 405) + def not_allowed!(message = nil) + render_api_error!(message || '405 Method Not Allowed', :method_not_allowed) end def not_acceptable! diff --git a/lib/api/helpers/discussions_helpers.rb b/lib/api/helpers/discussions_helpers.rb index 799d5582b38..3c0db1d0ea9 100644 --- a/lib/api/helpers/discussions_helpers.rb +++ b/lib/api/helpers/discussions_helpers.rb @@ -3,10 +3,15 @@ module API module Helpers module DiscussionsHelpers - def self.noteable_types + def self.feature_category_per_noteable_type # This is a method instead of a constant, allowing EE to more easily # extend it. - [Issue, Snippet, MergeRequest, Commit] + { + Issue => :issue_tracking, + Snippet => :snippets, + MergeRequest => :code_review, + Commit => :code_review + } end end end diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index 638b31cc7ba..b303f1f845d 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -5,6 +5,9 @@ module API module IssuesHelpers extend Grape::API::Helpers + params :negatable_issue_filter_params_ee do + end + params :optional_issue_params_ee do end diff --git a/lib/api/helpers/label_helpers.rb b/lib/api/helpers/label_helpers.rb index 2fb2d9b79cf..4018f2dec21 100644 --- a/lib/api/helpers/label_helpers.rb +++ b/lib/api/helpers/label_helpers.rb @@ -28,23 +28,23 @@ module API at_least_one_of :new_name, :color, :description end - def find_label(parent, id_or_title, include_ancestor_groups: true) - labels = available_labels_for(parent, include_ancestor_groups: include_ancestor_groups) + def find_label(parent, id_or_title, params = { include_ancestor_groups: true }) + labels = available_labels_for(parent, params) label = labels.find_by_id(id_or_title) || labels.find_by_title(id_or_title) label || not_found!('Label') end - def get_labels(parent, entity, include_ancestor_groups: true) - present paginate(available_labels_for(parent, include_ancestor_groups: include_ancestor_groups)), + def get_labels(parent, entity, params = {}) + present paginate(available_labels_for(parent, params)), with: entity, current_user: current_user, parent: parent, with_counts: params[:with_counts] end - def get_label(parent, entity, include_ancestor_groups: true) - label = find_label(parent, params_id_or_title, include_ancestor_groups: include_ancestor_groups) + def get_label(parent, entity, params = {}) + label = find_label(parent, params_id_or_title, params) present label, with: entity, current_user: current_user, parent: parent end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 5cc435e6801..431001c227d 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -20,12 +20,23 @@ module API # rubocop: disable CodeReuse/ActiveRecord def retrieve_members(source, params:, deep: false) - members = deep ? find_all_members(source) : source.members.where.not(user_id: nil) + members = deep ? find_all_members(source) : source_members(source).where.not(user_id: nil) members = members.includes(:user) members = members.references(:user).merge(User.search(params[:query])) if params[:query].present? members = members.where(user_id: params[:user_ids]) if params[:user_ids].present? members end + + def retrieve_member_invitations(source, query = nil) + members = source_members(source).where.not(invite_token: nil) + members = members.includes(:user) + members = members.where(invite_email: query) if query.present? + members + end + + def source_members(source) + source.members + end # rubocop: enable CodeReuse/ActiveRecord def find_all_members(source) @@ -48,6 +59,10 @@ module API def present_members(members) present members, with: Entities::Member, current_user: current_user, show_seat_info: params[:show_seat_info] end + + def present_member_invitations(invitations) + present invitations, with: Entities::Invitation, current_user: current_user + end end end end diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index f61bcfe963e..6798c4d284b 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -5,10 +5,12 @@ module API module NotesHelpers include ::RendersNotes - def self.noteable_types - # This is a method instead of a constant, allowing EE to more easily - # extend it. - [Issue, MergeRequest, Snippet] + def self.feature_category_per_noteable_type + { + Issue => :issue_tracking, + MergeRequest => :code_review, + Snippet => :snippets + } end def update_note(noteable, note_id) diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb new file mode 100644 index 00000000000..c1f6a001201 --- /dev/null +++ b/lib/api/helpers/packages/npm.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module API + module Helpers + module Packages + module Npm + include Gitlab::Utils::StrongMemoize + include ::API::Helpers::PackagesHelpers + + NPM_ENDPOINT_REQUIREMENTS = { + package_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + def endpoint_scope + params[:id].present? ? :project : :instance + end + + def project + strong_memoize(:project) do + case endpoint_scope + when :project + user_project + when :instance + # Simulate the same behavior as #user_project by re-using #find_project! + # but take care if the project_id is nil as #find_project! is not designed + # to handle it. + project_id = project_id_or_nil + + not_found!('Project') unless project_id + + find_project!(project_id) + end + end + end + + def project_or_nil + # mainly used by the metadata endpoint where we need to get a project + # and return nil if not found (no errors should be raised) + strong_memoize(:project_or_nil) do + next unless project_id_or_nil + + find_project(project_id_or_nil) + end + end + + def project_id_or_nil + strong_memoize(:project_id_or_nil) do + case endpoint_scope + when :project + params[:id] + when :instance + ::Packages::Package.npm + .with_name(params[:package_name]) + .first + &.project_id + end + end + end + end + end + end +end diff --git a/lib/api/helpers/resource_label_events_helpers.rb b/lib/api/helpers/resource_label_events_helpers.rb index 423bd4e704b..ad2733baffc 100644 --- a/lib/api/helpers/resource_label_events_helpers.rb +++ b/lib/api/helpers/resource_label_events_helpers.rb @@ -3,10 +3,13 @@ module API module Helpers module ResourceLabelEventsHelpers - def self.eventable_types + def self.feature_category_per_eventable_type # This is a method instead of a constant, allowing EE to more easily # extend it. - [Issue, MergeRequest] + { + Issue => :issue_tracking, + MergeRequest => :code_review + } end end end diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb index a0238c24f3b..ecd78c6e6db 100644 --- a/lib/api/import_bitbucket_server.rb +++ b/lib/api/import_bitbucket_server.rb @@ -2,6 +2,8 @@ module API class ImportBitbucketServer < ::API::Base + feature_category :importers + helpers do def client @client ||= BitbucketServer::Client.new(credentials) diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 61fce7a2c1b..c91a7700f58 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -2,6 +2,8 @@ module API class ImportGithub < ::API::Base + feature_category :importers + rescue_from Octokit::Unauthorized, with: :provider_unauthorized before do @@ -11,7 +13,7 @@ module API helpers do def client @client ||= if Feature.enabled?(:remove_legacy_github_client) - Gitlab::GithubImport::Client.new(params[:personal_access_token]) + Gitlab::GithubImport::Client.new(params[:personal_access_token], host: params[:github_hostname]) else Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options) end @@ -22,7 +24,7 @@ module API end def client_options - {} + { host: params[:github_hostname] } end def provider @@ -43,6 +45,7 @@ module API requires :repo_id, type: Integer, desc: 'GitHub repository ID' optional :new_name, type: String, desc: 'New repo name' requires :target_namespace, type: String, desc: 'Namespace to import repo into' + optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname' end post 'import/github' do result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 6d8f13c36e6..61ef1d5bde0 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -7,10 +7,16 @@ module API before { authenticate_by_gitlab_shell_token! } before do + api_endpoint = env['api.endpoint'] + feature_category = api_endpoint.options[:for].try(:feature_category_for_app, api_endpoint).to_s + + header[Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER] = feature_category + Gitlab::ApplicationContext.push( user: -> { actor&.user }, project: -> { project }, - caller_id: route.origin + caller_id: route.origin, + feature_category: feature_category ) end @@ -28,10 +34,10 @@ module API { status: success, message: message }.merge(extra_options).compact end - def lfs_authentication_url(project) + def lfs_authentication_url(container) # This is a separate method so that EE can alter its behaviour more # easily. - project.http_url_to_repo + container.lfs_http_url_to_repo end def check_allowed(params) @@ -122,13 +128,15 @@ module API # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList # check_ip - optional, only in EE version, may limit access to # group resources based on its IP restrictions - post "/allowed" do + post "/allowed", feature_category: :source_code_management do # It was moved to a separate method so that EE can alter its behaviour more # easily. check_allowed(params) end - post "/lfs_authenticate" do + post "/lfs_authenticate", feature_category: :source_code_management do + not_found! unless container&.lfs_enabled? + status 200 unless actor.key_or_user @@ -139,14 +147,14 @@ module API Gitlab::LfsToken .new(actor.key_or_user) - .authentication_payload(lfs_authentication_url(project)) + .authentication_payload(lfs_authentication_url(container)) end # # Get a ssh key using the fingerprint # # rubocop: disable CodeReuse/ActiveRecord - get '/authorized_keys' do + get '/authorized_keys', feature_category: :source_code_management do fingerprint = params.fetch(:fingerprint) do Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint end @@ -159,11 +167,11 @@ module API # # Discover user by ssh key, user id or username # - get '/discover' do + get '/discover', feature_category: :authentication_and_authorization do present actor.user, with: Entities::UserSafe end - get '/check' do + get '/check', feature_category: :not_owned do { api_version: API.version, gitlab_version: Gitlab::VERSION, @@ -172,7 +180,7 @@ module API } end - post '/two_factor_recovery_codes' do + post '/two_factor_recovery_codes', feature_category: :authentication_and_authorization do status 200 actor.update_last_used_at! @@ -201,7 +209,7 @@ module API { success: true, recovery_codes: codes } end - post '/personal_access_token' do + post '/personal_access_token', feature_category: :authentication_and_authorization do status 200 actor.update_last_used_at! @@ -239,7 +247,7 @@ module API end result = ::PersonalAccessTokens::CreateService.new( - user, name: params[:name], scopes: params[:scopes], expires_at: expires_at + current_user: user, target_user: user, params: { name: params[:name], scopes: params[:scopes], expires_at: expires_at } ).execute unless result.status == :success @@ -251,7 +259,7 @@ module API { success: true, token: access_token.token, scopes: access_token.scopes, expires_at: access_token.expires_at } end - post '/pre_receive' do + post '/pre_receive', feature_category: :source_code_management do status 200 reference_counter_increased = Gitlab::ReferenceCounter.new(params[:gl_repository]).increase @@ -259,7 +267,7 @@ module API { reference_counter_increased: reference_counter_increased } end - post '/post_receive' do + post '/post_receive', feature_category: :source_code_management do status 200 response = PostReceiveService.new(actor.user, repository, project, params).execute @@ -267,7 +275,7 @@ module API present response, with: Entities::InternalPostReceive::Response end - post '/two_factor_config' do + post '/two_factor_config', feature_category: :authentication_and_authorization do status 200 break { success: false } unless Feature.enabled?(:two_factor_for_cli) @@ -289,7 +297,7 @@ module API end end - post '/two_factor_otp_check' do + post '/two_factor_otp_check', feature_category: :authentication_and_authorization do status 200 break { success: false } unless Feature.enabled?(:two_factor_for_cli) diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 90e224b2ccb..d4690709de4 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -4,6 +4,8 @@ module API # Kubernetes Internal API module Internal class Kubernetes < ::API::Base + feature_category :kubernetes_management + before do check_feature_enabled authenticate_gitlab_kas_request! diff --git a/lib/api/internal/lfs.rb b/lib/api/internal/lfs.rb index 630f0ec77a8..66baa4f1034 100644 --- a/lib/api/internal/lfs.rb +++ b/lib/api/internal/lfs.rb @@ -7,6 +7,8 @@ module API before { authenticate_by_gitlab_shell_token! } + feature_category :source_code_management + helpers do def find_lfs_object(lfs_oid) LfsObject.find_by_oid(lfs_oid) diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 51136144c19..690f52d89f3 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -4,6 +4,8 @@ module API # Pages Internal API module Internal class Pages < ::API::Base + feature_category :pages + before do authenticate_gitlab_pages_request! end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb new file mode 100644 index 00000000000..be8147908e9 --- /dev/null +++ b/lib/api/invitations.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module API + class Invitations < ::API::Base + include PaginationParams + + feature_category :users + + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Invite non-members by email address to a group or project.' do + detail 'This feature was introduced in GitLab 13.6' + success Entities::Invitation + end + params do + requires :email, types: [String, Array[String]], email_or_email_list: true, desc: 'The email address to invite, or multiple emails separated by comma' + requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end + post ":id/invitations" do + source = find_source(source_type, params[:id]) + + authorize_admin_source!(source_type, source) + + ::Members::InviteService.new(current_user, params).execute(source) + end + + desc 'Get a list of group or project invitations viewable by the authenticated user' do + detail 'This feature was introduced in GitLab 13.6' + success Entities::Invitation + end + params do + optional :query, type: String, desc: 'A query string to search for members' + use :pagination + end + get ":id/invitations" do + source = find_source(source_type, params[:id]) + query = params[:query] + + invitations = paginate(retrieve_member_invitations(source, query)) + + present_member_invitations invitations + end + end + end + end +end diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index db4979c9052..e938dbbae87 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :issue_tracking + params do requires :id, type: String, desc: 'The ID of a project' requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 143f9e40736..6a6ee7a4e1c 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -8,6 +8,8 @@ module API before { authenticate_non_get! } + feature_category :issue_tracking + helpers do params :negatable_issue_filter_params do optional :labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' @@ -26,6 +28,8 @@ module API coerce_with: Validations::Validators::CheckAssigneesCount.coerce, desc: 'Return issues which are assigned to the user with the given username' mutually_exclusive :assignee_id, :assignee_username + + use :negatable_issue_filter_params_ee end params :issues_stats_params do diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 536b361b308..1faa28d6f07 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -4,6 +4,8 @@ module API class JobArtifacts < ::API::Base before { authenticate_non_get! } + feature_category :continuous_integration + # EE::API::JobArtifacts would override the following helpers helpers do def authorize_download_artifacts! diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index bdb23b4a9be..51659c2e8a1 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :continuous_integration + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 2e4568029b5..fb1bedd5e92 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -5,6 +5,8 @@ module API class Keys < ::API::Base before { authenticate! } + feature_category :authentication_and_authorization + resource :keys do desc 'Get single ssh key by id. Only available to admin users' do success Entities::SSHKeyWithUser diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 0cc9f33bd07..a8fc277989e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :issue_tracking + params do requires :id, type: String, desc: 'The ID of a project' end @@ -19,10 +21,12 @@ module API desc: 'Include issue and merge request counts' optional :include_ancestor_groups, type: Boolean, default: true, desc: 'Include ancestor groups' + optional :search, type: String, + desc: 'Keyword to filter labels by. This feature was added in GitLab 13.6' use :pagination end get ':id/labels' do - get_labels(user_project, Entities::ProjectLabel, include_ancestor_groups: params[:include_ancestor_groups]) + get_labels(user_project, Entities::ProjectLabel, declared_params) end desc 'Get a single label' do @@ -34,7 +38,7 @@ module API desc: 'Include ancestor groups' end get ':id/labels/:name' do - get_label(user_project, Entities::ProjectLabel, include_ancestor_groups: params[:include_ancestor_groups]) + get_label(user_project, Entities::ProjectLabel, declared_params) end desc 'Create a new label' do diff --git a/lib/api/lint.rb b/lib/api/lint.rb index bfd152f70b1..58181adaa93 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -2,6 +2,8 @@ module API class Lint < ::API::Base + feature_category :pipeline_authoring + namespace :ci do desc 'Validation of .gitlab-ci.yml content' params do @@ -15,9 +17,9 @@ module API status 200 response = if error.blank? - { status: 'valid', errors: [] } + { status: 'valid', errors: [], warnings: result.warnings } else - { status: 'invalid', errors: [error] } + { status: 'invalid', errors: [error], warnings: result.warnings } end response.tap do |response| @@ -44,5 +46,25 @@ module API present result, with: Entities::Ci::Lint::Result, current_user: current_user end end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Validation of .gitlab-ci.yml content' do + detail 'This feature was introduced in GitLab 13.6.' + end + params do + requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.' + end + post ':id/ci/lint' do + authorize! :download_code, user_project + + result = Gitlab::Ci::Lint + .new(project: user_project, current_user: current_user) + .validate(params[:content], dry_run: params[:dry_run]) + + status 200 + present result, with: Entities::Ci::Lint::Result, current_user: current_user + end + end end end diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb index 97549abd273..de612ff8321 100644 --- a/lib/api/markdown.rb +++ b/lib/api/markdown.rb @@ -2,6 +2,8 @@ module API class Markdown < ::API::Base + feature_category :not_owned + params do requires :text, type: String, desc: "The markdown text to render" optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown" diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index a3e2fa84c32..7b4e52d18e8 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -5,6 +5,8 @@ module API file_name: API::NO_SLASH_URL_PART_REGEX }.freeze + feature_category :package_registry + content_type :md5, 'text/plain' content_type :sha1, 'text/plain' content_type :binary, 'application/octet-stream' @@ -244,7 +246,7 @@ module API file_md5: params['file.md5'] } - ::Packages::CreatePackageFileService.new(package, file_params).execute + ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)).execute end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index c28b3b1cc7c..803de51651a 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :authentication_and_authorization + helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| @@ -134,7 +136,7 @@ module API source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) - member = source.members.find_by!(user_id: params[:user_id]) + member = source_members(source).find_by!(user_id: params[:user_id]) updated_member = ::Members::UpdateService .new(current_user, declared_params(include_missing: false)) @@ -157,7 +159,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - member = source.members.find_by!(user_id: params[:user_id]) + member = source_members(source).find_by!(user_id: params[:user_id]) destroy_conditionally!(member) do ::Members::DestroyService.new(current_user).execute(member, unassign_issuables: params[:unassign_issuables]) diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb index 14d6e3995ea..27ef0b9c7cd 100644 --- a/lib/api/merge_request_approvals.rb +++ b/lib/api/merge_request_approvals.rb @@ -4,6 +4,8 @@ module API class MergeRequestApprovals < ::API::Base before { authenticate_non_get! } + feature_category :code_review + helpers do params :ee_approval_params do end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 22023888bbd..0ffb38438eb 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :code_review + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index b24dd870c8b..d17e451093b 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -8,6 +8,8 @@ module API before { authenticate_non_get! } + feature_category :code_review + helpers Helpers::MergeRequestsHelpers # EE::API::MergeRequests would override the following helpers @@ -350,7 +352,11 @@ module API get ':id/merge_requests/:merge_request_iid/changes' do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request, with: Entities::MergeRequestChanges, current_user: current_user, project: user_project + present merge_request, + with: Entities::MergeRequestChanges, + current_user: current_user, + project: user_project, + access_raw_diffs: params.fetch(:access_raw_diffs, false) end desc 'Get the merge request pipelines' do diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb index b6bc0af2202..0989340b3ea 100644 --- a/lib/api/metrics/dashboard/annotations.rb +++ b/lib/api/metrics/dashboard/annotations.rb @@ -4,6 +4,8 @@ module API module Metrics module Dashboard class Annotations < ::API::Base + feature_category :metrics + desc 'Create a new monitoring dashboard annotation' do success Entities::Metrics::Dashboard::Annotation end diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb index cb6e7099247..909f7f0405d 100644 --- a/lib/api/metrics/user_starred_dashboards.rb +++ b/lib/api/metrics/user_starred_dashboards.rb @@ -3,6 +3,8 @@ module API module Metrics class UserStarredDashboards < ::API::Base + feature_category :metrics + resource :projects do desc 'Marks selected metrics dashboard as starred' do success Entities::Metrics::UserStarredDashboard diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index f98a1f6dd1d..25a901c18b6 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :subgroups + helpers do params :optional_list_params_ee do # EE::API::Namespaces would override this helper diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 0db537ca616..d249431b2f8 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - Helpers::NotesHelpers.noteable_types.each do |noteable_type| + Helpers::NotesHelpers.feature_category_per_noteable_type.each do |noteable_type, feature_category| parent_type = noteable_type.parent_class.to_s.underscore noteables_str = noteable_type.to_s.underscore.pluralize @@ -29,7 +29,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get ":id/#{noteables_str}/:noteable_id/notes" do + get ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) # We exclude notes that are cross-references and that cannot be viewed @@ -57,7 +57,7 @@ module API requires :note_id, type: Integer, desc: 'The ID of a note' requires :noteable_id, type: Integer, desc: 'The ID of the noteable' end - get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + get ":id/#{noteables_str}/:noteable_id/notes/:note_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) get_note(noteable, params[:note_id]) end @@ -71,7 +71,7 @@ module API optional :confidential, type: Boolean, desc: 'Confidentiality note flag, default is false' optional :created_at, type: String, desc: 'The creation date of the note' end - post ":id/#{noteables_str}/:noteable_id/notes" do + post ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) opts = { @@ -104,7 +104,7 @@ module API optional :body, type: String, allow_blank: false, desc: 'The content of a note' optional :confidential, type: Boolean, desc: 'Confidentiality note flag' end - put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + put ":id/#{noteables_str}/:noteable_id/notes/:note_id", feature_category: feature_category do noteable = find_noteable(noteable_type, params[:noteable_id]) update_note(noteable, params[:note_id]) diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index bad3f5ead7a..7d28394e034 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -5,6 +5,8 @@ module API class NotificationSettings < ::API::Base before { authenticate! } + feature_category :users + helpers ::API::Helpers::MembersHelpers resource :notification_settings do diff --git a/lib/api/npm_instance_packages.rb b/lib/api/npm_instance_packages.rb new file mode 100644 index 00000000000..12fc008e00f --- /dev/null +++ b/lib/api/npm_instance_packages.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +module API + class NpmInstancePackages < ::API::Base + helpers ::API::Helpers::Packages::Npm + + feature_category :package_registry + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + namespace 'packages/npm' do + include ::API::Concerns::Packages::NpmEndpoints + end + end +end diff --git a/lib/api/npm_packages.rb b/lib/api/npm_packages.rb deleted file mode 100644 index 1443b28c1ee..00000000000 --- a/lib/api/npm_packages.rb +++ /dev/null @@ -1,173 +0,0 @@ -# frozen_string_literal: true -module API - class NpmPackages < ::API::Base - helpers ::API::Helpers::PackagesHelpers - helpers ::API::Helpers::Packages::DependencyProxyHelpers - - NPM_ENDPOINT_REQUIREMENTS = { - package_name: API::NO_SLASH_URL_PART_REGEX - }.freeze - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end - - before do - require_packages_enabled! - authenticate_non_get! - end - - helpers do - def project_by_package_name - strong_memoize(:project_by_package_name) do - ::Packages::Package.npm.with_name(params[:package_name]).first&.project - end - end - end - - desc 'Get all tags for a given an NPM package' do - detail 'This feature was introduced in GitLab 12.7' - success ::API::Entities::NpmPackageTag - end - params do - requires :package_name, type: String, desc: 'Package name' - end - get 'packages/npm/-/package/*package_name/dist-tags', format: false, requirements: NPM_ENDPOINT_REQUIREMENTS do - package_name = params[:package_name] - - bad_request!('Package Name') if package_name.blank? - - authorize_read_package!(project_by_package_name) - - packages = ::Packages::Npm::PackageFinder.new(project_by_package_name, package_name) - .execute - - present ::Packages::Npm::PackagePresenter.new(package_name, packages), - with: ::API::Entities::NpmPackageTag - end - - params do - requires :package_name, type: String, desc: 'Package name' - requires :tag, type: String, desc: "Package dist-tag" - end - namespace 'packages/npm/-/package/*package_name/dist-tags/:tag', requirements: NPM_ENDPOINT_REQUIREMENTS do - desc 'Create or Update the given tag for the given NPM package and version' do - detail 'This feature was introduced in GitLab 12.7' - end - put format: false do - package_name = params[:package_name] - version = env['api.request.body'] - tag = params[:tag] - - bad_request!('Package Name') if package_name.blank? - bad_request!('Version') if version.blank? - bad_request!('Tag') if tag.blank? - - authorize_create_package!(project_by_package_name) - - package = ::Packages::Npm::PackageFinder - .new(project_by_package_name, package_name) - .find_by_version(version) - not_found!('Package') unless package - - ::Packages::Npm::CreateTagService.new(package, tag).execute - - no_content! - end - - desc 'Deletes the given tag' do - detail 'This feature was introduced in GitLab 12.7' - end - delete format: false do - package_name = params[:package_name] - tag = params[:tag] - - bad_request!('Package Name') if package_name.blank? - bad_request!('Tag') if tag.blank? - - authorize_destroy_package!(project_by_package_name) - - package_tag = ::Packages::TagsFinder - .new(project_by_package_name, package_name, package_type: :npm) - .find_by_name(tag) - - not_found!('Package tag') unless package_tag - - ::Packages::RemoveTagService.new(package_tag).execute - - no_content! - end - end - - desc 'NPM registry endpoint at instance level' do - detail 'This feature was introduced in GitLab 11.8' - end - params do - requires :package_name, type: String, desc: 'Package name' - end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true - get 'packages/npm/*package_name', format: false, requirements: NPM_ENDPOINT_REQUIREMENTS do - package_name = params[:package_name] - - redirect_registry_request(project_by_package_name.blank?, :npm, package_name: package_name) do - authorize_read_package!(project_by_package_name) - - packages = ::Packages::Npm::PackageFinder - .new(project_by_package_name, package_name).execute - - present ::Packages::Npm::PackagePresenter.new(package_name, packages), - with: ::API::Entities::NpmPackage - end - end - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Download the NPM tarball' do - detail 'This feature was introduced in GitLab 11.8' - end - params do - requires :package_name, type: String, desc: 'Package name' - requires :file_name, type: String, desc: 'Package file name' - end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true - get ':id/packages/npm/*package_name/-/*file_name', format: false do - authorize_read_package!(user_project) - - package = user_project.packages.npm - .by_name_and_file_name(params[:package_name], params[:file_name]) - - package_file = ::Packages::PackageFileFinder - .new(package, params[:file_name]).execute! - - track_package_event('pull_package', package) - - present_carrierwave_file!(package_file.file) - end - - desc 'Create NPM package' do - detail 'This feature was introduced in GitLab 11.8' - end - params do - requires :package_name, type: String, desc: 'Package name' - requires :versions, type: Hash, desc: 'Package version info' - end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true - put ':id/packages/npm/:package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do - authorize_create_package!(user_project) - - track_package_event('push_package', :npm) - - created_package = ::Packages::Npm::CreatePackageService - .new(user_project, current_user, params.merge(build: current_authenticated_job)).execute - - if created_package[:status] == :error - render_api_error!(created_package[:message], created_package[:http_status]) - else - created_package - end - end - end - end -end diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb new file mode 100644 index 00000000000..887084dc9ae --- /dev/null +++ b/lib/api/npm_project_packages.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true +module API + class NpmProjectPackages < ::API::Base + helpers ::API::Helpers::Packages::Npm + + feature_category :package_registry + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + namespace 'projects/:id/packages/npm' do + desc 'Download the NPM tarball' do + detail 'This feature was introduced in GitLab 11.8' + end + params do + requires :package_name, type: String, desc: 'Package name' + requires :file_name, type: String, desc: 'Package file name' + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + get '*package_name/-/*file_name', format: false do + authorize_read_package!(project) + + package = project.packages.npm + .by_name_and_file_name(params[:package_name], params[:file_name]) + + not_found!('Package') unless package + + package_file = ::Packages::PackageFileFinder + .new(package, params[:file_name]).execute! + + track_package_event('pull_package', package, category: 'API::NpmPackages') + + present_carrierwave_file!(package_file.file) + end + + desc 'Create NPM package' do + detail 'This feature was introduced in GitLab 11.8' + end + params do + requires :package_name, type: String, desc: 'Package name' + requires :versions, type: Hash, desc: 'Package version info' + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do + authorize_create_package!(project) + + track_package_event('push_package', :npm, category: 'API::NpmPackages') + + created_package = ::Packages::Npm::CreatePackageService + .new(project, current_user, params.merge(build: current_authenticated_job)).execute + + if created_package[:status] == :error + render_api_error!(created_package[:message], created_package[:http_status]) + else + created_package + end + end + + include ::API::Concerns::Packages::NpmEndpoints + end + end +end diff --git a/lib/api/nuget_packages.rb b/lib/api/nuget_packages.rb index 0f2c956a9df..65a85f3c930 100644 --- a/lib/api/nuget_packages.rb +++ b/lib/api/nuget_packages.rb @@ -10,6 +10,8 @@ module API helpers ::API::Helpers::PackagesManagerClientsHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers + feature_category :package_registry + POSITIVE_INTEGER_REGEX = %r{\A[1-9]\d*\z}.freeze NON_NEGATIVE_INTEGER_REGEX = %r{\A0|[1-9]\d*\z}.freeze diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb index c1fc9a6e4d8..4a33f3e8af2 100644 --- a/lib/api/package_files.rb +++ b/lib/api/package_files.rb @@ -8,6 +8,8 @@ module API authorize_packages_access!(user_project) end + feature_category :package_registry + helpers ::API::Helpers::PackagesHelpers params do diff --git a/lib/api/pages.rb b/lib/api/pages.rb index 813307c498f..5f695f3853d 100644 --- a/lib/api/pages.rb +++ b/lib/api/pages.rb @@ -2,6 +2,8 @@ module API class Pages < ::API::Base + feature_category :pages + before do require_pages_config_enabled! authenticated_with_can_read_all_resources! diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb index 00c51298c45..2e7f8475509 100644 --- a/lib/api/pages_domains.rb +++ b/lib/api/pages_domains.rb @@ -4,6 +4,8 @@ module API class PagesDomains < ::API::Base include PaginationParams + feature_category :pages + PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX) before do diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb new file mode 100644 index 00000000000..2c60938b75a --- /dev/null +++ b/lib/api/personal_access_tokens.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module API + class PersonalAccessTokens < ::API::Base + include ::API::PaginationParams + + feature_category :authentication_and_authorization + + desc 'Get all Personal Access Tokens' do + detail 'This feature was added in GitLab 13.3' + success Entities::PersonalAccessToken + end + params do + optional :user_id, type: Integer, desc: 'User ID' + + use :pagination + end + + before do + authenticate! + restrict_non_admins! unless current_user.admin? + end + + helpers do + def finder_params(current_user) + current_user.admin? ? { user: user(params[:user_id]) } : { user: current_user } + end + + def user(user_id) + UserFinder.new(user_id).find_by_id + end + + def restrict_non_admins! + return if params[:user_id].blank? + + unauthorized! unless Ability.allowed?(current_user, :read_user_personal_access_tokens, user(params[:user_id])) + end + + def find_token(id) + PersonalAccessToken.find(id) || not_found! + end + end + + resources :personal_access_tokens do + get do + tokens = PersonalAccessTokensFinder.new(finder_params(current_user), current_user).execute + + present paginate(tokens), with: Entities::PersonalAccessToken + end + + delete ':id' do + service = ::PersonalAccessTokens::RevokeService.new( + current_user, + token: find_token(params[:id]) + ).execute + + service.success? ? no_content! : bad_request!(nil) + end + end + end +end diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index 46ccb4ba1a0..cfb0c5fd705 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :kubernetes_management + params do requires :id, type: String, desc: 'The ID of the project' end diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index d565531d372..3125de88de5 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -10,6 +10,8 @@ module API before { authorize_read_container_images! } + feature_category :package_registry + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb index 3765473bc0e..69b47f9420d 100644 --- a/lib/api/project_events.rb +++ b/lib/api/project_events.rb @@ -6,6 +6,8 @@ module API include APIGuard helpers ::API::Helpers::EventsHelpers + feature_category :users + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 184f89200ab..76b3dea723a 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -4,6 +4,8 @@ module API class ProjectExport < ::API::Base helpers Helpers::RateLimiter + feature_category :importers + before do not_found! unless Gitlab::CurrentSettings.project_export_enabled? authorize_admin_project diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index bc2d8c816a8..431ba199131 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -7,6 +7,8 @@ module API before { authenticate! } before { authorize_admin_project } + feature_category :integrations + helpers do params :project_hook_properties do requires :url, type: String, desc: "The URL to send the request to" @@ -21,6 +23,7 @@ module API optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" optional :deployment_events, type: Boolean, desc: "Trigger hook on deployment events" + optional :releases_events, type: Boolean, desc: "Trigger hook on release events" optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only" diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 5c4e1d73ee1..15b06cea385 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -8,6 +8,8 @@ module API helpers Helpers::FileUploadHelpers helpers Helpers::RateLimiter + feature_category :importers + helpers do def import_params declared_params(include_missing: false) diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb index a81118f44bd..8675de33923 100644 --- a/lib/api/project_milestones.rb +++ b/lib/api/project_milestones.rb @@ -7,6 +7,8 @@ module API before { authenticate! } + feature_category :issue_tracking + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index b8d97b1243a..56e94333433 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -8,6 +8,8 @@ module API authorize_packages_access!(user_project) end + feature_category :package_registry + helpers ::API::Helpers::PackagesHelpers params do diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb index 38eb74663d3..fe6de3ea385 100644 --- a/lib/api/project_repository_storage_moves.rb +++ b/lib/api/project_repository_storage_moves.rb @@ -6,6 +6,8 @@ module API before { authenticated_as_admin! } + feature_category :gitaly + resource :project_repository_storage_moves do desc 'Get a list of all project repository storage moves' do detail 'This feature was introduced in GitLab 13.0.' diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb index e19afb6e8e4..d33d2976b1c 100644 --- a/lib/api/project_snapshots.rb +++ b/lib/api/project_snapshots.rb @@ -6,6 +6,8 @@ module API before { authorize_read_git_snapshot! } + feature_category :source_code_management + resource :projects do desc 'Download a (possibly inconsistent) snapshot of a repository' do detail 'This feature was introduced in GitLab 10.7' diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index b4de260fe49..899984fe0ba 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -6,6 +6,8 @@ module API before { check_snippets_enabled } + feature_category :snippets + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb index 1ead969fc81..3db8d20ebac 100644 --- a/lib/api/project_statistics.rb +++ b/lib/api/project_statistics.rb @@ -2,6 +2,8 @@ module API class ProjectStatistics < ::API::Base + feature_category :source_code_management + before do authenticate! authorize! :daily_statistics, user_project diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index 7d851de0237..af5d96969ef 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -12,6 +12,8 @@ module API before { authenticate_non_get! } + feature_category :templates + params do requires :id, type: String, desc: 'The ID of a project' requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ecee76ae60c..2012c348cd1 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,6 +11,8 @@ module API before { authenticate_non_get! } + feature_category :projects, ['/projects/:id/custom_attributes', '/projects/:id/custom_attributes/:key'] + helpers do # EE::API::Projects would override this method def apply_filters(projects) @@ -150,7 +152,7 @@ module API use :statistics_params use :with_custom_attributes end - get ":user_id/projects" do + get ":user_id/projects", feature_category: :projects do user = find_user(params[:user_id]) not_found!('User') unless user @@ -167,7 +169,7 @@ module API use :collection_params use :statistics_params end - get ":user_id/starred_projects" do + get ":user_id/starred_projects", feature_category: :projects do user = find_user(params[:user_id]) not_found!('User') unless user @@ -187,7 +189,7 @@ module API use :statistics_params use :with_custom_attributes end - get do + get feature_category: :projects do present_projects load_projects end @@ -234,7 +236,7 @@ module API use :create_params end # rubocop: disable CodeReuse/ActiveRecord - post "user/:user_id" do + post "user/:user_id", feature_category: :projects do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139') authenticated_as_admin! user = User.find_by(id: params.delete(:user_id)) @@ -270,7 +272,7 @@ module API optional :license, type: Boolean, default: false, desc: 'Include project license data' end - get ":id" do + get ":id", feature_category: :projects do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, @@ -294,7 +296,7 @@ module API optional :path, type: String, desc: 'The path that will be assigned to the fork' optional :name, type: String, desc: 'The name that will be assigned to the fork' end - post ':id/fork' do + post ':id/fork', feature_category: :source_code_management do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284') not_found! unless can?(current_user, :fork_project, user_project) @@ -332,14 +334,14 @@ module API use :collection_params use :with_custom_attributes end - get ':id/forks' do + get ':id/forks', feature_category: :source_code_management do forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute present_projects forks, request_scope: user_project end desc 'Check pages access of this project' - get ':id/pages_access' do + get ':id/pages_access', feature_category: :pages do authorize! :read_pages_content, user_project unless user_project.public_pages? status 200 end @@ -357,7 +359,7 @@ module API at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of) end - put ':id' do + put ':id', feature_category: :projects do authorize_admin_project attrs = declared_params(include_missing: false) authorize! :rename_project, user_project if attrs[:name].present? @@ -381,7 +383,7 @@ module API desc 'Archive a project' do success Entities::Project end - post ':id/archive' do + post ':id/archive', feature_category: :projects do authorize!(:archive_project, user_project) ::Projects::UpdateService.new(user_project, current_user, archived: true).execute @@ -392,7 +394,7 @@ module API desc 'Unarchive a project' do success Entities::Project end - post ':id/unarchive' do + post ':id/unarchive', feature_category: :projects do authorize!(:archive_project, user_project) ::Projects::UpdateService.new(user_project, current_user, archived: false).execute @@ -403,7 +405,7 @@ module API desc 'Star a project' do success Entities::Project end - post ':id/star' do + post ':id/star', feature_category: :projects do if current_user.starred?(user_project) not_modified! else @@ -417,7 +419,7 @@ module API desc 'Unstar a project' do success Entities::Project end - post ':id/unstar' do + post ':id/unstar', feature_category: :projects do if current_user.starred?(user_project) current_user.toggle_star(user_project) user_project.reset @@ -435,21 +437,21 @@ module API optional :search, type: String, desc: 'Return list of users matching the search criteria' use :pagination end - get ':id/starrers' do + get ':id/starrers', feature_category: :projects do starrers = UsersStarProjectsFinder.new(user_project, params, current_user: current_user).execute present paginate(starrers), with: Entities::UserStarsProject end desc 'Get languages in project repository' - get ':id/languages' do + get ':id/languages', feature_category: :source_code_management do ::Projects::RepositoryLanguagesService .new(user_project, current_user) .execute.map { |lang| [lang.name, lang.share] }.to_h end desc 'Delete a project' - delete ":id" do + delete ":id", feature_category: :projects do authorize! :remove_project, user_project delete_project(user_project) @@ -459,7 +461,7 @@ module API params do requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' end - post ":id/fork/:forked_from_id" do + post ":id/fork/:forked_from_id", feature_category: :source_code_management do authorize! :admin_project, user_project fork_from_project = find_project!(params[:forked_from_id]) @@ -478,7 +480,7 @@ module API end desc 'Remove a forked_from relationship' - delete ":id/fork" do + delete ":id/fork", feature_category: :source_code_management do authorize! :remove_fork_project, user_project result = destroy_conditionally!(user_project) do @@ -496,7 +498,7 @@ module API requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end - post ":id/share" do + post ":id/share", feature_category: :authentication_and_authorization do authorize! :admin_project, user_project group = Group.find_by_id(params[:group_id]) @@ -518,7 +520,7 @@ module API requires :group_id, type: Integer, desc: 'The ID of the group' end # rubocop: disable CodeReuse/ActiveRecord - delete ":id/share/:group_id" do + delete ":id/share/:group_id", feature_category: :authentication_and_authorization do authorize! :admin_project, user_project link = user_project.project_group_links.find_by(group_id: params[:group_id]) @@ -535,7 +537,7 @@ module API # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 requires :file, type: File, desc: 'The file to be uploaded' # rubocop:disable Scalability/FileUploads end - post ":id/uploads" do + post ":id/uploads", feature_category: :not_owned do upload = UploadService.new(user_project, params[:file]).execute present upload, with: Entities::ProjectUpload @@ -549,7 +551,7 @@ module API optional :skip_users, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Filter out users with the specified IDs' use :pagination end - get ':id/users' do + get ':id/users', feature_category: :authentication_and_authorization do users = DeclarativePolicy.subject_scope { user_project.team.users } users = users.search(params[:search]) if params[:search].present? users = users.where_not_in(params[:skip_users]) if params[:skip_users].present? @@ -560,7 +562,7 @@ module API desc 'Start the housekeeping task for a project' do detail 'This feature was introduced in GitLab 9.0.' end - post ':id/housekeeping' do + post ':id/housekeeping', feature_category: :source_code_management do authorize_admin_project begin @@ -574,7 +576,7 @@ module API params do requires :namespace, type: String, desc: 'The ID or path of the new namespace' end - put ":id/transfer" do + put ":id/transfer", feature_category: :projects do authorize! :change_namespace, user_project namespace = find_namespace!(params[:namespace]) diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index a448682d8bd..17574739a7c 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -8,6 +8,8 @@ module API before { authorize_admin_project } + feature_category :source_code_management + helpers Helpers::ProtectedBranchesHelpers params do diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb index dd3e407ffc9..b9385df1f8d 100644 --- a/lib/api/protected_tags.rb +++ b/lib/api/protected_tags.rb @@ -8,6 +8,8 @@ module API before { authorize_admin_project } + feature_category :source_code_management + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 5622bc6e42d..7104fb8d999 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -12,6 +12,8 @@ module API helpers ::API::Helpers::Packages::BasicAuthHelpers include ::API::Helpers::Packages::BasicAuthHelpers::Constants + feature_category :package_registry + default_format :json rescue_from ArgumentError do |e| diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index 23de9f9fc9f..d3a185a51c8 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -10,6 +10,8 @@ module API before { authorize! :read_release, user_project } + feature_category :release_orchestration + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/releases.rb b/lib/api/releases.rb index 3bd6ea77403..c20e618efd1 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -9,6 +9,8 @@ module API before { authorize_read_releases! } + feature_category :release_orchestration + params do requires :id, type: String, desc: 'The ID of a project' end @@ -89,7 +91,7 @@ module API optional :name, type: String, desc: 'The name of the release' optional :description, type: String, desc: 'Release notes with markdown support' optional :released_at, type: DateTime, desc: 'The date when the release will be/was ready.' - optional :milestones, type: Array, desc: 'The titles of the related milestones' + optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The titles of the related milestones' end put ':id/releases/:tag_name', requirements: RELEASE_ENDPOINT_REQUIREMENTS do authorize_update_release! diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb index f63ea04a529..83096772d32 100644 --- a/lib/api/remote_mirrors.rb +++ b/lib/api/remote_mirrors.rb @@ -4,6 +4,8 @@ module API class RemoteMirrors < ::API::Base include PaginationParams + feature_category :source_code_management + before do unauthorized! unless can?(current_user, :admin_remote_mirror, user_project) end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 38ac1f22a48..8af8ffc3b63 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -6,10 +6,14 @@ module API class Repositories < ::API::Base include PaginationParams + content_type :txt, 'text/plain' + helpers ::API::Helpers::HeadersHelpers before { authorize! :download_code, user_project } + feature_category :source_code_management + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb index d3a219f0810..33589f6c393 100644 --- a/lib/api/resource_label_events.rb +++ b/lib/api/resource_label_events.rb @@ -7,7 +7,7 @@ module API before { authenticate! } - Helpers::ResourceLabelEventsHelpers.eventable_types.each do |eventable_type| + Helpers::ResourceLabelEventsHelpers.feature_category_per_eventable_type.each do |eventable_type, feature_category| parent_type = eventable_type.parent_class.to_s.underscore eventables_str = eventable_type.to_s.underscore.pluralize @@ -24,7 +24,7 @@ module API use :pagination end - get ":id/#{eventables_str}/:eventable_id/resource_label_events" do + get ":id/#{eventables_str}/:eventable_id/resource_label_events", feature_category: feature_category do eventable = find_noteable(eventable_type, params[:eventable_id]) events = eventable.resource_label_events.inc_relations @@ -40,7 +40,7 @@ module API requires :event_id, type: String, desc: 'The ID of a resource label event' requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' end - get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do + get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id", feature_category: feature_category do eventable = find_noteable(eventable_type, params[:eventable_id]) event = eventable.resource_label_events.find(params[:event_id]) diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb index 21411f68dd5..aeedd7ad109 100644 --- a/lib/api/resource_milestone_events.rb +++ b/lib/api/resource_milestone_events.rb @@ -7,7 +7,10 @@ module API before { authenticate! } - [Issue, MergeRequest].each do |eventable_type| + { + Issue => :issue_tracking, + MergeRequest => :code_review + }.each do |eventable_type, feature_category| parent_type = eventable_type.parent_class.to_s.underscore eventables_str = eventable_type.to_s.underscore.pluralize @@ -23,7 +26,7 @@ module API use :pagination end - get ":id/#{eventables_str}/:eventable_id/resource_milestone_events" do + get ":id/#{eventables_str}/:eventable_id/resource_milestone_events", feature_category: feature_category do eventable = find_noteable(eventable_type, params[:eventable_id]) events = ResourceMilestoneEventFinder.new(current_user, eventable).execute @@ -38,7 +41,7 @@ module API requires :event_id, type: String, desc: 'The ID of a resource milestone event' requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' end - get ":id/#{eventables_str}/:eventable_id/resource_milestone_events/:event_id" do + get ":id/#{eventables_str}/:eventable_id/resource_milestone_events/:event_id", feature_category: feature_category do eventable = find_noteable(eventable_type, params[:eventable_id]) event = eventable.resource_milestone_events.find(params[:event_id]) diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb index 9bfda39be90..3460aa2c00e 100644 --- a/lib/api/resource_state_events.rb +++ b/lib/api/resource_state_events.rb @@ -7,7 +7,10 @@ module API before { authenticate! } - [Issue, MergeRequest].each do |eventable_class| + { + Issue => :issue_tracking, + MergeRequest => :code_review + }.each do |eventable_class, feature_category| eventable_name = eventable_class.to_s.underscore params do @@ -22,7 +25,7 @@ module API use :pagination end - get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events" do + get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events", feature_category: feature_category do eventable = find_noteable(eventable_class, params[:eventable_iid]) events = ResourceStateEventFinder.new(current_user, eventable).execute @@ -37,7 +40,7 @@ module API requires :eventable_iid, types: Integer, desc: "The IID of the #{eventable_name}" requires :event_id, type: Integer, desc: 'The ID of a resource state event' end - get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events/:event_id" do + get ":id/#{eventable_name.pluralize}/:eventable_iid/resource_state_events/:event_id", feature_category: feature_category do eventable = find_noteable(eventable_class, params[:eventable_iid]) event = ResourceStateEventFinder.new(current_user, eventable).find(params[:event_id]) diff --git a/lib/api/search.rb b/lib/api/search.rb index 85f0a8e2e60..f0ffe6ba443 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :global_search + helpers do SCOPE_ENTITY = { merge_requests: Entities::MergeRequestBasic, @@ -35,8 +37,11 @@ module API state: params[:state], confidential: params[:confidential], snippets: snippets?, + basic_search: params[:basic_search], page: params[:page], - per_page: params[:per_page] + per_page: params[:per_page], + order_by: params[:order_by], + sort: params[:sort] }.merge(additional_params) results = SearchService.new(current_user, search_params).search_objects(preload_method) diff --git a/lib/api/services.rb b/lib/api/services.rb index 5f3d14010a8..cfcae13e518 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module API class Services < ::API::Base + feature_category :integrations + services = Helpers::ServicesHelpers.services service_classes = Helpers::ServicesHelpers.service_classes diff --git a/lib/api/settings.rb b/lib/api/settings.rb index dc917d9c529..b95856d99d1 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -4,6 +4,8 @@ module API class Settings < ::API::Base before { authenticated_as_admin! } + feature_category :not_owned + helpers Helpers::SettingsHelpers helpers do @@ -51,9 +53,9 @@ module API optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' optional :disabled_oauth_sign_in_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Disable certain OAuth sign-in sources' - optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' - optional :domain_blacklist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' - optional :domain_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :domain_denylist_enabled, type: Boolean, desc: 'Enable domain denylist for sign ups' + optional :domain_denylist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :domain_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' optional :eks_integration_enabled, type: Boolean, desc: 'Enable integration with Amazon EKS' given eks_integration_enabled: -> (val) { val } do requires :eks_account_id, type: String, desc: 'Amazon account ID for EKS integration' @@ -157,6 +159,7 @@ module API optional :issues_create_limit, type: Integer, desc: "Maximum number of issue creation requests allowed per minute per user. Set to 0 for unlimited requests per minute." optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute." optional :wiki_page_max_content_bytes, type: Integer, desc: "Maximum wiki page content size in bytes" + optional :require_admin_approval_after_user_signup, type: Boolean, desc: 'Require explicit admin approval for new signups' ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index b025dbfab37..680363d036e 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -6,6 +6,8 @@ module API class SidekiqMetrics < ::API::Base before { authenticated_as_admin! } + feature_category :not_owned + helpers do def queue_metrics Sidekiq::Queue.all.each_with_object({}) do |queue, hash| diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index 2e67b9649bc..52b597fb788 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -5,6 +5,8 @@ module API class Snippets < ::API::Base include PaginationParams + feature_category :snippets + resource :snippets do helpers Helpers::SnippetsHelpers helpers do diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb index fa7176491ba..1814e1a6782 100644 --- a/lib/api/statistics.rb +++ b/lib/api/statistics.rb @@ -4,6 +4,8 @@ module API class Statistics < ::API::Base before { authenticated_as_admin! } + feature_category :instance_statistics + COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue, MergeRequest, Note, Snippet, Key, Milestone].freeze diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb index e2ceb49c119..5c71a18c6d0 100644 --- a/lib/api/submodules.rb +++ b/lib/api/submodules.rb @@ -4,6 +4,8 @@ module API class Submodules < ::API::Base before { authenticate! } + feature_category :source_code_management + helpers do def commit_params(attrs) { diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 35a28da4736..914bab52929 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -11,25 +11,29 @@ module API type: 'merge_requests', entity: Entities::MergeRequest, source: Project, - finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) } + finder: ->(id) { find_merge_request_with_access(id, :update_merge_request) }, + feature_category: :code_review }, { type: 'issues', entity: Entities::Issue, source: Project, - finder: ->(id) { find_project_issue(id) } + finder: ->(id) { find_project_issue(id) }, + feature_category: :issue_tracking }, { type: 'labels', entity: Entities::ProjectLabel, source: Project, - finder: ->(id) { find_label(user_project, id) } + finder: ->(id) { find_label(user_project, id) }, + feature_category: :issue_tracking }, { type: 'labels', entity: Entities::GroupLabel, source: Group, - finder: ->(id) { find_label(user_group, id) } + finder: ->(id) { find_label(user_group, id) }, + feature_category: :issue_tracking } ] @@ -44,7 +48,7 @@ module API desc 'Subscribe to a resource' do success subscribable[:entity] end - post ":id/#{subscribable[:type]}/:subscribable_id/subscribe" do + post ":id/#{subscribable[:type]}/:subscribable_id/subscribe", subscribable.slice(:feature_category) do parent = parent_resource(source_type) resource = instance_exec(params[:subscribable_id], &subscribable[:finder]) @@ -59,7 +63,7 @@ module API desc 'Unsubscribe from a resource' do success subscribable[:entity] end - post ":id/#{subscribable[:type]}/:subscribable_id/unsubscribe" do + post ":id/#{subscribable[:type]}/:subscribable_id/unsubscribe", subscribable.slice(:feature_category) do parent = parent_resource(source_type) resource = instance_exec(params[:subscribable_id], &subscribable[:finder]) diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index f23d279c3f4..a024d6de874 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -4,6 +4,8 @@ module API class Suggestions < ::API::Base before { authenticate! } + feature_category :code_review + resource :suggestions do desc 'Apply suggestion patch in the Merge Request it was created' do success Entities::Suggestion diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 2820d305d0f..42e16d47a0b 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -4,6 +4,8 @@ module API class SystemHooks < ::API::Base include PaginationParams + feature_category :integrations + before do authenticate! authenticated_as_admin! diff --git a/lib/api/tags.rb b/lib/api/tags.rb index b969394ec47..7636c45bdac 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -23,7 +23,7 @@ module API optional :search, type: String, desc: 'Return list of tags matching the search criteria' use :pagination end - get ':id/repository/tags' do + get ':id/repository/tags', feature_category: :source_code_management do tags = ::TagsFinder.new(user_project.repository, sort: "#{params[:order_by]}_#{params[:sort]}", search: params[:search]).execute @@ -37,7 +37,7 @@ module API params do requires :tag_name, type: String, desc: 'The name of the tag' end - get ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do + get ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :source_code_management do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag @@ -54,7 +54,7 @@ module API optional :message, type: String, desc: 'Specifying a message creates an annotated tag' optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database (deprecated in GitLab 11.7)' end - post ':id/repository/tags' do + post ':id/repository/tags', :release_orchestration do authorize_admin_tag result = ::Tags::CreateService.new(user_project, current_user) @@ -86,7 +86,7 @@ module API params do requires :tag_name, type: String, desc: 'The name of the tag' end - delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS do + delete ':id/repository/tags/:tag_name', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :source_code_management do authorize_admin_tag tag = user_project.repository.find_tag(params[:tag_name]) @@ -112,7 +112,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end - post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do + post ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :release_orchestration do authorize_create_release! ## @@ -144,7 +144,7 @@ module API requires :tag_name, type: String, desc: 'The name of the tag', as: :tag requires :description, type: String, desc: 'Release notes with markdown support' end - put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS do + put ':id/repository/tags/:tag_name/release', requirements: TAG_ENDPOINT_REQUIREMENTS, feature_category: :release_orchestration do authorize_update_release! result = ::Releases::UpdateService diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 0b427bbf5b9..b7fb35eac03 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -4,6 +4,8 @@ module API class Templates < ::API::Base include PaginationParams + feature_category :templates + GLOBAL_TEMPLATE_TYPES = { gitignores: { gitlab_version: 8.8 diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index 3dbde4639ca..c664c0a4590 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -7,6 +7,8 @@ module API class State < ::API::Base include ::Gitlab::Utils::StrongMemoize + feature_category :infrastructure_as_code + default_format :json before do @@ -51,7 +53,7 @@ module API no_content! if data.empty? remote_state_handler.handle_with_lock do |state| - state.update_file!(CarrierWaveStringFile.new(data), version: params[:serial]) + state.update_file!(CarrierWaveStringFile.new(data), version: params[:serial], build: current_authenticated_job) end body false diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb index b4a0efd7a2b..d3680323b9f 100644 --- a/lib/api/terraform/state_version.rb +++ b/lib/api/terraform/state_version.rb @@ -5,6 +5,8 @@ module API class StateVersion < ::API::Base default_format :json + feature_category :infrastructure_as_code + before do authenticate! authorize! :read_terraform_state, user_project diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ce07d13cc9a..03850ba1c4e 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -6,6 +6,8 @@ module API before { authenticate! } + feature_category :issue_tracking + ISSUABLE_TYPES = { 'merge_requests' => ->(iid) { find_merge_request_with_access(iid) }, 'issues' => ->(iid) { find_project_issue(iid) } diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 960d004a04c..aebbc95cbea 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -6,6 +6,8 @@ module API HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase + feature_category :continuous_integration + params do requires :id, type: String, desc: 'The ID of a project' end diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb index 907422118f1..3148c56339a 100644 --- a/lib/api/unleash.rb +++ b/lib/api/unleash.rb @@ -4,6 +4,8 @@ module API class Unleash < ::API::Base include PaginationParams + feature_category :feature_flags + namespace :feature_flags do resource :unleash, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index fa5bfc1cbe9..7b038ec74bb 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -4,6 +4,8 @@ module API class UsageData < ::API::Base before { authenticate! } + feature_category :collection + namespace 'usage_data' do before do not_found! unless Feature.enabled?(:usage_data_api, default_enabled: true) diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 6d9db53fec8..3071f08e1de 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -2,6 +2,8 @@ module API class UserCounts < ::API::Base + feature_category :navigation + resource :user_counts do desc 'Return the user specific counts' do detail 'Open MR Count' diff --git a/lib/api/users.rb b/lib/api/users.rb index e7c1d644324..501ed629c7e 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -8,6 +8,8 @@ module API allow_access_with_scope :read_user, if: -> (request) { request.get? } + feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] + resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do include CustomAttributesEndpoints @@ -63,9 +65,9 @@ module API params :sort_params do optional :order_by, type: String, values: %w[id name username created_at updated_at], - default: 'id', desc: 'Return users ordered by a field' + default: 'id', desc: 'Return users ordered by a field' optional :sort, type: String, values: %w[asc desc], default: 'desc', - desc: 'Return users sorted in ascending and descending order' + desc: 'Return users sorted in ascending and descending order' end end @@ -93,7 +95,7 @@ module API use :optional_index_params_ee end # rubocop: disable CodeReuse/ActiveRecord - get do + get feature_category: :users do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) unless current_user&.admin? @@ -134,7 +136,7 @@ module API use :with_custom_attributes end # rubocop: disable CodeReuse/ActiveRecord - get ":id" do + get ":id", feature_category: :users do user = User.find_by(id: params[:id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -149,7 +151,7 @@ module API params do requires :user_id, type: String, desc: 'The ID or username of the user' end - get ":user_id/status", requirements: API::USER_REQUIREMENTS do + get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users do user = find_user(params[:user_id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -170,7 +172,7 @@ module API optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set' use :optional_attributes end - post do + post feature_category: :users do authenticated_as_admin! params = declared_params(include_missing: false) @@ -204,7 +206,7 @@ module API use :optional_attributes end # rubocop: disable CodeReuse/ActiveRecord - put ":id" do + put ":id", feature_category: :users do authenticated_as_admin! user = User.find_by(id: params.delete(:id)) @@ -245,7 +247,7 @@ module API requires :provider, type: String, desc: 'The external provider' end # rubocop: disable CodeReuse/ActiveRecord - delete ":id/identities/:provider" do + delete ":id/identities/:provider", feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -268,7 +270,7 @@ module API optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)' end # rubocop: disable CodeReuse/ActiveRecord - post ":id/keys" do + post ":id/keys", feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params.delete(:id)) @@ -291,7 +293,7 @@ module API requires :user_id, type: String, desc: 'The ID or username of the user' use :pagination end - get ':user_id/keys', requirements: API::USER_REQUIREMENTS do + get ':user_id/keys', requirements: API::USER_REQUIREMENTS, feature_category: :authentication_and_authorization do user = find_user(params[:user_id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -307,7 +309,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the SSH key' end # rubocop: disable CodeReuse/ActiveRecord - delete ':id/keys/:key_id' do + delete ':id/keys/:key_id', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -332,7 +334,7 @@ module API requires :key, type: String, desc: 'The new GPG key' end # rubocop: disable CodeReuse/ActiveRecord - post ':id/gpg_keys' do + post ':id/gpg_keys', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params.delete(:id)) @@ -357,7 +359,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get ':id/gpg_keys' do + get ':id/gpg_keys', feature_category: :authentication_and_authorization do user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -374,7 +376,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the GPG key' end # rubocop: disable CodeReuse/ActiveRecord - get ':id/gpg_keys/:key_id' do + get ':id/gpg_keys/:key_id', feature_category: :authentication_and_authorization do user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -393,7 +395,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the GPG key' end # rubocop: disable CodeReuse/ActiveRecord - delete ':id/gpg_keys/:key_id' do + delete ':id/gpg_keys/:key_id', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -417,7 +419,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the GPG key' end # rubocop: disable CodeReuse/ActiveRecord - post ':id/gpg_keys/:key_id/revoke' do + post ':id/gpg_keys/:key_id/revoke', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -440,7 +442,7 @@ module API optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified' end # rubocop: disable CodeReuse/ActiveRecord - post ":id/emails" do + post ":id/emails", feature_category: :users do authenticated_as_admin! user = User.find_by(id: params.delete(:id)) @@ -464,7 +466,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get ':id/emails' do + get ':id/emails', feature_category: :users do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -481,7 +483,7 @@ module API requires :email_id, type: Integer, desc: 'The ID of the email' end # rubocop: disable CodeReuse/ActiveRecord - delete ':id/emails/:email_id' do + delete ':id/emails/:email_id', feature_category: :users do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -503,7 +505,7 @@ module API optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end # rubocop: disable CodeReuse/ActiveRecord - delete ":id" do + delete ":id", feature_category: :users do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/20757') authenticated_as_admin! @@ -523,7 +525,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' end # rubocop: disable CodeReuse/ActiveRecord - post ':id/activate' do + post ':id/activate', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -538,7 +540,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' end # rubocop: disable CodeReuse/ActiveRecord - post ':id/deactivate' do + post ':id/deactivate', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -564,7 +566,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' end # rubocop: disable CodeReuse/ActiveRecord - post ':id/block' do + post ':id/block', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -589,7 +591,7 @@ module API requires :id, type: Integer, desc: 'The ID of the user' end # rubocop: disable CodeReuse/ActiveRecord - post ':id/unblock' do + post ':id/unblock', feature_category: :authentication_and_authorization do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -612,7 +614,7 @@ module API optional :type, type: String, values: %w[Project Namespace] use :pagination end - get ":user_id/memberships" do + get ":user_id/memberships", feature_category: :users do authenticated_as_admin! user = find_user_by_id(params) @@ -656,7 +658,9 @@ module API use :pagination optional :state, type: String, default: 'all', values: %w[all active inactive], desc: 'Filters (all|active|inactive) impersonation_tokens' end - get { present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken } + get feature_category :authentication_and_authorization do + present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken + end desc 'Create a impersonation token. Available only for admins.' do detail 'This feature was introduced in GitLab 9.0' @@ -667,7 +671,7 @@ module API optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token' optional :scopes, type: Array, desc: 'The array of scopes of the impersonation token' end - post do + post feature_category: :authentication_and_authorization do impersonation_token = finder.build(declared_params(include_missing: false)) if impersonation_token.save @@ -684,7 +688,7 @@ module API params do requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' end - get ':impersonation_token_id' do + get ':impersonation_token_id', feature_category: :authentication_and_authorization do present find_impersonation_token, with: Entities::ImpersonationToken end @@ -694,7 +698,7 @@ module API params do requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' end - delete ':impersonation_token_id' do + delete ':impersonation_token_id', feature_category: :authentication_and_authorization do token = find_impersonation_token destroy_conditionally!(token) do @@ -702,6 +706,40 @@ module API end end end + + resource :personal_access_tokens do + helpers do + def target_user + find_user_by_id(params) + end + end + + before { authenticated_as_admin! } + + desc 'Create a personal access token. Available only for admins.' do + detail 'This feature was introduced in GitLab 13.6' + success Entities::PersonalAccessTokenWithToken + end + params do + requires :name, type: String, desc: 'The name of the personal access token' + requires :scopes, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, values: ::Gitlab::Auth.all_available_scopes.map(&:to_s), + desc: 'The array of scopes of the personal access token' + optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the personal access token' + end + post feature_category: :authentication_and_authorization do + not_found! unless Feature.enabled?(:pat_creation_api_for_admin) + + response = ::PersonalAccessTokens::CreateService.new( + current_user: current_user, target_user: target_user, params: declared_params(include_missing: false) + ).execute + + if response.success? + present response.payload[:personal_access_token], with: Entities::PersonalAccessTokenWithToken + else + render_api_error!(response.message, response.http_status || :unprocessable_entity) + end + end + end end end @@ -716,7 +754,7 @@ module API desc 'Get the currently authenticated user' do success Entities::UserPublic end - get do + get feature_category: :users do entity = if current_user.admin? Entities::UserWithAdmin @@ -734,7 +772,7 @@ module API params do use :pagination end - get "keys" do + get "keys", feature_category: :authentication_and_authorization do keys = current_user.keys.preload_users present paginate(keys), with: Entities::SSHKey @@ -747,7 +785,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the SSH key' end # rubocop: disable CodeReuse/ActiveRecord - get "keys/:key_id" do + get "keys/:key_id", feature_category: :authentication_and_authorization do key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key @@ -763,7 +801,7 @@ module API requires :title, type: String, desc: 'The title of the new SSH key' optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)' end - post "keys" do + post "keys", feature_category: :authentication_and_authorization do key = ::Keys::CreateService.new(current_user, declared_params(include_missing: false)).execute if key.persisted? @@ -780,7 +818,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the SSH key' end # rubocop: disable CodeReuse/ActiveRecord - delete "keys/:key_id" do + delete "keys/:key_id", feature_category: :authentication_and_authorization do key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key @@ -798,7 +836,7 @@ module API params do use :pagination end - get 'gpg_keys' do + get 'gpg_keys', feature_category: :authentication_and_authorization do present paginate(current_user.gpg_keys), with: Entities::GpgKey end @@ -810,7 +848,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the GPG key' end # rubocop: disable CodeReuse/ActiveRecord - get 'gpg_keys/:key_id' do + get 'gpg_keys/:key_id', feature_category: :authentication_and_authorization do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -825,7 +863,7 @@ module API params do requires :key, type: String, desc: 'The new GPG key' end - post 'gpg_keys' do + post 'gpg_keys', feature_category: :authentication_and_authorization do key = ::GpgKeys::CreateService.new(current_user, declared_params(include_missing: false)).execute if key.persisted? @@ -842,7 +880,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the GPG key' end # rubocop: disable CodeReuse/ActiveRecord - post 'gpg_keys/:key_id/revoke' do + post 'gpg_keys/:key_id/revoke', feature_category: :authentication_and_authorization do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -858,7 +896,7 @@ module API requires :key_id, type: Integer, desc: 'The ID of the SSH key' end # rubocop: disable CodeReuse/ActiveRecord - delete 'gpg_keys/:key_id' do + delete 'gpg_keys/:key_id', feature_category: :authentication_and_authorization do key = current_user.gpg_keys.find_by(id: params[:key_id]) not_found!('GPG Key') unless key @@ -875,7 +913,7 @@ module API params do use :pagination end - get "emails" do + get "emails", feature_category: :users do present paginate(current_user.emails), with: Entities::Email end @@ -886,7 +924,7 @@ module API requires :email_id, type: Integer, desc: 'The ID of the email' end # rubocop: disable CodeReuse/ActiveRecord - get "emails/:email_id" do + get "emails/:email_id", feature_category: :users do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email @@ -900,7 +938,7 @@ module API params do requires :email, type: String, desc: 'The new email' end - post "emails" do + post "emails", feature_category: :users do email = Emails::CreateService.new(current_user, declared_params.merge(user: current_user)).execute if email.errors.blank? @@ -915,7 +953,7 @@ module API requires :email_id, type: Integer, desc: 'The ID of the email' end # rubocop: disable CodeReuse/ActiveRecord - delete "emails/:email_id" do + delete "emails/:email_id", feature_category: :users do email = current_user.emails.find_by(id: params[:email_id]) not_found!('Email') unless email @@ -931,7 +969,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get "activities" do + get "activities", feature_category: :users do authenticated_as_admin! activities = User @@ -948,8 +986,9 @@ module API params do optional :emoji, type: String, desc: "The emoji to set on the status" optional :message, type: String, desc: "The status message to set" + optional :availability, type: String, desc: "The availability of user to set" end - put "status" do + put "status", feature_category: :users do forbidden! unless can?(current_user, :update_user_status, current_user) if ::Users::SetStatusService.new(current_user, declared_params).execute @@ -962,7 +1001,7 @@ module API desc 'get the status of the current user' do success Entities::UserStatus end - get 'status' do + get 'status', feature_category: :users do present current_user.status || {}, with: Entities::UserStatus end end diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index aed88e6091c..327335aec2d 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -22,6 +22,8 @@ module API include PaginationParams + feature_category :integrations + before do authorize_jira_user_agent!(request) authenticate! diff --git a/lib/api/validations/validators/email_or_email_list.rb b/lib/api/validations/validators/email_or_email_list.rb new file mode 100644 index 00000000000..b7f2a0cd443 --- /dev/null +++ b/lib/api/validations/validators/email_or_email_list.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module API + module Validations + module Validators + class EmailOrEmailList < Grape::Validations::Base + def validate_param!(attr_name, params) + value = params[attr_name] + + return unless value + + return if value.split(',').map { |v| ValidateEmail.valid?(v) }.all? + + raise Grape::Exceptions::Validation, + params: [@scope.full_name(attr_name)], + message: "contains an invalid email address" + end + end + end + end +end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f5de3d844e6..94fa98b7a14 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -7,6 +7,8 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } + feature_category :continuous_integration + helpers do def filter_variable_parameters(params) # This method exists so that EE can more easily filter out certain diff --git a/lib/api/version.rb b/lib/api/version.rb index 841b55f8d6c..f8072658cc6 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -9,6 +9,8 @@ module API before { authenticate! } + feature_category :not_owned + METADATA_QUERY = <<~EOF { metadata { diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 21f457046f1..3fa42be47a9 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -4,6 +4,8 @@ module API class Wikis < ::API::Base helpers ::API::Helpers::WikisHelpers + feature_category :wiki + helpers do attr_reader :container diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index 0b578c03782..f81ed462174 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -3,19 +3,25 @@ module Atlassian module JiraConnect class Client < Gitlab::HTTP + def self.generate_update_sequence_id + Gitlab::Metrics::System.monotonic_time.to_i + end + def initialize(base_uri, shared_secret) @base_uri = base_uri @shared_secret = shared_secret end - def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil) + def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil) dev_info_json = { repositories: [ Serializers::RepositoryEntity.represent( project, commits: commits, branches: branches, - merge_requests: merge_requests + merge_requests: merge_requests, + user_notes_count: user_notes_count(merge_requests), + update_sequence_id: update_sequence_id ) ] }.to_json @@ -32,6 +38,14 @@ module Atlassian private + def user_notes_count(merge_requests) + return unless merge_requests + + Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').map do |count_group| + [count_group.noteable_id, count_group.count] + end.to_h + end + def jwt_token(http_method, uri) claims = Atlassian::Jwt.build_claims( Atlassian::JiraConnect.app_key, diff --git a/lib/atlassian/jira_connect/serializers/base_entity.rb b/lib/atlassian/jira_connect/serializers/base_entity.rb index c5490aa3f54..94deb174a45 100644 --- a/lib/atlassian/jira_connect/serializers/base_entity.rb +++ b/lib/atlassian/jira_connect/serializers/base_entity.rb @@ -9,12 +9,12 @@ module Atlassian format_with(:string) { |value| value.to_s } - expose :monotonic_time, as: :updateSequenceId + expose :update_sequence_id, as: :updateSequenceId private - def monotonic_time - Gitlab::Metrics::System.monotonic_time.to_i + def update_sequence_id + options[:update_sequence_id] || Client.generate_update_sequence_id end end end diff --git a/lib/atlassian/jira_connect/serializers/pull_request_entity.rb b/lib/atlassian/jira_connect/serializers/pull_request_entity.rb index 0ddfcbf52ea..e2dc197969b 100644 --- a/lib/atlassian/jira_connect/serializers/pull_request_entity.rb +++ b/lib/atlassian/jira_connect/serializers/pull_request_entity.rb @@ -20,7 +20,13 @@ module Atlassian end expose :title expose :author, using: JiraConnect::Serializers::AuthorEntity - expose :user_notes_count, as: :commentCount + expose :commentCount do |mr| + if options[:user_notes_count] + options[:user_notes_count].fetch(mr.id, 0) + else + mr.user_notes_count + end + end expose :source_branch, as: :sourceBranch expose :target_branch, as: :destinationBranch expose :lastUpdate do |mr| diff --git a/lib/atlassian/jira_connect/serializers/repository_entity.rb b/lib/atlassian/jira_connect/serializers/repository_entity.rb index 819ca2b62e0..616bbc85bfe 100644 --- a/lib/atlassian/jira_connect/serializers/repository_entity.rb +++ b/lib/atlassian/jira_connect/serializers/repository_entity.rb @@ -15,13 +15,17 @@ module Atlassian end expose :commits do |project, options| - JiraConnect::Serializers::CommitEntity.represent options[:commits], project: project + JiraConnect::Serializers::CommitEntity.represent options[:commits], project: project, update_sequence_id: options[:update_sequence_id] end expose :branches do |project, options| - JiraConnect::Serializers::BranchEntity.represent options[:branches], project: project + JiraConnect::Serializers::BranchEntity.represent options[:branches], project: project, update_sequence_id: options[:update_sequence_id] end expose :pullRequests do |project, options| - JiraConnect::Serializers::PullRequestEntity.represent options[:merge_requests], project: project + JiraConnect::Serializers::PullRequestEntity.represent( + options[:merge_requests], + update_sequence_id: options[:update_sequence_id], + user_notes_count: options[:user_notes_count] + ) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 619a62fd6f6..a0948f8c0f5 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -26,7 +26,7 @@ module Backup FileUtils.rm_f(backup_tarball) if ENV['STRATEGY'] == 'copy' - cmd = [%w(rsync -a), exclude_dirs(:rsync), %W(#{app_files_dir} #{Gitlab.config.backup.path})].flatten + cmd = [%w[rsync -a], exclude_dirs(:rsync), %W[#{app_files_dir} #{Gitlab.config.backup.path}]].flatten output, status = Gitlab::Popen.popen(cmd) unless status == 0 @@ -34,19 +34,27 @@ module Backup raise Backup::Error, 'Backup failed' end - tar_cmd = [tar, exclude_dirs(:tar), %W(-C #{@backup_files_dir} -cf - .)].flatten - run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600]) + tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{@backup_files_dir} -cf - .]].flatten + status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(@backup_files_dir) else - tar_cmd = [tar, exclude_dirs(:tar), %W(-C #{app_files_dir} -cf - .)].flatten - run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600]) + tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{app_files_dir} -cf - .]].flatten + status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600]) + end + + unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output) + raise Backup::Error, "Backup operation failed: #{output}" end end def restore backup_existing_files_dir - run_pipeline!([%w(gzip -cd), %W(#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -)], in: backup_tarball) + cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_dir} -xf -]] + status_list, output = run_pipeline!(cmd_list, in: backup_tarball) + unless pipeline_succeeded?(gzip_status: status_list[0], tar_status: status_list[1], output: output) + raise Backup::Error, "Restore operation failed: #{output}" + end end def tar @@ -78,13 +86,44 @@ module Backup def run_pipeline!(cmd_list, options = {}) err_r, err_w = IO.pipe options[:err] = err_w - status = Open3.pipeline(*cmd_list, options) + status_list = Open3.pipeline(*cmd_list, options) err_w.close - return if status.compact.all?(&:success?) - regex = /^g?tar: \.: Cannot mkdir: No such file or directory$/ - error = err_r.read - raise Backup::Error, "Backup failed. #{error}" unless error =~ regex + [status_list, err_r.read] + end + + def noncritical_warning?(warning) + noncritical_warnings = [ + /^g?tar: \.: Cannot mkdir: No such file or directory$/ + ] + + noncritical_warnings.map { |w| warning =~ w }.any? + end + + def pipeline_succeeded?(tar_status:, gzip_status:, output:) + return false unless gzip_status&.success? + + tar_status&.success? || tar_ignore_non_success?(tar_status.exitstatus, output) + end + + def tar_ignore_non_success?(exitstatus, output) + # tar can exit with nonzero code: + # 1 - if some files changed (i.e. a CI job is currently writes to log) + # 2 - if it cannot create `.` directory (see issue https://gitlab.com/gitlab-org/gitlab/-/issues/22442) + # http://www.gnu.org/software/tar/manual/html_section/tar_19.html#Synopsis + # so check tar status 1 or stderr output against some non-critical warnings + if exitstatus == 1 + $stdout.puts "Ignoring tar exit status 1 'Some files differ': #{output}" + return true + end + + # allow tar to fail with other non-success status if output contain non-critical warning + if noncritical_warning?(output) + $stdout.puts "Ignoring non-success exit status #{exitstatus} due to output of non-critical warning(s): #{output}" + return true + end + + false end def exclude_dirs(fmt) diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index b32fe5e8301..8952a3ff6b4 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -8,6 +8,7 @@ module Banzai # Based on HTML::Pipeline::EmojiFilter class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + IGNORE_UNICODE_EMOJIS = %w(™ © ®).freeze def call doc.search(".//text()").each do |node| @@ -60,7 +61,11 @@ module Banzai # Build a regexp that matches all valid unicode emojis names. def self.emoji_unicode_pattern - @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/ + @emoji_unicode_pattern ||= + begin + filtered_emojis = Gitlab::Emoji.emojis_unicodes - IGNORE_UNICODE_EMOJIS + /(#{filtered_emojis.map { |moji| Regexp.escape(moji) }.join('|')})/ + end end private diff --git a/lib/banzai/filter/normalize_source_filter.rb b/lib/banzai/filter/normalize_source_filter.rb new file mode 100644 index 00000000000..975cd540873 --- /dev/null +++ b/lib/banzai/filter/normalize_source_filter.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class NormalizeSourceFilter < HTML::Pipeline::Filter + UTF8_BOM = "\xEF\xBB\xBF" + + def call + # Remove UTF8_BOM from beginning of source text + html.delete_prefix(UTF8_BOM) + end + end + end +end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index cfd4b932568..d22a0e0b504 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -119,7 +119,7 @@ module Banzai # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. def yield_valid_link(node) - link = CGI.unescape(node.attr('href').to_s) + link = unescape_link(node.attr('href').to_s) inner_html = node.inner_html return unless link.force_encoding('UTF-8').valid_encoding? @@ -127,6 +127,10 @@ module Banzai yield link, inner_html end + def unescape_link(href) + CGI.unescape(href) + end + def replace_text_when_pattern_matches(node, index, pattern) return unless node.text =~ pattern diff --git a/lib/banzai/filter/vulnerability_reference_filter.rb b/lib/banzai/filter/vulnerability_reference_filter.rb new file mode 100644 index 00000000000..a59e9836d69 --- /dev/null +++ b/lib/banzai/filter/vulnerability_reference_filter.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # The actual filter is implemented in the EE mixin + class VulnerabilityReferenceFilter < IssuableReferenceFilter + self.reference_type = :vulnerability + + def self.object_class + Vulnerability + end + + private + + def project + context[:project] + end + end + end +end + +Banzai::Filter::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::VulnerabilityReferenceFilter') diff --git a/lib/banzai/pipeline/pre_process_pipeline.rb b/lib/banzai/pipeline/pre_process_pipeline.rb index 4c2b4ca1665..1f7cb437fcd 100644 --- a/lib/banzai/pipeline/pre_process_pipeline.rb +++ b/lib/banzai/pipeline/pre_process_pipeline.rb @@ -5,6 +5,7 @@ module Banzai class PreProcessPipeline < BasePipeline def self.filters FilterArray[ + Filter::NormalizeSourceFilter, Filter::FrontMatterFilter, Filter::BlockquoteFenceFilter, ] diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index c4d7e40b46c..3dfea8ee895 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -178,7 +178,10 @@ module Banzai collection.where(id: to_query).each { |row| cache[row.id] = row } end - ids.uniq.map { |id| cache[id] }.compact + ids.each_with_object([]) do |id, array| + row = cache[id] + array << row if row + end else collection.where(id: ids) end diff --git a/lib/banzai/reference_parser/vulnerability_parser.rb b/lib/banzai/reference_parser/vulnerability_parser.rb new file mode 100644 index 00000000000..143f2605927 --- /dev/null +++ b/lib/banzai/reference_parser/vulnerability_parser.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Banzai + module ReferenceParser + # The actual parser is implemented in the EE mixin + class VulnerabilityParser < IssuableParser + self.reference_type = :vulnerability + + def records_for_nodes(_nodes) + {} + end + end + end +end + +Banzai::ReferenceParser::VulnerabilityParser.prepend_if_ee('::EE::Banzai::ReferenceParser::VulnerabilityParser') diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index cf55c692271..8e84afe51d7 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -8,9 +8,9 @@ module BitbucketServer @connection = Connection.new(options) end - def pull_requests(project_key, repo) + def pull_requests(project_key, repo, page_offset: 0, limit: nil) path = "/projects/#{project_key}/repos/#{repo}/pull-requests?state=ALL" - get_collection(path, :pull_request) + get_collection(path, :pull_request, page_offset: page_offset, limit: limit) end def activities(project_key, repo, pull_request_id) diff --git a/lib/bulk_imports/clients/graphql.rb b/lib/bulk_imports/clients/graphql.rb new file mode 100644 index 00000000000..b067431aeae --- /dev/null +++ b/lib/bulk_imports/clients/graphql.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module BulkImports + module Clients + class Graphql + class HTTP < Graphlient::Adapters::HTTP::Adapter + def execute(document:, operation_name: nil, variables: {}, context: {}) + response = ::Gitlab::HTTP.post( + url, + headers: headers, + follow_redirects: false, + body: { + query: document.to_query_string, + operationName: operation_name, + variables: variables + }.to_json + ) + + ::Gitlab::Json.parse(response.body) + end + end + private_constant :HTTP + + attr_reader :client + + delegate :query, :parse, :execute, to: :client + + def initialize(url: Gitlab::COM_URL, token: nil) + @url = Gitlab::Utils.append_path(url, '/api/graphql') + @token = token + @client = Graphlient::Client.new( + @url, + options(http: HTTP) + ) + end + + def options(extra = {}) + return extra unless @token + + { + headers: { + 'Content-Type' => 'application/json', + 'Authorization' => "Bearer #{@token}" + } + }.merge(extra) + end + end + end +end diff --git a/lib/gitlab/bulk_import/client.rb b/lib/bulk_imports/clients/http.rb index c6e77a158cd..2e81863e53a 100644 --- a/lib/gitlab/bulk_import/client.rb +++ b/lib/bulk_imports/clients/http.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -module Gitlab - module BulkImport - class Client +module BulkImports + module Clients + class Http API_VERSION = 'v4'.freeze DEFAULT_PAGE = 1.freeze DEFAULT_PER_PAGE = 30.freeze @@ -18,7 +18,7 @@ module Gitlab end def get(resource, query = {}) - response = with_error_handling do + with_error_handling do Gitlab::HTTP.get( resource_url(resource), headers: request_headers, @@ -26,8 +26,22 @@ module Gitlab query: query.merge(request_query) ) end + end + + def each_page(method, resource, query = {}, &block) + return to_enum(__method__, method, resource, query) unless block_given? + + next_page = @page - response.parsed_response + while next_page + @page = next_page.to_i + + response = self.public_send(method, resource, query) # rubocop: disable GitlabSecurity/PublicSend + collection = response.parsed_response + next_page = response.headers['x-next-page'].presence + + yield collection + end end private diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb new file mode 100644 index 00000000000..7d58032cfcc --- /dev/null +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Extractors + class GraphqlExtractor + def initialize(query) + @query = query[:query] + @query_string = @query.to_s + @variables = @query.variables + end + + def extract(context) + @context = context + + Enumerator.new do |yielder| + result = graphql_client.execute(parsed_query, query_variables(context.entity)) + + yielder << result.original_hash.deep_dup + end + end + + private + + def graphql_client + @graphql_client ||= BulkImports::Clients::Graphql.new( + url: @context.configuration.url, + token: @context.configuration.access_token + ) + end + + def parsed_query + @parsed_query ||= graphql_client.parse(@query.to_s) + end + + def query_variables(entity) + return unless @variables + + @variables.transform_values do |entity_attribute| + entity.public_send(entity_attribute) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/bulk_imports/common/loaders/entity_loader.rb b/lib/bulk_imports/common/loaders/entity_loader.rb new file mode 100644 index 00000000000..4540b892c88 --- /dev/null +++ b/lib/bulk_imports/common/loaders/entity_loader.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Loaders + class EntityLoader + def initialize(*args); end + + def load(context, entity) + context.entity.bulk_import.entities.create!(entity) + end + end + end + end +end diff --git a/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb new file mode 100644 index 00000000000..dce0fac6999 --- /dev/null +++ b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Cleanup GraphQL original response hash from unnecessary nesting +# 1. Remove ['data']['group'] or ['data']['project'] hash nesting +# 2. Remove ['edges'] & ['nodes'] array wrappings +# 3. Remove ['node'] hash wrapping +# +# @example +# data = {"data"=>{"group"=> { +# "name"=>"test", +# "fullName"=>"test", +# "description"=>"test", +# "labels"=>{"edges"=>[{"node"=>{"title"=>"label1"}}, {"node"=>{"title"=>"label2"}}, {"node"=>{"title"=>"label3"}}]}}}} +# +# BulkImports::Common::Transformers::GraphqlCleanerTransformer.new.transform(nil, data) +# +# {"name"=>"test", "fullName"=>"test", "description"=>"test", "labels"=>[{"title"=>"label1"}, {"title"=>"label2"}, {"title"=>"label3"}]} +module BulkImports + module Common + module Transformers + class GraphqlCleanerTransformer + EDGES = 'edges' + NODE = 'node' + + def initialize(options = {}) + @options = options + end + + def transform(_, data) + return data unless data.is_a?(Hash) + + data = data.dig('data', 'group') || data.dig('data', 'project') || data + + clean_edges_and_nodes(data) + end + + def clean_edges_and_nodes(data) + case data + when Array + data.map(&method(:clean_edges_and_nodes)) + when Hash + if data.key?(NODE) + clean_edges_and_nodes(data[NODE]) + else + data.transform_values { |value| clean_edges_and_nodes(value.try(:fetch, EDGES, value) || value) } + end + else + data + end + end + end + end + end +end diff --git a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb new file mode 100644 index 00000000000..b32ab28fdbb --- /dev/null +++ b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Transformers + class UnderscorifyKeysTransformer + def initialize(options = {}) + @options = options + end + + def transform(_, data) + data.deep_transform_keys do |key| + key.to_s.underscore + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb new file mode 100644 index 00000000000..5c5e686cec5 --- /dev/null +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Extractors + class SubgroupsExtractor + def initialize(*args); end + + def extract(context) + encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) + + http_client(context.entity.bulk_import.configuration) + .each_page(:get, "groups/#{encoded_parent_path}/subgroups") + .flat_map(&:itself) + end + + private + + def http_client(configuration) + @http_client ||= BulkImports::Clients::Http.new( + uri: configuration.url, + token: configuration.access_token, + per_page: 100 + ) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb new file mode 100644 index 00000000000..c50b99aae4e --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetGroupQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + group(fullPath: $full_path) { + name + path + fullPath + description + visibility + emailsDisabled + lfsEnabled + mentionsDisabled + projectCreationLevel + requestAccessEnabled + requireTwoFactorAuthentication + shareWithGroupLock + subgroupCreationLevel + twoFactorGracePeriod + } + } + GRAPHQL + end + + def variables + { full_path: :source_full_path } + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb new file mode 100644 index 00000000000..386fc695182 --- /dev/null +++ b/lib/bulk_imports/groups/loaders/group_loader.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Loaders + class GroupLoader + def initialize(options = {}) + @options = options + end + + def load(context, data) + return unless user_can_create_group?(context.current_user, data) + + group = ::Groups::CreateService.new(context.current_user, data).execute + + context.entity.update!(group: group) + + group + end + + private + + def user_can_create_group?(current_user, data) + if data['parent_id'] + parent = Namespace.find_by_id(data['parent_id']) + + Ability.allowed?(current_user, :create_subgroup, parent) + else + Ability.allowed?(current_user, :create_group) + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb new file mode 100644 index 00000000000..2b7d0ef7658 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class GroupPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery + + transformer Common::Transformers::GraphqlCleanerTransformer + transformer Common::Transformers::UnderscorifyKeysTransformer + transformer Groups::Transformers::GroupAttributesTransformer + + loader Groups::Loaders::GroupLoader + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb new file mode 100644 index 00000000000..6384e9d5972 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class SubgroupEntitiesPipeline + include Pipeline + + extractor BulkImports::Groups::Extractors::SubgroupsExtractor + transformer BulkImports::Groups::Transformers::SubgroupToEntityTransformer + loader BulkImports::Common::Loaders::EntityLoader + end + end + end +end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb new file mode 100644 index 00000000000..7de9a430421 --- /dev/null +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Transformers + class GroupAttributesTransformer + def initialize(options = {}) + @options = options + end + + def transform(context, data) + import_entity = context.entity + + data + .then { |data| transform_name(import_entity, data) } + .then { |data| transform_path(import_entity, data) } + .then { |data| transform_full_path(data) } + .then { |data| transform_parent(context, import_entity, data) } + .then { |data| transform_visibility_level(data) } + .then { |data| transform_project_creation_level(data) } + .then { |data| transform_subgroup_creation_level(data) } + end + + private + + def transform_name(import_entity, data) + data['name'] = import_entity.destination_name + data + end + + def transform_path(import_entity, data) + data['path'] = import_entity.destination_name.parameterize + data + end + + def transform_full_path(data) + data.delete('full_path') + data + end + + def transform_parent(context, import_entity, data) + current_user = context.current_user + namespace = Namespace.find_by_full_path(import_entity.destination_namespace) + + return data if namespace == current_user.namespace + + data['parent_id'] = namespace.id + data + end + + def transform_visibility_level(data) + visibility = data['visibility'] + + return data unless visibility.present? + + data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] + data.delete('visibility') + data + end + + def transform_project_creation_level(data) + project_creation_level = data['project_creation_level'] + + return data unless project_creation_level.present? + + data['project_creation_level'] = Gitlab::Access.project_creation_string_options[project_creation_level] + data + end + + def transform_subgroup_creation_level(data) + subgroup_creation_level = data['subgroup_creation_level'] + + return data unless subgroup_creation_level.present? + + data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level] + data + end + end + end + end +end diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb new file mode 100644 index 00000000000..6c3c299c2d2 --- /dev/null +++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Transformers + class SubgroupToEntityTransformer + def initialize(*args); end + + def transform(context, entry) + { + source_type: :group_entity, + source_full_path: entry['full_path'], + destination_name: entry['name'], + destination_namespace: context.entity.group.full_path, + parent_id: context.entity.id + } + end + end + end + end +end diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb new file mode 100644 index 00000000000..c7253590c87 --- /dev/null +++ b/lib/bulk_imports/importers/group_importer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module BulkImports + module Importers + class GroupImporter + def initialize(entity) + @entity = entity + end + + def execute + entity.start! + bulk_import = entity.bulk_import + configuration = bulk_import.configuration + + context = BulkImports::Pipeline::Context.new( + current_user: bulk_import.user, + entity: entity, + configuration: configuration + ) + + BulkImports::Groups::Pipelines::GroupPipeline.new.run(context) + BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline.new.run(context) + + entity.finish! + end + + private + + attr_reader :entity + end + end +end diff --git a/lib/bulk_imports/importers/groups_importer.rb b/lib/bulk_imports/importers/groups_importer.rb new file mode 100644 index 00000000000..8641577ff47 --- /dev/null +++ b/lib/bulk_imports/importers/groups_importer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module BulkImports + module Importers + class GroupsImporter + def initialize(bulk_import_id) + @bulk_import = BulkImport.find(bulk_import_id) + end + + def execute + bulk_import.start! unless bulk_import.started? + + if entities_to_import.empty? + bulk_import.finish! + else + entities_to_import.each do |entity| + BulkImports::Importers::GroupImporter.new(entity).execute + end + + # A new BulkImportWorker job is enqueued to either + # - Process the new BulkImports::Entity created for the subgroups + # - Or to mark the `bulk_import` as finished. + BulkImportWorker.perform_async(bulk_import.id) + end + end + + private + + attr_reader :bulk_import + + def entities_to_import + @entities_to_import ||= bulk_import.entities.with_status(:created) + end + end + end +end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb new file mode 100644 index 00000000000..70e6030ea2c --- /dev/null +++ b/lib/bulk_imports/pipeline.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + extend ActiveSupport::Concern + + included do + include Attributes + include Runner + end + end +end diff --git a/lib/bulk_imports/pipeline/attributes.rb b/lib/bulk_imports/pipeline/attributes.rb new file mode 100644 index 00000000000..ebfbaf6f6ba --- /dev/null +++ b/lib/bulk_imports/pipeline/attributes.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + module Attributes + extend ActiveSupport::Concern + include Gitlab::ClassAttributes + + class_methods do + def extractor(klass, options = nil) + add_attribute(:extractors, klass, options) + end + + def transformer(klass, options = nil) + add_attribute(:transformers, klass, options) + end + + def loader(klass, options = nil) + add_attribute(:loaders, klass, options) + end + + def add_attribute(sym, klass, options) + class_attributes[sym] ||= [] + class_attributes[sym] << { klass: klass, options: options } + end + + def extractors + class_attributes[:extractors] + end + + def transformers + class_attributes[:transformers] + end + + def loaders + class_attributes[:loaders] + end + end + end + end +end diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb new file mode 100644 index 00000000000..ad19f5cad7d --- /dev/null +++ b/lib/bulk_imports/pipeline/context.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + class Context + include Gitlab::Utils::LazyAttributes + + Attribute = Struct.new(:name, :type) + + PIPELINE_ATTRIBUTES = [ + Attribute.new(:current_user, User), + Attribute.new(:entity, ::BulkImports::Entity), + Attribute.new(:configuration, ::BulkImports::Configuration) + ].freeze + + def initialize(args) + assign_attributes(args) + end + + private + + PIPELINE_ATTRIBUTES.each do |attr| + lazy_attr_reader attr.name, type: attr.type + end + + def assign_attributes(values) + values.slice(*PIPELINE_ATTRIBUTES.map(&:name)).each do |name, value| + instance_variable_set("@#{name}", value) + end + end + end + end +end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb new file mode 100644 index 00000000000..04038e50399 --- /dev/null +++ b/lib/bulk_imports/pipeline/runner.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + module Runner + extend ActiveSupport::Concern + + included do + private + + def extractors + @extractors ||= self.class.extractors.map(&method(:instantiate)) + end + + def transformers + @transformers ||= self.class.transformers.map(&method(:instantiate)) + end + + def loaders + @loaders ||= self.class.loaders.map(&method(:instantiate)) + end + + def pipeline_name + @pipeline ||= self.class.name + end + + def instantiate(class_config) + class_config[:klass].new(class_config[:options]) + end + end + + def run(context) + info(context, message: "Pipeline started", pipeline: pipeline_name) + + extractors.each do |extractor| + extractor.extract(context).each do |entry| + info(context, extractor: extractor.class.name) + + transformers.each do |transformer| + info(context, transformer: transformer.class.name) + entry = transformer.transform(context, entry) + end + + loaders.each do |loader| + info(context, loader: loader.class.name) + loader.load(context, entry) + end + end + end + end + + private # rubocop:disable Lint/UselessAccessModifier + + def info(context, extra = {}) + logger.info({ + entity: context.entity.id, + entity_type: context.entity.source_type + }.merge(extra)) + end + + def logger + @logger ||= Gitlab::Import::Logger.build + end + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index e6ca33d749b..35f299c17e4 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,6 +15,7 @@ module ContainerRegistry CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json' REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version' REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features' + REGISTRY_TAG_DELETE_FEATURE = 'tag_delete' ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze @@ -25,8 +26,6 @@ module ContainerRegistry registry_config = Gitlab.config.registry return false unless registry_config.enabled && registry_config.api_url.present? - return true if ::Gitlab.com? - token = Auth::ContainerRegistryAuthenticationService.access_token([], []) client = new(registry_config.api_url, token: token) client.supports_tag_delete? @@ -81,6 +80,9 @@ module ContainerRegistry # the DELETE method in the Allow header. Others reply with an 404 Not Found. def supports_tag_delete? strong_memoize(:supports_tag_delete) do + registry_features = Gitlab::CurrentSettings.container_registry_features || [] + next true if ::Gitlab.com? && registry_features.include?(REGISTRY_TAG_DELETE_FEATURE) + response = faraday.run_request(:options, '/v2/name/tags/reference/tag', '', {}) response.success? && response.headers['allow']&.include?('DELETE') end diff --git a/lib/csv_builders/stream.rb b/lib/csv_builders/stream.rb new file mode 100644 index 00000000000..a2b9fca84cb --- /dev/null +++ b/lib/csv_builders/stream.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module CsvBuilders + class Stream < CsvBuilder + def render(max_rows = 100_000) + max_rows_including_header = max_rows + 1 + + Enumerator.new do |csv| + csv << CSV.generate_line(headers) + + each do |object| + csv << CSV.generate_line(row(object)) + end + end.lazy.take(max_rows_including_header) # rubocop: disable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index 3a50925d628..dc8f9d0c970 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -1,17 +1,39 @@ # frozen_string_literal: true module ExpandVariables + VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/.freeze + class << self def expand(value, variables) + replace_with(value, variables) do |vars_hash, last_match| + match_or_blank_value(vars_hash, last_match) + end + end + + def expand_existing(value, variables) + replace_with(value, variables) do |vars_hash, last_match| + match_or_original_value(vars_hash, last_match) + end + end + + private + + def replace_with(value, variables) variables_hash = nil - value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + value.gsub(VARIABLES_REGEXP) do variables_hash ||= transform_variables(variables) - variables_hash[Regexp.last_match(1) || Regexp.last_match(2)] + yield(variables_hash, Regexp.last_match) end end - private + def match_or_blank_value(variables, last_match) + variables[last_match[1] || last_match[2]] + end + + def match_or_original_value(variables, last_match) + match_or_blank_value(variables, last_match) || last_match[0] + end def transform_variables(variables) # Lazily initialise variables diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index adbe93cfa3a..34511423d4a 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -68,7 +68,7 @@ module ExtractsRef raise InvalidPathError if @ref.match?(/\s/) - @commit = @repo.commit(@ref) + @commit = @repo.commit(@ref) if @ref.present? end # rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 9ec56ee6b52..1fcbc8fa173 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -10,6 +10,8 @@ class Feature # rollout_issue: defines if `bin/feature-flag` asks for rollout issue # default_enabled: defines a default state of a feature flag when created by `bin/feature-flag` # ee_only: defines that a feature flag can only be created in a context of EE + # deprecated: defines if a feature flag type that is deprecated and to be removed, + # the deprecated types are hidden from all interfaces # example: usage being shown when exception is raised TYPES = { development: { @@ -37,6 +39,7 @@ class Feature }, licensed: { description: 'Permanent feature flags used to temporarily disable licensed features.', + deprecated: true, optional: true, rollout_issue: false, ee_only: true, @@ -54,6 +57,7 @@ class Feature name introduced_by_url rollout_issue_url + milestone type group default_enabled diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index c5f843d5f1a..4bb225b63f1 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -18,24 +18,46 @@ module Gitlab end def timestamp_projection - issue_metrics_table[:first_mentioned_in_commit_at] + Arel::Nodes::NamedFunction.new('COALESCE', column_list) end override :column_list def column_list - [timestamp_projection] + [ + issue_metrics_table[:first_mentioned_in_commit_at], + mr_metrics_table[:first_commit_at] + ] end # rubocop: disable CodeReuse/ActiveRecord def apply_query_customization(query) - issue_metrics_join = mr_closing_issues_table - .join(issue_metrics_table) + query + .joins(merge_requests_closing_issues_join) + .joins(issue_metrics_join) + .joins(mr_metrics_join) + end + # rubocop: enable CodeReuse/ActiveRecord + + def issue_metrics_join + mr_closing_issues_table + .join(issue_metrics_table, Arel::Nodes::OuterJoin) .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id])) .join_sources + end - query.joins(:merge_requests_closing_issues).joins(issue_metrics_join) + def merge_requests_closing_issues_join + mr_table + .join(mr_closing_issues_table, Arel::Nodes::OuterJoin) + .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) + .join_sources + end + + def mr_metrics_join + mr_metrics_table + .join(mr_metrics_table, Arel::Nodes::OuterJoin) + .on(mr_metrics_table[:merge_request_id].eq(mr_table[:id])) + .join_sources end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb index 636bba22c23..54b3bbb3ce6 100644 --- a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb +++ b/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb @@ -11,21 +11,36 @@ module Gitlab def execute measurement_identifiers.map do |measurement_identifier| - query_scope = ::Analytics::InstanceStatistics::Measurement::IDENTIFIER_QUERY_MAPPING[measurement_identifier]&.call + query_scope = query_mappings[measurement_identifier]&.call next if query_scope.nil? - # Determining the query range (id range) as early as possible in order to get more accurate counts. - start = query_scope.minimum(:id) - finish = query_scope.maximum(:id) - - [measurement_identifier, start, finish, recorded_at] + [measurement_identifier, *determine_start_and_finish(measurement_identifier, query_scope), recorded_at] end.compact end private attr_reader :measurement_identifiers, :recorded_at + + # Determining the query range (id range) as early as possible in order to get more accurate counts. + def determine_start_and_finish(measurement_identifier, query_scope) + queries = custom_min_max_queries[measurement_identifier] + + if queries + [queries[:minimum_query].call, queries[:maximum_query].call] + else + [query_scope.minimum(:id), query_scope.maximum(:id)] + end + end + + def custom_min_max_queries + ::Analytics::InstanceStatistics::Measurement.identifier_min_max_queries + end + + def query_mappings + ::Analytics::InstanceStatistics::Measurement.identifier_query_mapping + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 6173918b453..e92bbe4f529 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -33,7 +33,8 @@ module Gitlab group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, - profile_resend_email_confirmation: { threshold: 5, interval: 1.minute } + profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, + update_environment_canary_ingress: { threshold: 1, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 001c083c778..fadd6eb848d 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -158,7 +158,7 @@ module Gitlab if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included - # in the Service.available_services_names whitelist. + # in the Service.available_services_names allowlist. service = project.public_send("#{underscored_service}_service") # rubocop:disable GitlabSecurity/PublicSend if service && service.activated? && service.valid_token?(password) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 3d3f7212053..f3975fe219a 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -83,6 +83,8 @@ module Gitlab return unless ::Gitlab::Auth::CI_JOB_USER == login job = find_valid_running_job_by_token!(password) + @current_authenticated_job = job # rubocop:disable Gitlab/ModuleWithInstanceVariables + job.user end diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb new file mode 100644 index 00000000000..553571d5d00 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill design.iid for a range of projects + class BackfillDesignInternalIds + # See app/models/internal_id + # This is a direct copy of the application code with the following changes: + # - usage enum is hard-coded to the value for design_management_designs + # - init is not passed around, but ignored + class InternalId < ActiveRecord::Base + def self.track_greatest(subject, scope, new_value) + InternalIdGenerator.new(subject, scope).track_greatest(new_value) + end + + # Increments #last_value with new_value if it is greater than the current, + # and saves the record + # + # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). + # As such, the increment is atomic and safe to be called concurrently. + def track_greatest_and_save!(new_value) + update_and_save { self.last_value = [last_value || 0, new_value].max } + end + + private + + def update_and_save(&block) + lock! + yield + # update_and_save_counter.increment(usage: usage, changed: last_value_changed?) + save! + last_value + end + end + + # See app/models/internal_id + class InternalIdGenerator + attr_reader :subject, :scope, :scope_attrs + + def initialize(subject, scope) + @subject = subject + @scope = scope + + raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? + end + + # Create a record in internal_ids if one does not yet exist + # and set its new_value if it is higher than the current last_value + # + # Note this will acquire a ROW SHARE lock on the InternalId record + def track_greatest(new_value) + subject.transaction do + record.track_greatest_and_save!(new_value) + end + end + + def record + @record ||= (lookup || create_record) + end + + def lookup + InternalId.find_by(**scope, usage: usage_value) + end + + def usage_value + 10 # see Enums::InternalId - this is the value for design_management_designs + end + + # Create InternalId record for (scope, usage) combination, if it doesn't exist + # + # We blindly insert without synchronization. If another process + # was faster in doing this, we'll realize once we hit the unique key constraint + # violation. We can safely roll-back the nested transaction and perform + # a lookup instead to retrieve the record. + def create_record + subject.transaction(requires_new: true) do + InternalId.create!( + **scope, + usage: usage_value, + last_value: 0 + ) + end + rescue ActiveRecord::RecordNotUnique + lookup + end + end + + attr_reader :design_class + + def initialize(design_class) + @design_class = design_class + end + + def perform(relation) + start_id, end_id = relation.pluck("min(project_id), max(project_id)").flatten + table = 'design_management_designs' + + ActiveRecord::Base.connection.execute <<~SQL + WITH + starting_iids(project_id, iid) as ( + SELECT project_id, MAX(COALESCE(iid, 0)) + FROM #{table} + WHERE project_id BETWEEN #{start_id} AND #{end_id} + GROUP BY project_id + ), + with_calculated_iid(id, iid) as ( + SELECT design.id, + init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC) + FROM #{table} as design, starting_iids as init + WHERE design.project_id BETWEEN #{start_id} AND #{end_id} + AND design.iid IS NULL + AND init.project_id = design.project_id + ) + + UPDATE #{table} + SET iid = with_calculated_iid.iid + FROM with_calculated_iid + WHERE #{table}.id = with_calculated_iid.id + SQL + + # track the new greatest IID value + relation.each do |design| + current_max = design_class.where(project_id: design.project_id).maximum(:iid) + scope = { project_id: design.project_id } + InternalId.track_greatest(design, scope, current_max) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb new file mode 100644 index 00000000000..61145f6a445 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2, +# it's enough at the moment to simply notice if the url is from `atlassian.net` +module Gitlab + module BackgroundMigration + # Backfill the deployment_type in jira_tracker_data table + class BackfillJiraTrackerDeploymentType2 + # Migration only version of jira_tracker_data table + class JiraTrackerDataTemp < ApplicationRecord + self.table_name = 'jira_tracker_data' + + def self.encryption_options + { + key: Settings.attr_encrypted_db_key_base_32, + encode: true, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm' + } + end + + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + + enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment + end + + # Migration only version of services table + class JiraServiceTemp < ApplicationRecord + self.table_name = 'services' + self.inheritance_column = :_type_disabled + end + + def perform(start_id, stop_id) + @server_ids = [] + @cloud_ids = [] + + JiraTrackerDataTemp + .where(id: start_id..stop_id, deployment_type: 0) + .each do |jira_tracker_data| + collect_deployment_type(jira_tracker_data) + end + + unless cloud_ids.empty? + JiraTrackerDataTemp.where(id: cloud_ids) + .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud]) + end + + unless server_ids.empty? + JiraTrackerDataTemp.where(id: server_ids) + .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server]) + end + + mark_jobs_as_succeeded(start_id, stop_id) + end + + private + + attr_reader :server_ids, :cloud_ids + + def client_url(jira_tracker_data) + jira_tracker_data.api_url.presence || jira_tracker_data.url.presence + end + + def server_type(url) + url.downcase.include?('.atlassian.net') ? :cloud : :server + end + + def collect_deployment_type(jira_tracker_data) + url = client_url(jira_tracker_data) + return unless url + + case server_type(url) + when :cloud + cloud_ids << jira_tracker_data.id + else + server_ids << jira_tracker_data.id + end + end + + def mark_jobs_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb new file mode 100644 index 00000000000..8a58cf9b302 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_merge_request_cleanup_schedules.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill merge request cleanup schedules of closed/merged merge requests + # without any corresponding records. + class BackfillMergeRequestCleanupSchedules + # Model used for migration added in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46782. + class MergeRequest < ActiveRecord::Base + include EachBatch + + self.table_name = 'merge_requests' + + def self.eligible + where('merge_requests.state_id IN (2, 3)') + end + end + + def perform(start_id, end_id) + eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id) + scheduled_at_column = "COALESCE(metrics.merged_at, COALESCE(metrics.latest_closed_at, merge_requests.updated_at)) + interval '14 days'" + query = + eligible_mrs + .select("merge_requests.id, #{scheduled_at_column}, NOW(), NOW()") + .joins('LEFT JOIN merge_request_metrics metrics ON metrics.merge_request_id = merge_requests.id') + + result = ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO merge_request_cleanup_schedules (merge_request_id, scheduled_at, created_at, updated_at) + #{query.to_sql} + ON CONFLICT (merge_request_id) DO NOTHING; + SQL + + ::Gitlab::BackgroundMigration::Logger.info( + message: 'Backfilled merge_request_cleanup_schedules records', + count: result.cmd_tuples + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb new file mode 100644 index 00000000000..78140b768fc --- /dev/null +++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates missing dismissal information for + # vulnerability entries. + class PopulateHasVulnerabilities + class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'project_settings' + + UPSERT_SQL = <<~SQL + WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS ( + SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) + ) + INSERT INTO project_settings + (project_id, has_vulnerabilities, created_at, updated_at) + (SELECT * FROM upsert_data) + ON CONFLICT (project_id) + DO UPDATE SET + has_vulnerabilities = true, + updated_at = EXCLUDED.updated_at + SQL + + def self.upsert_for(project_ids) + connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') }) + end + end + + class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerabilities' + end + + def perform(*project_ids) + ProjectSetting.upsert_for(project_ids) + rescue StandardError => e + log_error(e, project_ids) + ensure + log_info(project_ids) + end + + private + + def log_error(error, project_ids) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + project_ids: project_ids + ) + end + + def log_info(project_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Projects has been processed to populate `has_vulnerabilities` information', + count: project_ids.length + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb new file mode 100644 index 00000000000..bc0a181a06c --- /dev/null +++ b/lib/gitlab/background_migration/populate_missing_vulnerability_dismissal_information.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates missing dismissal information for + # vulnerability entries. + class PopulateMissingVulnerabilityDismissalInformation + class Vulnerability < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerabilities' + + has_one :finding, class_name: '::Gitlab::BackgroundMigration::PopulateMissingVulnerabilityDismissalInformation::Finding' + + scope :broken, -> { where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } + + def copy_dismissal_information + return unless finding&.dismissal_feedback + + update_columns( + dismissed_at: finding.dismissal_feedback.created_at, + dismissed_by_id: finding.dismissal_feedback.author_id + ) + end + end + + class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation + include ShaAttribute + + self.table_name = 'vulnerability_occurrences' + + sha_attribute :project_fingerprint + + def dismissal_feedback + Feedback.dismissal.where(category: report_type, project_fingerprint: project_fingerprint, project_id: project_id).first + end + end + + class Feedback < ActiveRecord::Base # rubocop:disable Style/Documentation + DISMISSAL_TYPE = 0 + + self.table_name = 'vulnerability_feedback' + + scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } + end + + def perform(*vulnerability_ids) + Vulnerability.includes(:finding).where(id: vulnerability_ids).each { |vulnerability| populate_for(vulnerability) } + + log_info(vulnerability_ids) + end + + private + + def populate_for(vulnerability) + log_warning(vulnerability) unless vulnerability.copy_dismissal_information + rescue StandardError => error + log_error(error, vulnerability) + end + + def log_info(vulnerability_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Dismissal information has been copied', + count: vulnerability_ids.length + ) + end + + def log_warning(vulnerability) + ::Gitlab::BackgroundMigration::Logger.warn( + migrator: self.class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability.id + ) + end + + def log_error(error, vulnerability) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + vulnerability_id: vulnerability.id + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb new file mode 100644 index 00000000000..fc79f7125e3 --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_feedback_pipeline_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class updates vulnerability feedback entities with no pipeline id assigned. + class PopulateVulnerabilityFeedbackPipelineId + def perform(project_ids) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityFeedbackPipelineId') diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb index 26626aaef79..0c29887bb00 100644 --- a/lib/gitlab/background_migration/replace_blocked_by_links.rb +++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb @@ -12,14 +12,19 @@ module Gitlab blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2) ActiveRecord::Base.transaction do - # if there is duplicit bi-directional relation (issue2 is blocked by issue1 - # and issue1 already links issue2), then we can just delete 'blocked by'. - # This should be rare as we have a pre-create check which checks if issues are - # already linked - blocked_by_links + # There could be two edge cases: + # 1) issue1 is blocked by issue2 AND issue2 blocks issue1 (type 1) + # 2) issue1 is blocked by issue2 AND issue2 is related to issue1 (type 0) + # In both cases cases we couldn't convert blocked by relation to + # `issue2 blocks issue` because there is already a link with the same + # source/target id. To avoid these conflicts, we first delete any + # "opposite" links before we update `blocked by` relation. This + # should be rare as we have a pre-create check which checks if issues + # are already linked + opposite_ids = blocked_by_links + .select('opposite_links.id') .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id') - .where('opposite_links.link_type': 1) - .delete_all + IssueLink.where(id: opposite_ids).delete_all blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1') end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 0df6e858bf4..390da014a5a 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -17,8 +17,6 @@ module Gitlab key_width: opts[:key_width].to_i, key_text: opts[:key_text] } - - @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end def entity @@ -42,19 +40,35 @@ module Gitlab private - # rubocop: disable CodeReuse/ActiveRecord + def successful_pipeline + @successful_pipeline ||= @project.ci_pipelines.latest_successful_for_ref(@ref) + end + + def failed_pipeline + @failed_pipeline ||= @project.ci_pipelines.latest_failed_for_ref(@ref) + end + + def running_pipeline + @running_pipeline ||= @project.ci_pipelines.latest_running_for_ref(@ref) + end + def raw_coverage - return unless @pipeline + latest = + if @job.present? + builds = ::Ci::Build + .in_pipelines([successful_pipeline, running_pipeline, failed_pipeline]) + .latest + .success + .for_ref(@ref) + .by_name(@job) + + builds.max_by(&:created_at) + else + successful_pipeline + end - if @job.blank? - @pipeline.coverage - else - @pipeline.builds - .find_by(name: @job) - .try(:coverage) - end + latest&.coverage end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index aca5a63a424..d29799f1029 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -4,11 +4,14 @@ module Gitlab module BitbucketServerImport class Importer attr_reader :recover_missing_commits - attr_reader :project, :project_key, :repository_slug, :client, :errors, :users + attr_reader :project, :project_key, :repository_slug, :client, :errors, :users, :already_imported_cache_key attr_accessor :logger REMOTE_NAME = 'bitbucket_server' BATCH_SIZE = 100 + # The base cache key to use for tracking already imported objects. + ALREADY_IMPORTED_CACHE_KEY = + 'bitbucket_server-importer/already-imported/%{project}/%{collection}' TempBranch = Struct.new(:name, :sha) @@ -36,17 +39,25 @@ module Gitlab @users = {} @temp_branches = [] @logger = Gitlab::Import::Logger.build + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def collection_method + :pull_requests end def execute import_repository import_pull_requests + download_lfs_objects delete_temp_branches handle_errors metrics.track_finished_import log_info(stage: "complete") + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, 15.minutes.to_i) true end @@ -148,6 +159,14 @@ module Gitlab raise end + def download_lfs_objects + result = Projects::LfsPointers::LfsImportService.new(project).execute + + if result[:status] == :error + errors << { type: :lfs_objects, errors: "The Lfs import process failed. #{result[:message]}" } + end + end + # Bitbucket Server keeps tracks of references for open pull requests in # refs/heads/pull-requests, but closed and merged requests get moved # into hidden internal refs under stash-refs/pull-requests. Unless the @@ -158,16 +177,29 @@ module Gitlab # on the remote server. Then we have to issue a `git fetch` to download these # branches. def import_pull_requests - pull_requests = client.pull_requests(project_key, repository_slug).to_a + page = 0 + + log_info(stage: 'import_pull_requests', message: "starting") + + loop do + log_debug(stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") + + pull_requests = client.pull_requests(project_key, repository_slug, page_offset: page, limit: BATCH_SIZE).to_a - # Creating branches on the server and fetching the newly-created branches - # may take a number of network round-trips. Do this in batches so that we can - # avoid doing a git fetch for every new branch. - pull_requests.each_slice(BATCH_SIZE) do |batch| - restore_branches(batch) if recover_missing_commits + break if pull_requests.empty? - batch.each do |pull_request| - import_bitbucket_pull_request(pull_request) + # Creating branches on the server and fetching the newly-created branches + # may take a number of network round-trips. This used to be done in batches to + # avoid doing a git fetch for every new branch, as the whole process is now + # batched, we do not need to separately do this in batches. + restore_branches(pull_requests) if recover_missing_commits + + pull_requests.each do |pull_request| + if already_imported?(pull_request) + log_info(stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) + else + import_bitbucket_pull_request(pull_request) + end rescue StandardError => e Gitlab::ErrorTracking.log_exception( e, @@ -177,9 +209,25 @@ module Gitlab backtrace = Gitlab::BacktraceCleaner.clean_backtrace(e.backtrace) errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } end + + log_debug(stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") + page += 1 end end + # Returns true if the given object has already been imported, false + # otherwise. + # + # object - The object to check. + def already_imported?(pull_request) + Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, pull_request.iid) + end + + # Marks the given object as "already imported". + def mark_as_imported(pull_request) + Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, pull_request.iid) + end + def delete_temp_branches @temp_branches.each do |branch| client.delete_branch(project_key, repository_slug, branch.name, branch.sha) @@ -227,6 +275,7 @@ module Gitlab end log_info(stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) + mark_as_imported(pull_request) end def import_pull_request_comments(pull_request, merge_request) @@ -378,6 +427,10 @@ module Gitlab } end + def log_debug(details) + logger.debug(log_base_data.merge(details)) + end + def log_info(details) logger.info(log_base_data.merge(details)) end diff --git a/lib/gitlab/chat/output.rb b/lib/gitlab/chat/output.rb index 411b1555a7d..4a55b81a9eb 100644 --- a/lib/gitlab/chat/output.rb +++ b/lib/gitlab/chat/output.rb @@ -12,7 +12,10 @@ module Gitlab PRIMARY_SECTION = 'chat_reply' # The backup trace section in case the primary one could not be found. - FALLBACK_SECTION = 'build_script' + FALLBACK_SECTION = 'step_script' + + # `step_script` used to be `build_script` before runner 13.1 + LEGACY_SECTION = 'build_script' # build - The `Ci::Build` to obtain the output from. def initialize(build) @@ -37,24 +40,6 @@ module Gitlab end end - # Returns the offset to seek to and the number of bytes to read relative - # to the offset. - def read_offset_and_length - section = find_build_trace_section(PRIMARY_SECTION) || - find_build_trace_section(FALLBACK_SECTION) - - unless section - raise( - MissingBuildSectionError, - "The build_script trace section could not be found for build #{build.id}" - ) - end - - length = section[:byte_end] - section[:byte_start] - - [section[:byte_start], length] - end - # Removes the line containing the executed command from the build output. # # output - A `String` containing the output of a trace section. @@ -88,6 +73,27 @@ module Gitlab def trace @trace ||= build.trace end + + private + + # Returns the offset to seek to and the number of bytes to read relative + # to the offset. + def read_offset_and_length + section = find_build_trace_section(PRIMARY_SECTION) || + find_build_trace_section(FALLBACK_SECTION) || + find_build_trace_section(LEGACY_SECTION) + + unless section + raise( + MissingBuildSectionError, + "The build_script trace section could not be found for build #{build.id}" + ) + end + + length = section[:byte_end] - section[:byte_start] + + [section[:byte_start], length] + end end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 728a66ca87f..cbecce57163 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -11,12 +11,22 @@ module Gitlab def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? + expanded_globs = expand_globs(pipeline, context) pipeline.modified_paths.any? do |path| - @globs.any? do |glob| + expanded_globs.any? do |glob| File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) end end end + + def expand_globs(pipeline, context) + return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project, default_enabled: true) + return @globs unless context + + @globs.map do |glob| + ExpandVariables.expand_existing(glob, context.variables) + end + end end end end diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 3fbfdffe277..25fb9c0ca97 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -3,38 +3,8 @@ module Gitlab module Ci module Charts - module DailyInterval - # rubocop: disable CodeReuse/ActiveRecord - def grouped_count(query) - query - .group("DATE(#{::Ci::Pipeline.table_name}.created_at)") - .count(:created_at) - .transform_keys { |date| date.strftime(@format) } # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - # rubocop: enable CodeReuse/ActiveRecord - - def interval_step - @interval_step ||= 1.day - end - end - - module MonthlyInterval - # rubocop: disable CodeReuse/ActiveRecord - def grouped_count(query) - query - .group("to_char(#{::Ci::Pipeline.table_name}.created_at, '01 Month YYYY')") - .count(:created_at) - .transform_keys(&:squish) - end - # rubocop: enable CodeReuse/ActiveRecord - - def interval_step - @interval_step ||= 1.month - end - end - class Chart - attr_reader :labels, :total, :success, :project, :pipeline_times + attr_reader :from, :to, :labels, :total, :success, :project, :pipeline_times def initialize(project) @labels = [] @@ -46,48 +16,59 @@ module Gitlab collect end + private + + attr_reader :interval + # rubocop: disable CodeReuse/ActiveRecord def collect query = project.all_pipelines - .where("? > #{::Ci::Pipeline.table_name}.created_at AND #{::Ci::Pipeline.table_name}.created_at > ?", @to, @from) # rubocop:disable GitlabSecurity/SqlInjection + .where(::Ci::Pipeline.arel_table['created_at'].gteq(@from)) + .where(::Ci::Pipeline.arel_table['created_at'].lteq(@to)) totals_count = grouped_count(query) success_count = grouped_count(query.success) current = @from - while current < @to - label = current.strftime(@format) - - @labels << label - @total << (totals_count[label] || 0) - @success << (success_count[label] || 0) + while current <= @to + @labels << current.strftime(@format) + @total << (totals_count[current] || 0) + @success << (success_count[current] || 0) current += interval_step end end # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def grouped_count(query) + query + .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)") + .count(:created_at) + end + # rubocop: enable CodeReuse/ActiveRecord + + def interval_step + @interval_step ||= 1.public_send(interval) # rubocop: disable GitlabSecurity/PublicSend + end end class YearChart < Chart - include MonthlyInterval - attr_reader :to, :from - def initialize(*) @to = Date.today.end_of_month.end_of_day - @from = @to.years_ago(1).beginning_of_month.beginning_of_day - @format = '%d %B %Y' + @from = (@to - 1.year).beginning_of_month.beginning_of_day + @interval = :month + @format = '%B %Y' super end end class MonthChart < Chart - include DailyInterval - attr_reader :to, :from - def initialize(*) @to = Date.today.end_of_day - @from = 1.month.ago.beginning_of_day + @from = (@to - 1.month).beginning_of_day + @interval = :day @format = '%d %B' super @@ -95,12 +76,10 @@ module Gitlab end class WeekChart < Chart - include DailyInterval - attr_reader :to, :from - def initialize(*) @to = Date.today.end_of_day - @from = 1.week.ago.beginning_of_day + @from = (@to - 1.week).beginning_of_day + @interval = :day @format = '%d %B' super diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index 1740032e5c7..70fcc1d586a 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -18,7 +18,6 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do - validates :allow_failure, boolean: true validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -48,7 +47,7 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - attributes :when, :allow_failure + attributes :when def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -60,14 +59,6 @@ module Gitlab true end - def manual_action? - self.when == 'manual' - end - - def ignored? - allow_failure.nil? ? manual_action? : allow_failure - end - def value super.merge( trigger: (trigger_value if trigger_defined?), diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index ecc2c5cb729..1ce7060df22 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -24,7 +24,6 @@ module Gitlab validates :script, presence: true with_options allow_nil: true do - validates :allow_failure, boolean: true validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -118,7 +117,7 @@ module Gitlab description: 'Parallel configuration for this job.', inherit: false - attributes :script, :tags, :allow_failure, :when, :dependencies, + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release @@ -141,18 +140,10 @@ module Gitlab end end - def manual_action? - self.when == 'manual' - end - def delayed? self.when == 'delayed' end - def ignored? - allow_failure.nil? ? manual_action? : allow_failure - end - def value super.merge( before_script: before_script_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index f10c509d0cc..c0315e5f901 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -32,6 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :allow_failure, boolean: true end end @@ -64,7 +65,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules + attributes :extends, :rules, :allow_failure end def compose!(deps = nil) @@ -136,6 +137,14 @@ module Gitlab root_variables.merge(variables_value.to_h) end + + def manual_action? + self.when == 'manual' + end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index 2481989060e..aa34cfb3acc 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: :minimum, + minimum: 1, too_short: 'requires at least %{count} items' } end @@ -28,10 +28,6 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end - - def minimum - ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 - end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 97ae6c4ceba..90692eafc3f 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -33,6 +33,7 @@ module Gitlab locations .compact .map(&method(:normalize_location)) + .flat_map(&method(:expand_project_files)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -52,6 +53,15 @@ module Gitlab end end + def expand_project_files(location) + return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true) + return location unless location[:project] + + Array.wrap(location[:file]).map do |file| + location.merge(file: file) + end + end + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) { remote: location } diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 1b58e3ec71a..661189eea50 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -32,16 +32,12 @@ module Gitlab end # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` - # is a safe switch to disable the feature for a parituclar project when something went wrong, + # is a safe switch to disable the feature for a particular project when something went wrong, # therefore it's not supposed to be enabled by default. def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project) ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.lint_creates_pipeline_with_dry_run?(project) - ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true) - end - def self.project_transactionless_destroy?(project) Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end @@ -59,13 +55,21 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.one_dimensional_matrix_enabled? - ::Feature.enabled?(:one_dimensional_matrix, default_enabled: true) - end - def self.manual_bridges_enabled?(project) ::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true) end + + def self.auto_rollback_available?(project) + ::Feature.enabled?(:cd_auto_rollback, project) && project&.feature_available?(:auto_rollback) + end + + def self.seed_block_run_before_workflow_rules_enabled?(project) + ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true) + end + + def self.ci_pipeline_editor_page_enabled?(project) + ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 491facd0a43..a8943eadf4f 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -6,6 +6,8 @@ module Gitlab NOT_BEFORE_TIME = 5 DEFAULT_EXPIRE_TIME = 60 * 5 + NoSigningKeyError = Class.new(StandardError) + def self.for_build(build) self.new(build, ttl: build.metadata_timeout).encoded end @@ -27,7 +29,7 @@ module Gitlab private - attr_reader :build, :ttl, :key_data + attr_reader :build, :ttl def reserved_claims now = Time.now.to_i @@ -60,7 +62,17 @@ module Gitlab end def key - @key ||= OpenSSL::PKey::RSA.new(Rails.application.secrets.openid_connect_signing_key) + @key ||= begin + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + Gitlab::CurrentSettings.ci_jwt_signing_key + else + Rails.application.secrets.openid_connect_signing_key + end + + raise NoSigningKeyError unless key_data + + OpenSSL::PKey::RSA.new(key_data) + end end def public_key diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 44f2ac23ce3..fb795152abe 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -24,7 +24,7 @@ module Gitlab end def validate(content, dry_run: false) - if dry_run && Gitlab::Ci::Features.lint_creates_pipeline_with_dry_run?(@project) + if dry_run simulate_pipeline_creation(content) else static_validation(content) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 468f3bc4689..a864c843dd8 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -25,7 +25,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - project.ci_pipelines + pipelines .where(ref: pipeline.ref) .where.not(id: pipeline.same_family_pipeline_ids) .where.not(sha: project.commit(pipeline.ref).try(:id)) @@ -33,6 +33,14 @@ module Gitlab .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord + + def pipelines + if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: false) + project.all_pipelines.ci_and_parent_sources + else + project.ci_pipelines + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index e10a0bc3718..ba86b08d209 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -19,10 +19,12 @@ module Gitlab # Build to prevent erroring out on ambiguous refs. pipeline.protected = @command.protected_ref? - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) + unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + end ## # Gather all runtime build/stage errors diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb new file mode 100644 index 00000000000..f8e62949bea --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class SeedBlock < Chain::Base + include Chain::Helpers + include Gitlab::Utils::StrongMemoize + + def perform! + return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) + + ## + # Populate pipeline with block argument of CreatePipelineService#execute. + # + @command.seeds_block&.call(pipeline) + + raise "Pipeline cannot be persisted by `seeds_block`" if pipeline.persisted? + end + + def break? + return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) + + pipeline.errors.any? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 42e8c365824..b20dc383419 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -12,12 +12,23 @@ module Gitlab end def to_resource - job.project.environments - .safe_find_or_create_by(name: expanded_environment_name) + environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| + environment.auto_stop_in = auto_stop_in + end end private + def environments + job.project.environments + end + + def auto_stop_in + if Feature.enabled?(:environment_auto_stop_start_on_create) + job.environment_auto_stop_in + end + end + def expanded_environment_name job.expanded_environment_name end diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index 8c70dbb6931..09121191047 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -10,7 +10,7 @@ module Gitlab STATUS_ERROR = 'error' STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze - attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job + attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job, :recent_failures def initialize(params) @suite_name = params.fetch(:suite_name) @@ -24,9 +24,15 @@ module Gitlab @attachment = params.fetch(:attachment, nil) @job = params.fetch(:job, nil) + @recent_failures = nil + @key = hash_key("#{suite_name}_#{classname}_#{name}") end + def set_recent_failures(count, base_branch) + @recent_failures = { count: count, base_branch: base_branch } + end + def has_attachment? attachment.present? end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb new file mode 100644 index 00000000000..beceac5423a --- /dev/null +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class TestFailureHistory + include Gitlab::Utils::StrongMemoize + + def initialize(failed_test_cases, project) + @failed_test_cases = build_map(failed_test_cases) + @project = project + end + + def load! + return unless Feature.enabled?(:test_failure_history, project) + + recent_failures_count.each do |key_hash, count| + failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) + end + end + + private + + attr_reader :report, :project, :failed_test_cases + + def recent_failures_count + ::Ci::TestCaseFailure.recent_failures_count( + project: project, + test_case_keys: failed_test_cases.keys + ) + end + + def build_map(test_cases) + {}.tap do |hash| + test_cases.each do |test_case| + hash[test_case.key] = test_case + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index a58de43e55e..239fc3b15e7 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -6,6 +6,9 @@ module Gitlab class TestSuiteComparer include Gitlab::Utils::StrongMemoize + DEFAULT_MAX_TESTS = 100 + DEFAULT_MIN_TESTS = 10 + attr_reader :name, :base_suite, :head_suite def initialize(name, base_suite, head_suite) @@ -81,6 +84,29 @@ module Gitlab def error_count new_errors.count + existing_errors.count end + + # This is used to limit the presented test cases but does not affect + # total count of tests in the summary + def limited_tests + strong_memoize(:limited_tests) do + # rubocop: disable CodeReuse/ActiveRecord + OpenStruct.new( + new_failures: new_failures.take(max_tests), + existing_failures: existing_failures.take(max_tests(new_failures)), + resolved_failures: resolved_failures.take(max_tests(new_failures, existing_failures)), + new_errors: new_errors.take(max_tests), + existing_errors: existing_errors.take(max_tests(new_errors)), + resolved_errors: resolved_errors.take(max_tests(new_errors, existing_errors)) + ) + # rubocop: enable CodeReuse/ActiveRecord + end + end + + private + + def max_tests(*used) + [DEFAULT_MAX_TESTS - used.map(&:count).sum, DEFAULT_MIN_TESTS].max + end end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 2171637687f..dd0bfa768a8 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -106,7 +106,7 @@ module Gitlab end def get_file(path) - File.read(path) + File.read(Rails.root.join(path).to_s) end def registration_token diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 82b2f5c035e..453803a6f7e 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -4,6 +4,7 @@ stages: - review - deploy - production + - cleanup variables: AUTO_DEVOPS_PLATFORM_TARGET: ECS diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index cba13f374f4..a13f2046291 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -160,9 +160,10 @@ include: - template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml - template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml - template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml - - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml + - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml - template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml - - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml + - template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml + - template: Jobs/Deploy/EC2.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml - template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 0c3598a61a7..1c25d9d583b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -16,4 +16,14 @@ build: fi - /build/build.sh rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' + when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +build_artifact: + stage: build + script: + - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n" + - exit 1 + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"' diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index ec33020205b..fe23641802b 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18" needs: [] script: - export SOURCE_CODE=$PWD @@ -34,6 +34,7 @@ code_quality: CODECLIMATE_DEBUG \ CODECLIMATE_DEV \ REPORT_STDOUT \ + REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ ) \ --volume "$PWD":/code \ diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 33d77e39bc9..c4e194bd658 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -186,7 +186,7 @@ production_manual: when: never - if: '$CI_COMMIT_BRANCH != "master"' when: never - # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' when: manual diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 8b921305c11..385959389de 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -183,7 +183,7 @@ production_manual: when: never - if: '$CI_COMMIT_BRANCH != "master"' when: never - # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax + # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' when: manual diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 317e8bfab0e..0289ba1c473 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -8,8 +8,11 @@ # # More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate -.deploy_to_ecs: +.ecs_image: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + +.deploy_to_ecs: + extends: .ecs_image dependencies: [] script: - ecs update-task-definition @@ -17,8 +20,6 @@ .review_ecs_base: stage: review extends: .deploy_to_ecs - environment: - name: review/$CI_COMMIT_REF_NAME .production_ecs_base: stage: production @@ -26,8 +27,18 @@ environment: name: production +.stop_review_ecs_base: + extends: .ecs_image + stage: cleanup + allow_failure: true + script: + - ecs stop-task + review_ecs: extends: .review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + on_stop: stop_review_ecs rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' when: never @@ -39,8 +50,46 @@ review_ecs: when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' +stop_review_ecs: + extends: .stop_review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual + review_fargate: extends: .review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + on_stop: stop_review_fargate + script: + - ecs update-task-definition + rules: + - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' + when: never + - if: '$CI_KUBERNETES_ACTIVE' + when: never + - if: '$REVIEW_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + +stop_review_fargate: + extends: .stop_review_ecs_base + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop rules: - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "FARGATE"' when: never @@ -51,6 +100,7 @@ review_fargate: - if: '$CI_COMMIT_BRANCH == "master"' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' + when: manual production_ecs: extends: .production_ecs_base diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index a9638f564f3..3f62d92ad13 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.33.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.34.1" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index c3a92b67a8b..0ae8fd833c4 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,3 +1,9 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + stages: - build - test @@ -7,7 +13,7 @@ stages: variables: FUZZAPI_PROFILE: Quick FUZZAPI_VERSION: latest - FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml" + FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 FUZZAPI_REPORT: gl-api-fuzzing-report.xml # @@ -17,9 +23,70 @@ variables: # available (non 500 response to HTTP(s)) FUZZAPI_SERVICE_START_TIMEOUT: "300" # + FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + # + +apifuzzer_fuzz_unlicensed: + stage: fuzz + allow_failure: true + rules: + - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' + - when: never + script: + - | + echo "Error: Your GitLab project is not licensed for API Fuzzing." + - exit 1 apifuzzer_fuzz: stage: fuzz + image: + name: $FUZZAPI_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + TZ: America/Los_Angeles + services: + - name: $FUZZAPI_IMAGE + alias: apifuzzer + entrypoint: ["dotnet", "/peach/Peach.Web.dll"] + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE + when: never + - if: $FUZZAPI_D_WORKER_IMAGE + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + script: + # + # Validate options + - | + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + artifacts: + reports: + junit: $FUZZAPI_REPORT + +apifuzzer_fuzz_dnd: + stage: fuzz image: docker:19.03.12 variables: DOCKER_DRIVER: overlay2 @@ -28,20 +95,19 @@ apifuzzer_fuzz: FUZZAPI_API: http://apifuzzer:80 allow_failure: true rules: + - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null + when: never - if: $API_FUZZING_DISABLED when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && - $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME - when: never - - if: $FUZZAPI_HAR == null && - $FUZZAPI_OPENAPI == null && - $FUZZAPI_D_WORKER_IMAGE == null + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ services: - docker:19.03.12-dind script: # + # - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY # - docker network create --driver bridge $FUZZAPI_D_NETWORK @@ -56,30 +122,13 @@ apifuzzer_fuzz: --network $FUZZAPI_D_NETWORK \ -e Proxy:Port=8000 \ -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://127.0.0.1:80 \ - -e FUZZAPI_PROJECT \ - -e FUZZAPI_PROFILE \ - -e FUZZAPI_CONFIG \ - -e FUZZAPI_REPORT \ - -e FUZZAPI_HAR \ - -e FUZZAPI_OPENAPI \ - -e FUZZAPI_TARGET_URL \ - -e FUZZAPI_OVERRIDES_FILE \ - -e FUZZAPI_OVERRIDES_ENV \ - -e FUZZAPI_OVERRIDES_CMD \ - -e FUZZAPI_OVERRIDES_INTERVAL \ - -e FUZZAPI_TIMEOUT \ - -e FUZZAPI_VERBOSE \ - -e FUZZAPI_SERVICE_START_TIMEOUT \ - -e FUZZAPI_HTTP_USERNAME \ - -e FUZZAPI_HTTP_PASSWORD \ -e GITLAB_FEATURES \ - -v $CI_PROJECT_DIR:/app \ -p 80:80 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ - registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + $FUZZAPI_IMAGE \ + dotnet /peach/Peach.Web.dll # # Start target container - | @@ -94,19 +143,31 @@ apifuzzer_fuzz: $FUZZAPI_D_TARGET_IMAGE \ ; fi # - # Start worker container + # Start worker container if provided - | if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ - echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ docker run \ --name worker \ --network $FUZZAPI_D_NETWORK \ -e FUZZAPI_API=http://apifuzzer:80 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ - -e FUZZAPI_AUTOMATION_CMD \ -e FUZZAPI_CONFIG \ -e FUZZAPI_REPORT \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ $FUZZAPI_D_WORKER_ENV \ $FUZZAPI_D_WORKER_PORTS \ @@ -115,13 +176,49 @@ apifuzzer_fuzz: $FUZZAPI_D_WORKER_IMAGE \ ; fi # - # Wait for testing to complete if api fuzzer is scanning - - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi + # Start API Fuzzing provided worker if no other worker present + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -v $CI_PROJECT_DIR:/app \ + -p 81:80 \ + -p 8001:8000 \ + -p 515:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + worker-entry \ + ; fi # - # Propagate exit code from api fuzzer (if any) - - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing exited with an error. Logs are available as job artifacts."; docker logs apifuzzer; exit 1; fi + # Propagate exit code from api fuzzing scanner (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi # - # Run user provided pre-script + # Run user provided post-script - sh -c "$FUZZAPI_POST_SCRIPT" # after_script: @@ -129,13 +226,13 @@ apifuzzer_fuzz: # Shutdown all containers - echo "Stopping all containers" - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi - - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi + - docker stop worker - docker stop apifuzzer # # Save docker logs - docker logs apifuzzer &> gl-api_fuzzing-logs.log - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi - - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi + - docker logs worker &> gl-api_fuzzing-worker-logs.log # artifacts: when: always diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 21bcdd8d9b5..3cbde9d30c8 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -4,8 +4,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - - CS_MAJOR_VERSION: 2 + CS_MAJOR_VERSION: 3 container_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index e268b48d133..a1b6dc2cc1b 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -11,6 +11,14 @@ variables: COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" +coverage_fuzzing_unlicensed: + stage: test + allow_failure: true + rules: + - if: $GITLAB_FEATURES !~ /\bcoverage_fuzzing\b/ && $COVFUZZ_DISABLED == null + script: + - echo "ERROR Your GitLab project is missing licensing for Coverage Fuzzing" && exit 1 + .fuzz_base: stage: fuzz allow_failure: true diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 4418ff18d73..a51cb61da6d 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -134,6 +134,7 @@ mobsf-android-sast: name: "$SAST_ANALYZER_IMAGE" variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + MOBSF_API_KEY: key rules: - if: $SAST_DISABLED when: never @@ -152,6 +153,7 @@ mobsf-ios-sast: name: "$SAST_ANALYZER_IMAGE" variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" + MOBSF_API_KEY: key rules: - if: $SAST_DISABLED when: never diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 2d2e0859373..232c320562b 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -131,6 +131,8 @@ secrets: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" sobelow: extends: .download_images @@ -162,6 +164,8 @@ klar: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ + variables: + SECURE_BINARIES_ANALYZER_VERSION: "3" clair-vulnerabilities-db: extends: .download_images diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index b08ccf18b58..5963d7138c5 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -1,11 +1,12 @@ include: - - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml + - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: - init - validate - build - deploy + - cleanup init: extends: .init diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 000a1a7f580..e455bfac9de 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -12,9 +12,6 @@ image: name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest -before_script: - - cd ${TF_ROOT} - variables: TF_ROOT: ${CI_PROJECT_DIR} @@ -26,16 +23,19 @@ cache: .init: &init stage: init script: + - cd ${TF_ROOT} - gitlab-terraform init .validate: &validate stage: validate script: + - cd ${TF_ROOT} - gitlab-terraform validate .build: &build stage: build script: + - cd ${TF_ROOT} - gitlab-terraform plan - gitlab-terraform plan-json artifacts: @@ -47,7 +47,14 @@ cache: .deploy: &deploy stage: deploy script: + - cd ${TF_ROOT} - gitlab-terraform apply when: manual only: - master + +.destroy: &destroy + stage: cleanup + script: + - gitlab-terraform destroy + when: manual diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index e99889f4a25..6f3e4ccf48d 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -75,7 +75,7 @@ module Gitlab until length <= 0 || eof? data = chunk_slice_from_offset - raise FailedToGetChunkError if data.empty? + raise FailedToGetChunkError if data.to_s.empty? chunk_bytes = [CHUNK_SIZE - chunk_offset, length].min chunk_data_slice = data.byteslice(0, chunk_bytes) @@ -100,7 +100,7 @@ module Gitlab until eof? data = chunk_slice_from_offset - raise FailedToGetChunkError if data.empty? + raise FailedToGetChunkError if data.to_s.empty? new_line = data.index("\n") diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 0ca99506311..4d7590a8e38 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -9,6 +9,11 @@ module Gitlab CONTEXT_LINES = 3 + CONFLICT_TYPES = { + "old" => "conflict_marker_their", + "new" => "conflict_marker_our" + }.freeze + attr_reader :merge_request # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps @@ -46,6 +51,34 @@ module Gitlab end end + def diff_lines_for_serializer + # calculate sections and highlight lines before changing types + sections && highlight_lines! + + sections.flat_map do |section| + if section[:conflict] + lines = [] + + initial_type = nil + section[:lines].each do |line| + if line.type != initial_type + lines << create_separator_line(line) + initial_type = line.type + end + + line.type = CONFLICT_TYPES[line.type] + lines << line + end + + lines << create_separator_line(lines.last) + + lines + else + section[:lines] + end + end + end + def sections return @sections if @sections @@ -93,9 +126,15 @@ module Gitlab lines = tail_lines elsif conflict_before - # We're at the end of the file (no conflicts after), so just remove extra - # trailing lines. + # We're at the end of the file (no conflicts after) + number_of_trailing_lines = lines.size + + # Remove extra trailing lines lines = lines.first(CONTEXT_LINES) + + if number_of_trailing_lines > CONTEXT_LINES + lines << create_match_line(lines.last) + end end end @@ -117,6 +156,10 @@ module Gitlab Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) end + def create_separator_line(line) + Gitlab::Diff::Line.new('', 'conflict_marker', line.index, nil, nil) + end + # Any line beginning with a letter, an underscore, or a dollar can be used in a # match line header. Only context sections can contain match lines, as match lines # have to exist in both versions of the file. diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 2b08d3c63bb..d0579a44219 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -16,8 +16,8 @@ module Gitlab @in_memory_application_settings = nil end - def method_missing(name, *args, &block) - current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(name, *args, **kwargs, &block) + current_application_settings.send(name, *args, **kwargs, &block) # rubocop:disable GitlabSecurity/PublicSend end def respond_to_missing?(name, include_private = false) diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 7b01db125a9..2e469359bdc 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -191,7 +191,7 @@ module Gitlab end def subject_starts_with_lowercase? - first_char = subject.sub(/\A\[.+\]\s/, '')[0] + first_char = subject.sub(/\A(\[.+\]|\w+:)\s/, '')[0] first_char_downcased = first_char.downcase return true unless ('a'..'z').cover?(first_char_downcased) diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 783a5f1715c..89f21e8bd23 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -168,7 +168,7 @@ module Gitlab %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, %r{\A\.codeclimate\.yml\z} => :engineering_productivity, - %r{\A\.overcommit\.yml\.example\z} => :engineering_productivity, + %r{\Alefthook.yml\z} => :engineering_productivity, %r{\A\.editorconfig\z} => :engineering_productivity, %r{Dangerfile\z} => :engineering_productivity, %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, @@ -190,7 +190,7 @@ module Gitlab %r{\A(ee/)?vendor/} => :backend, %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A\.rubocop(_todo)?\.yml\z} => :backend, + %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend, %r{\Afile_hooks/} => :backend, %r{\A(ee/)?qa/} => :qa, @@ -200,6 +200,9 @@ module Gitlab %r{\Alocale/gitlab\.pot\z} => :none, %r{\Adata/whats_new/} => :none, + # GraphQL auto generated doc files and schema + %r{\Adoc/api/graphql/reference/} => :backend, + # Fallbacks in case the above patterns miss anything %r{\.rb\z} => :backend, %r{( diff --git a/lib/gitlab/data_builder/feature_flag.rb b/lib/gitlab/data_builder/feature_flag.rb new file mode 100644 index 00000000000..2f675ace7e1 --- /dev/null +++ b/lib/gitlab/data_builder/feature_flag.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module FeatureFlag + extend self + + def build(feature_flag, user) + { + object_kind: 'feature_flag', + project: feature_flag.project.hook_attrs, + user: user.hook_attrs, + user_url: Gitlab::UrlBuilder.build(user), + object_attributes: feature_flag.hook_attrs + } + end + end + end +end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 11d9881aac2..6f79e965cd5 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -128,9 +128,9 @@ module Gitlab end def between_condition(start, finish) - return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute) + return @column.between(start...finish) if @column.is_a?(Arel::Attributes::Attribute) - { @column => start..(finish - 1) } + { @column => start...finish } end def actual_start(start) diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index ecc05d9654a..82ea1ce26fb 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -17,23 +17,8 @@ module Gitlab end def current_partitions - result = connection.select_all(<<~SQL) - select - pg_class.relname, - parent_class.relname as base_table, - pg_get_expr(pg_class.relpartbound, inhrelid) as condition - from pg_class - inner join pg_inherits i on pg_class.oid = inhrelid - inner join pg_class parent_class on parent_class.oid = inhparent - inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace - where pg_namespace.nspname = #{connection.quote(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA)} - and parent_class.relname = #{connection.quote(table_name)} - and pg_class.relispartition - order by pg_class.relname - SQL - - result.map do |record| - TimePartition.from_sql(table_name, record['relname'], record['condition']) + Gitlab::Database::PostgresPartition.for_parent_table(table_name).map do |partition| + TimePartition.from_sql(table_name, partition.name, partition.condition) end end diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb new file mode 100644 index 00000000000..6f6af223fa2 --- /dev/null +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class ReplaceTable + DELIMITER = ";\n\n" + + attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column, + :sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key + + def initialize(original_table, replacement_table, replaced_table, primary_key_column) + @original_table = original_table + @replacement_table = replacement_table + @replaced_table = replaced_table + @primary_key_column = primary_key_column + + @sequence = default_sequence(original_table, primary_key_column) + @original_primary_key = default_primary_key(original_table) + @replacement_primary_key = default_primary_key(replacement_table) + @replaced_primary_key = default_primary_key(replaced_table) + end + + def perform + yield sql_to_replace_table if block_given? + + execute(sql_to_replace_table) + end + + private + + delegate :execute, :quote_table_name, :quote_column_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + + def default_sequence(table, column) + "#{table}_#{column}_seq" + end + + def default_primary_key(table) + "#{table}_pkey" + end + + def sql_to_replace_table + @sql_to_replace_table ||= combined_sql_statements.map(&:chomp).join(DELIMITER) + end + + def combined_sql_statements + statements = [] + + statements << alter_column_default(original_table, primary_key_column, expression: nil) + statements << alter_column_default(replacement_table, primary_key_column, + expression: "nextval('#{quote_table_name(sequence)}'::regclass)") + + statements << alter_sequence_owned_by(sequence, replacement_table, primary_key_column) + + rename_table_objects(statements, original_table, replaced_table, original_primary_key, replaced_primary_key) + rename_table_objects(statements, replacement_table, original_table, replacement_primary_key, original_primary_key) + + statements + end + + def rename_table_objects(statements, old_table, new_table, old_primary_key, new_primary_key) + statements << rename_table(old_table, new_table) + statements << rename_constraint(new_table, old_primary_key, new_primary_key) + + rename_partitions(statements, old_table, new_table) + end + + def rename_partitions(statements, old_table_name, new_table_name) + Gitlab::Database::PostgresPartition.for_parent_table(old_table_name).each do |partition| + new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name) + old_primary_key = default_primary_key(partition.name) + new_primary_key = default_primary_key(new_partition_name) + + statements << rename_constraint(partition.identifier, old_primary_key, new_primary_key) + statements << rename_table(partition.identifier, new_partition_name) + end + end + + def alter_column_default(table_name, column_name, expression:) + default_clause = expression.nil? ? 'DROP DEFAULT' : "SET DEFAULT #{expression}" + + <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + ALTER COLUMN #{quote_column_name(column_name)} #{default_clause} + SQL + end + + def alter_sequence_owned_by(sequence_name, table_name, column_name) + <<~SQL + ALTER SEQUENCE #{quote_table_name(sequence_name)} + OWNED BY #{quote_table_name(table_name)}.#{quote_column_name(column_name)} + SQL + end + + def rename_table(old_name, new_name) + <<~SQL + ALTER TABLE #{quote_table_name(old_name)} + RENAME TO #{quote_table_name(new_name)} + SQL + end + + def rename_constraint(table_name, old_name, new_name) + <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb index 881177a195e..3196dd20356 100644 --- a/lib/gitlab/database/partitioning_migration_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers.rb @@ -5,6 +5,7 @@ module Gitlab module PartitioningMigrationHelpers include ForeignKeyHelpers include TableManagementHelpers + include IndexHelpers end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb new file mode 100644 index 00000000000..f367292f4b0 --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + module IndexHelpers + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::SchemaHelpers + + # Concurrently creates a new index on a partitioned table. In concept this works similarly to + # `add_concurrent_index`, and won't block reads or writes on the table while the index is being built. + # + # A special helper is required for partitioning because Postgres does not support concurrently building indexes + # on partitioned tables. This helper concurrently adds the same index to each partition, and creates the final + # index on the parent table once all of the partitions are indexed. This is the recommended safe way to add + # indexes to partitioned tables. + # + # Example: + # + # add_concurrent_partitioned_index :users, :some_column + # + # See Rails' `add_index` for more info on the available arguments. + def add_concurrent_partitioned_index(table_name, column_names, options = {}) + raise ArgumentError, 'A name is required for indexes added to partitioned tables' unless options[:name] + + partitioned_table = find_partitioned_table(table_name) + + if index_name_exists?(table_name, options[:name]) + Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted" \ + " migration or similar): table_name: #{table_name}, index_name: #{options[:name]}" + + return + end + + partitioned_table.postgres_partitions.each do |partition| + partition_index_name = generated_index_name(partition.identifier, options[:name]) + partition_options = options.merge(name: partition_index_name) + + add_concurrent_index(partition.identifier, column_names, partition_options) + end + + with_lock_retries do + add_index(table_name, column_names, options) + end + end + + # Safely removes an existing index from a partitioned table. The method name is a bit inaccurate as it does not + # drop the index concurrently, but it's named as such to maintain consistency with other similar helpers, and + # indicate that this should be safe to use in a production environment. + # + # In current versions of Postgres it's impossible to drop an index concurrently, or drop an index from an + # individual partition that exists across the entire partitioned table. As a result this helper drops the index + # from the parent table, which automatically cascades to all partitions. While this does require an exclusive + # lock, dropping an index is a fast operation that won't block the table for a significant period of time. + # + # Example: + # + # remove_concurrent_partitioned_index_by_name :users, 'index_name_goes_here' + def remove_concurrent_partitioned_index_by_name(table_name, index_name) + find_partitioned_table(table_name) + + unless index_name_exists?(table_name, index_name) + Gitlab::AppLogger.warn "Index not removed because it does not exist (this may be due to an aborted " \ + "migration or similar): table_name: #{table_name}, index_name: #{index_name}" + + return + end + + with_lock_retries do + remove_index(table_name, name: index_name) + end + end + + private + + def find_partitioned_table(table_name) + partitioned_table = Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name) + + raise ArgumentError, "#{table_name} is not a partitioned table" unless partitioned_table + + partitioned_table + end + + def generated_index_name(partition_name, index_name) + object_name("#{partition_name}_#{index_name}", 'index') + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index f7b0306b769..686dda80207 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -66,7 +66,10 @@ module Gitlab create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key) create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date) end - create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) + + with_lock_retries do + create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) + end end # Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were @@ -81,13 +84,9 @@ module Gitlab assert_not_in_transaction_block(scope: ERROR_SCOPE) with_lock_retries do - trigger_name = make_sync_trigger_name(table_name) - drop_trigger(table_name, trigger_name) + drop_sync_trigger(table_name) end - function_name = make_sync_function_name(table_name) - drop_function(function_name) - partitioned_table_name = make_partitioned_table_name(table_name) drop_table(partitioned_table_name) end @@ -177,6 +176,53 @@ module Gitlab end end + # Replaces a non-partitioned table with its partitioned copy. This is the final step in a partitioning + # migration, which makes the partitioned table ready for use by the application. The partitioned copy should be + # replaced with the original table in such a way that it appears seamless to any database clients. The replaced + # table will be renamed to "#{replaced_table}_archived". Partitions and primary key constraints will also be + # renamed to match the naming scheme of the parent table. + # + # **NOTE** This method should only be used after all other migration steps have completed successfully. + # There are several limitations to this method that MUST be handled before, or during, the swap migration: + # + # - Secondary indexes and foreign keys are not automatically recreated on the partitioned table. + # - Some types of constraints (UNIQUE and EXCLUDE) which rely on indexes, will not automatically be recreated + # on the partitioned table, since the underlying index will not be present. + # - Foreign keys referencing the original non-partitioned table, would also need to be updated to reference the + # partitioned table, but unfortunately this is not supported in PG11. + # - Views referencing the original table will not be automatically updated to reference the partitioned table. + # + # Example: + # + # replace_with_partitioned_table :audit_events + # + def replace_with_partitioned_table(table_name) + assert_table_is_allowed(table_name) + + partitioned_table_name = make_partitioned_table_name(table_name) + archived_table_name = make_archived_table_name(table_name) + primary_key_name = connection.primary_key(table_name) + + replace_table(table_name, partitioned_table_name, archived_table_name, primary_key_name) + end + + # Rolls back a migration that replaced a non-partitioned table with its partitioned copy. This can be used to + # restore the original non-partitioned table in the event of an unexpected issue. + # + # Example: + # + # rollback_replace_with_partitioned_table :audit_events + # + def rollback_replace_with_partitioned_table(table_name) + assert_table_is_allowed(table_name) + + partitioned_table_name = make_partitioned_table_name(table_name) + archived_table_name = make_archived_table_name(table_name) + primary_key_name = connection.primary_key(archived_table_name) + + replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name) + end + private def assert_table_is_allowed(table_name) @@ -190,6 +236,10 @@ module Gitlab tmp_table_name("#{table}_part") end + def make_archived_table_name(table) + "#{table}_archived" + end + def make_sync_function_name(table) object_name(table, 'table_sync_function') end @@ -270,12 +320,18 @@ module Gitlab function_name = make_sync_function_name(source_table_name) trigger_name = make_sync_trigger_name(source_table_name) - with_lock_retries do - create_sync_function(function_name, partitioned_table_name, unique_key) - create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") + create_sync_function(function_name, partitioned_table_name, unique_key) + create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") - create_sync_trigger(source_table_name, trigger_name, function_name) - end + create_sync_trigger(source_table_name, trigger_name, function_name) + end + + def drop_sync_trigger(source_table_name) + trigger_name = make_sync_trigger_name(source_table_name) + drop_trigger(source_table_name, trigger_name) + + function_name = make_sync_function_name(source_table_name) + drop_function(function_name) end def create_sync_function(name, partitioned_table_name, unique_key) @@ -358,6 +414,21 @@ module Gitlab end end end + + def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name) + replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s, + replacement_table_name, replaced_table_name, primary_key_name) + + with_lock_retries do + drop_sync_trigger(original_table_name) + + replace_table.perform do |sql| + say("replace_table(\"#{sql}\")") + end + + create_trigger_to_sync_tables(original_table_name, replaced_table_name, primary_key_name) + end + end end end end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb new file mode 100644 index 00000000000..0986372586b --- /dev/null +++ b/lib/gitlab/database/postgres_partition.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresPartition < ActiveRecord::Base + self.primary_key = :identifier + + belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb new file mode 100644 index 00000000000..5d2eaa22ee4 --- /dev/null +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresPartitionedTable < ActiveRecord::Base + DYNAMIC_PARTITION_STRATEGIES = %w[range list].freeze + + self.primary_key = :identifier + + has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + def self.find_by_name_in_current_schema(name) + find_by("identifier = concat(current_schema(), '.', ?)", name) + end + + def dynamic? + DYNAMIC_PARTITION_STRATEGIES.include?(strategy) + end + + def static? + !dynamic? + end + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 074752fe75b..c77e000254f 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -10,6 +10,7 @@ module Gitlab def self.candidate_indexes Gitlab::Database::PostgresIndex .regular + .where('NOT expression') .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index a4e265eba88..d735fb55652 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -6,6 +6,7 @@ module Gitlab URL_REGEX = %r{https?://[^'" ]+}.freeze GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze + VALID_LINK_ATTRIBUTES = %w[href rel target].freeze include ActionView::Helpers::SanitizeHelper @@ -66,7 +67,7 @@ module Gitlab def link_tag(name, url) sanitize( %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}, - attributes: %w[href rel target] + attributes: VALID_LINK_ATTRIBUTES ) end @@ -77,7 +78,7 @@ module Gitlab # # Will link `user/repo` in `github: "user/repo"` or `:github => "user/repo"` def link_regex(regex, &url_proc) highlighted_lines.map!.with_index do |rich_line, i| - marker = StringRegexMarker.new(plain_lines[i].chomp, rich_line.html_safe) + marker = StringRegexMarker.new((plain_lines[i].chomp! || plain_lines[i]), rich_line.html_safe) marker.mark(regex, group: :name) do |text, left:, right:| url = yield(text) diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml index 1d341e6520e..95f15bd6dee 100644 --- a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -29,6 +29,7 @@ ignore_design_attributes: - id - issue_id - project_id + - iid ignore_version_attributes: - id diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb index c6d1e0b93a7..9af66318b89 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_batch.rb @@ -18,10 +18,8 @@ module Gitlab def initialize(merge_request_diff, batch_page, batch_size, diff_options:) super(merge_request_diff, diff_options: diff_options) - batch_page ||= DEFAULT_BATCH_PAGE - batch_size ||= DEFAULT_BATCH_SIZE + @paginated_collection = load_paginated_collection(batch_page, batch_size, diff_options) - @paginated_collection = relation.page(batch_page).per(batch_size) @pagination_data = { current_page: @paginated_collection.current_page, next_page: @paginated_collection.next_page, @@ -63,6 +61,18 @@ module Gitlab def relation @merge_request_diff.merge_request_diff_files end + + def load_paginated_collection(batch_page, batch_size, diff_options) + batch_page ||= DEFAULT_BATCH_PAGE + batch_size ||= DEFAULT_BATCH_SIZE + + paths = diff_options&.fetch(:paths, nil) + + paginated_collection = relation.page(batch_page).per(batch_size) + paginated_collection = paginated_collection.by_paths(paths) if paths + + paginated_collection + end end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 379fc6af875..af9140215f0 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,9 +8,9 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :type, :old_pos, :new_pos + attr_reader :line_code, :old_pos, :new_pos attr_writer :rich_text - attr_accessor :text, :index + attr_accessor :text, :index, :type def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 803acef9a40..a5ace2be773 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -123,6 +123,7 @@ module Gitlab end extra = sanitize_request_parameters(extra) + inject_sql_query_into_extra(exception, extra) if sentry && Raven.configuration.server Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -149,6 +150,12 @@ module Gitlab filter.filter(parameters) end + def inject_sql_query_into_extra(exception, extra) + return unless exception.is_a?(ActiveRecord::StatementInvalid) + + extra[:sql] = PgQuery.normalize(exception.sql.to_s) + end + def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 303e1a23e6b..fc3c05c57b2 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -3,6 +3,14 @@ module Gitlab module EtagCaching class Middleware + SKIP_HEADER_KEY = 'X-Gitlab-Skip-Etag' + + class << self + def skip!(response) + response.set_header(SKIP_HEADER_KEY, '1') + end + end + def initialize(app) @app = app end @@ -22,9 +30,7 @@ module Gitlab else track_cache_miss(if_none_match, cached_value_present, route) - status, headers, body = @app.call(env) - headers['ETag'] = etag - [status, headers, body] + maybe_apply_etag(etag, *@app.call(env)) end end @@ -43,6 +49,13 @@ module Gitlab [weak_etag_format(current_value), cached_value_present] end + def maybe_apply_etag(etag, status, headers, body) + headers['ETag'] = etag unless + Gitlab::Utils.to_boolean(headers.delete(SKIP_HEADER_KEY)) + + [status, headers, body] + end + def weak_etag_format(value) %Q{W/"#{value}"} end @@ -54,7 +67,13 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) - [status_code, { 'ETag' => etag, 'X-Gitlab-From-Cache' => 'true' }, []] + new_headers = { + 'ETag' => etag, + 'X-Gitlab-From-Cache' => 'true', + ::Gitlab::Metrics::RequestsRackMiddleware::FEATURE_CATEGORY_HEADER => route.feature_category + } + + [status_code, new_headers, []] end def track_cache_miss(if_none_match, cached_value_present, route) diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 17d9cf08367..769ac2784d1 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,7 +3,7 @@ module Gitlab module EtagCaching class Router - Route = Struct.new(:regexp, :name) + Route = Struct.new(:regexp, :name, :feature_category) # We enable an ETag for every request matching the regex. # To match a regex the path needs to match the following: # - Don't contain a reserved word (expect for the words used in the @@ -20,59 +20,73 @@ module Gitlab ROUTES = [ Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), - 'issue_notes' + 'issue_notes', + 'issue_tracking' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), - 'merge_request_notes' + 'merge_request_notes', + 'code_review' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), - 'issue_title' + 'issue_title', + 'issue_tracking' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), - 'commit_pipelines' + 'commit_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), - 'new_merge_request_pipelines' + 'new_merge_request_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), - 'merge_request_pipelines' + 'merge_request_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), - 'project_pipelines' + 'project_pipelines', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), - 'project_pipeline' + 'project_pipeline', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), - 'project_build' + 'project_build', + 'continuous_integration' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), - 'cluster_environments' + 'cluster_environments', + 'continuous_delivery' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), - 'environments' + 'environments', + 'continuous_delivery' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), - 'realtime_changes_import_github' + 'realtime_changes_import_github', + 'importers' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), - 'realtime_changes_import_gitea' + 'realtime_changes_import_gitea', + 'importers' ), Gitlab::EtagCaching::Router::Route.new( %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), - 'merge_request_widget' + 'merge_request_widget', + 'code_review' ) ].freeze diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 1ce3ffe4c86..6e39776bbd4 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -6,6 +6,7 @@ # Experiment options: # - environment (optional, defaults to enabled for development and GitLab.com) # - tracking_category (optional, used to set the category when tracking an experiment event) +# - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility) # # The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), # which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. @@ -30,168 +31,60 @@ module Gitlab module Experimentation EXPERIMENTS = { - signup_flow: { - tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow' - }, onboarding_issues: { - tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues' - }, - suggest_pipeline: { - tracking_category: 'Growth::Expansion::Experiment::SuggestPipeline' + tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues', + use_backwards_compatible_subject_index: true }, ci_notification_dot: { - tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot' + tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot', + use_backwards_compatible_subject_index: true }, upgrade_link_in_user_menu_a: { - tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA' + tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA', + use_backwards_compatible_subject_index: true }, invite_members_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA' + tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA', + use_backwards_compatible_subject_index: true }, invite_members_version_b: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB' + tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', + use_backwards_compatible_subject_index: true + }, + invite_members_empty_group_version_a: { + tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', + use_backwards_compatible_subject_index: true }, new_create_project_ui: { - tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi' + tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi', + use_backwards_compatible_subject_index: true }, contact_sales_btn_in_app: { - tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp' + tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', + use_backwards_compatible_subject_index: true }, customize_homepage: { - tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage' + tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage', + use_backwards_compatible_subject_index: true }, invite_email: { - tracking_category: 'Growth::Acquisition::Experiment::InviteEmail' + tracking_category: 'Growth::Acquisition::Experiment::InviteEmail', + use_backwards_compatible_subject_index: true }, invitation_reminders: { - tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders' + tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders', + use_backwards_compatible_subject_index: true }, group_only_trials: { - tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials' + tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', + use_backwards_compatible_subject_index: true }, default_to_issues_board: { - tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard' + tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard', + use_backwards_compatible_subject_index: true } }.freeze - GROUP_CONTROL = :control - GROUP_EXPERIMENTAL = :experimental - - # Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. - # Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method - # to controllers and views. It returns true when the experiment is enabled and the user is selected as part - # of the experimental group. - # - module ControllerConcern - extend ActiveSupport::Concern - - included do - before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group - end - - def set_experimentation_subject_id_cookie - return if cookies[:experimentation_subject_id].present? - - cookies.permanent.signed[:experimentation_subject_id] = { - value: SecureRandom.uuid, - secure: ::Gitlab.config.gitlab.https, - httponly: true - } - end - - def push_frontend_experiment(experiment_key) - var_name = experiment_key.to_s.camelize(:lower) - enabled = experiment_enabled?(experiment_key) - - gon.push({ experiments: { var_name => enabled } }, true) - end - - def experiment_enabled?(experiment_key) - return false if dnt_enabled? - - return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index) - return true if forced_enabled?(experiment_key) - - false - end - - def track_experiment_event(experiment_key, action, value = nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) - end - end - - def frontend_experimentation_tracking_data(experiment_key, action, value = nil) - return if dnt_enabled? - - track_experiment_event_for(experiment_key, action, value) do |tracking_data| - gon.push(tracking_data: tracking_data) - end - end - - def record_experiment_user(experiment_key) - return if dnt_enabled? - return unless Experimentation.enabled?(experiment_key) && current_user - - ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) - end - - def experiment_tracking_category_and_group(experiment_key) - "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}" - end - - private - - def dnt_enabled? - Gitlab::Utils.to_boolean(request.headers['DNT']) - end - - def experimentation_subject_id - cookies.signed[:experimentation_subject_id] - end - - def experimentation_subject_index - return if experimentation_subject_id.blank? - - experimentation_subject_id.delete('-').hex % 100 - end - - def track_experiment_event_for(experiment_key, action, value) - return unless Experimentation.enabled?(experiment_key) - - yield experimentation_tracking_data(experiment_key, action, value) - end - - def experimentation_tracking_data(experiment_key, action, value) - { - category: tracking_category(experiment_key), - action: action, - property: tracking_group(experiment_key, "_group"), - label: experimentation_subject_id, - value: value - }.compact - end - - def tracking_category(experiment_key) - Experimentation.experiment(experiment_key).tracking_category - end - - def tracking_group(experiment_key, suffix = nil) - return unless Experimentation.enabled?(experiment_key) - - group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL - - suffix ? "#{group}#{suffix}" : group - end - - def forced_enabled?(experiment_key) - params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s - end - end - class << self def experiment(key) Experiment.new(EXPERIMENTS[key].merge(key: key)) @@ -201,7 +94,7 @@ module Gitlab return false unless EXPERIMENTS.key?(experiment_key) experiment = experiment(experiment_key) - experiment.enabled? && experiment.enabled_for_environment? + experiment.enabled_for_environment? && experiment.enabled? end def enabled_for_attribute?(experiment_key, attribute) @@ -209,13 +102,18 @@ module Gitlab enabled_for_value?(experiment_key, index) end - def enabled_for_value?(experiment_key, experimentation_subject_index) - enabled?(experiment_key) && - experiment(experiment_key).enabled_for_index?(experimentation_subject_index) + def enabled_for_value?(experiment_key, value) + enabled?(experiment_key) && experiment(experiment_key).enabled_for_index?(value) end end - Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do + Experiment = Struct.new( + :key, + :environment, + :tracking_category, + :use_backwards_compatible_subject_index, + keyword_init: true + ) do def enabled? experiment_percentage > 0 end diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb new file mode 100644 index 00000000000..c6d15d7d82d --- /dev/null +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +require 'zlib' + +# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent. +# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method +# to controllers and views. It returns true when the experiment is enabled and the user is selected as part +# of the experimental group. +# +module Gitlab + module Experimentation + module ControllerConcern + include ::Gitlab::Experimentation::GroupTypes + extend ActiveSupport::Concern + + included do + before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? + helper_method :experiment_enabled?, :experiment_tracking_category_and_group + end + + def set_experimentation_subject_id_cookie + return if cookies[:experimentation_subject_id].present? + + cookies.permanent.signed[:experimentation_subject_id] = { + value: SecureRandom.uuid, + secure: ::Gitlab.config.gitlab.https, + httponly: true + } + end + + def push_frontend_experiment(experiment_key) + var_name = experiment_key.to_s.camelize(:lower) + enabled = experiment_enabled?(experiment_key) + + gon.push({ experiments: { var_name => enabled } }, true) + end + + def experiment_enabled?(experiment_key) + return false if dnt_enabled? + + return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index(experiment_key)) + return true if forced_enabled?(experiment_key) + + false + end + + def track_experiment_event(experiment_key, action, value = nil) + return if dnt_enabled? + + track_experiment_event_for(experiment_key, action, value) do |tracking_data| + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) + end + end + + def frontend_experimentation_tracking_data(experiment_key, action, value = nil) + return if dnt_enabled? + + track_experiment_event_for(experiment_key, action, value) do |tracking_data| + gon.push(tracking_data: tracking_data) + end + end + + def record_experiment_user(experiment_key) + return if dnt_enabled? + return unless Experimentation.enabled?(experiment_key) && current_user + + ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user) + end + + def experiment_tracking_category_and_group(experiment_key) + "#{tracking_category(experiment_key)}:#{tracking_group(experiment_key, '_group')}" + end + + private + + def dnt_enabled? + Gitlab::Utils.to_boolean(request.headers['DNT']) + end + + def experimentation_subject_id + cookies.signed[:experimentation_subject_id] + end + + def experimentation_subject_index(experiment_key) + return if experimentation_subject_id.blank? + + if Experimentation.experiment(experiment_key).use_backwards_compatible_subject_index + experimentation_subject_id.delete('-').hex % 100 + else + Zlib.crc32("#{experiment_key}#{experimentation_subject_id}") % 100 + end + end + + def track_experiment_event_for(experiment_key, action, value) + return unless Experimentation.enabled?(experiment_key) + + yield experimentation_tracking_data(experiment_key, action, value) + end + + def experimentation_tracking_data(experiment_key, action, value) + { + category: tracking_category(experiment_key), + action: action, + property: tracking_group(experiment_key, "_group"), + label: experimentation_subject_id, + value: value + }.compact + end + + def tracking_category(experiment_key) + Experimentation.experiment(experiment_key).tracking_category + end + + def tracking_group(experiment_key, suffix = nil) + return unless Experimentation.enabled?(experiment_key) + + group = experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL + + suffix ? "#{group}#{suffix}" : group + end + + def forced_enabled?(experiment_key) + params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + end + end + end +end diff --git a/lib/gitlab/experimentation/group_types.rb b/lib/gitlab/experimentation/group_types.rb new file mode 100644 index 00000000000..8e8f7284b99 --- /dev/null +++ b/lib/gitlab/experimentation/group_types.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Experimentation + module GroupTypes + GROUP_CONTROL = :control + GROUP_EXPERIMENTAL = :experimental + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 78c47023c08..209917073c7 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -25,7 +25,7 @@ module Gitlab # # If this value ever changes, make sure to create a migration to update # current records, and default of `ApplicationSettings#diff_max_patch_bytes`. - DEFAULT_MAX_PATCH_BYTES = 100.kilobytes + DEFAULT_MAX_PATCH_BYTES = 200.kilobytes # This is a limitation applied on the source (Gitaly), therefore we don't allow # persisting limits over that. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 1a3409c1f84..bc712e87e99 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -302,7 +302,7 @@ module Gitlab private :archive_file_path def archive_version_path - return '' unless Feature.enabled?(:include_lfs_blobs_in_archive) + return '' unless Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) '@v2' end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 20ad6d0184b..e41a406ebd3 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -26,8 +26,8 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :cleanup, request, timeout: GitalyClient.fast_timeout) end - def garbage_collect(create_bitmap) - request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) + def garbage_collect(create_bitmap, prune:) + request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) end diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 9a7c406d981..c3cc15e10f7 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -6,10 +6,13 @@ module Gitlab [:heads, :tags, '+refs/pull/*/head:refs/merge-requests/*/head'] end - def self.new_client_for(project, token: nil, parallel: true) + def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - - Client.new(token_to_use, parallel: parallel) + Client.new( + token_to_use, + host: host.presence || self.formatted_import_url(project), + parallel: parallel + ) end # Returns the ID of the ghost user. @@ -18,5 +21,17 @@ module Gitlab Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id) end + + # Get formatted GitHub import URL. If github.com is in the import URL, this will return nil and octokit will use the default github.com API URL + def self.formatted_import_url(project) + url = URI.parse(project.import_url) + + unless url.host == 'github.com' + url.user = nil + url.password = nil + url.path = "/api/v3" + url.to_s + end + end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 22803c5cd71..dfe60fb5a03 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -18,6 +18,8 @@ module Gitlab attr_reader :octokit + SEARCH_MAX_REQUESTS_PER_MINUTE = 30 + # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -28,9 +30,12 @@ module Gitlab # rate limit at once. The threshold is put in place to not hit the limit # in most cases. RATE_LIMIT_THRESHOLD = 50 + SEARCH_RATE_LIMIT_THRESHOLD = 3 # token - The GitHub API token to use. # + # host - The GitHub hostname. If nil, github.com will be used. + # # per_page - The number of objects that should be displayed per page. # # parallel - When set to true hitting the rate limit will result in a @@ -39,11 +44,13 @@ module Gitlab # this value to `true` for parallel importing is crucial as # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. - def initialize(token, per_page: 100, parallel: true) + def initialize(token, host: nil, per_page: 100, parallel: true) + @host = host @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, - api_endpoint: api_endpoint + api_endpoint: api_endpoint, + web_endpoint: web_endpoint ) @octokit.connection_options[:ssl] = { verify: verify_ssl } @@ -148,8 +155,26 @@ module Gitlab end end + def search_repos_by_name(name) + each_page(:search_repositories, search_query(str: name, type: :name)) + end + + def search_query(str:, type:, include_collaborations: true, include_orgs: true) + query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}" + + query = [query, collaborations_subquery].join(' ') if include_collaborations + query = [query, organizations_subquery].join(' ') if include_orgs + + query + end + # Returns `true` if we're still allowed to perform API calls. + # Search API has rate limit of 30, use lowered threshold when search is used. def requests_remaining? + if requests_limit == SEARCH_MAX_REQUESTS_PER_MINUTE + return remaining_requests > SEARCH_RATE_LIMIT_THRESHOLD + end + remaining_requests > RATE_LIMIT_THRESHOLD end @@ -157,6 +182,10 @@ module Gitlab octokit.rate_limit.remaining end + def requests_limit + octokit.rate_limit.limit + end + def raise_or_wait_for_rate_limit rate_limit_counter.increment @@ -181,7 +210,11 @@ module Gitlab end def api_endpoint - custom_api_endpoint || default_api_endpoint + @host || custom_api_endpoint || default_api_endpoint + end + + def web_endpoint + @host || custom_api_endpoint || ::Octokit::Default.web_endpoint end def custom_api_endpoint @@ -213,6 +246,20 @@ module Gitlab 'The number of GitHub API calls performed when importing projects' ) end + + private + + def collaborations_subquery + each_object(:repos, nil, { affiliation: 'collaborator' }) + .map { |repo| "repo:#{repo.full_name}" } + .join(' ') + end + + def organizations_subquery + each_object(:organizations) + .map { |org| "org:#{org.login}" } + .join(' ') + end end end end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 6a181caf65d..cb6b2017208 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -25,10 +25,11 @@ module Gitlab # project - The project to import the data into. # token - The token to use for the GitHub API. - def initialize(project, token: nil) + # host - The GitHub hostname. If nil, github.com will be used. + def initialize(project, token: nil, host: nil) @project = project @client = GithubImport - .new_client_for(project, token: token, parallel: false) + .new_client_for(project, token: token, host: host, parallel: false) end def execute diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 10660649623..2d41ad76618 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -28,6 +28,7 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.emoji_sprites_css_path = ActionController::Base.helpers.stylesheet_path('emoji_sprites') + gon.select2_css_path = ActionController::Base.helpers.stylesheet_path('lazy_bundles/select2.css') gon.test_env = Rails.env.test? gon.disable_animations = Gitlab.config.gitlab['disable_animations'] gon.suggested_label_colors = LabelsHelper.suggested_colors @@ -58,9 +59,13 @@ module Gitlab # args - Any additional arguments to pass to `Feature.enabled?`. This allows # you to check if a flag is enabled for a particular user. def push_frontend_feature_flag(name, *args, **kwargs) - var_name = name.to_s.camelize(:lower) enabled = Feature.enabled?(name, *args, **kwargs) + push_to_gon_features(name, enabled) + end + + def push_to_gon_features(name, enabled) + var_name = name.to_s.camelize(:lower) # Here the `true` argument signals gon that the value should be merged # into any existing ones, instead of overwriting them. This allows you to # use this method to push multiple feature flags. diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index ac149cadb5b..a0dccbcdab3 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -41,6 +41,8 @@ module Gitlab data.map! { |v| utf8_encode_values(v) } when String encode_utf8(data) + when Integer + data end end end diff --git a/lib/gitlab/grape_logging/loggers/content_logger.rb b/lib/gitlab/grape_logging/loggers/content_logger.rb new file mode 100644 index 00000000000..658953adc80 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/content_logger.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + class ContentLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + { + content_length: request.env['CONTENT_LENGTH'], + content_range: request.env['HTTP_CONTENT_RANGE'] + }.compact + end + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index cbf3e7b8429..e8db619f88a 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -46,6 +46,8 @@ module Gitlab # Returns any authorize metadata from @field def field_authorizations + return [] if @field.metadata[:authorize] == true + Array.wrap(@field.metadata[:authorize]) end @@ -54,7 +56,7 @@ module Gitlab # The field is a built-in/scalar type, or a list of scalars # authorize using the parent's object parent_typed_object.object - elsif @field.connection? || resolved_type.is_a?(Array) + elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array) # The field is a connection or a list of non-built-in types, we'll # authorize each element when rendering nil @@ -75,16 +77,25 @@ module Gitlab # no need to do anything elsif authorizing_object # Authorizing fields representing scalars, or a simple field with an object - resolved_type if allowed_access?(current_user, authorizing_object) + ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object| + resolved_type if allowed_access?(current_user, object) + end elsif @field.connection? - # A connection with pagination, modify the visible nodes on the - # connection type in place - resolved_type.object.edge_nodes.to_a.keep_if { |node| allowed_access?(current_user, node) } - resolved_type - elsif resolved_type.is_a? Array + ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type| + # A connection with pagination, modify the visible nodes on the + # connection type in place + nodes = to_nodes(type) + nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes + type + end + elsif @field.type.list? || resolved_type.is_a?(Array) # A simple list of rendered types each object being an object to authorize - resolved_type.select do |single_object_type| - allowed_access?(current_user, realized(single_object_type).object) + ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items| + items.select do |single_object_type| + object_type = realized(single_object_type) + object = object_type.try(:object) || object_type + allowed_access?(current_user, object) + end end else raise "Can't authorize #{@field}" @@ -93,18 +104,23 @@ module Gitlab # Ensure that we are dealing with realized objects, not delayed promises def realized(thing) - case thing - when BatchLoader::GraphQL - thing.sync - when GraphQL::Execution::Lazy - thing.value # part of the private api, but we need to unwrap it here. + ::Gitlab::Graphql::Lazy.force(thing) + end + + # Try to get the connection + # can be at type.object or at type + def to_nodes(type) + if type.respond_to?(:nodes) + type.nodes + elsif type.respond_to?(:object) + to_nodes(type.object) else - thing + nil end end def allowed_access?(current_user, object) - object = object.sync if object.respond_to?(:sync) + object = realized(object) authorizations.all? do |ability| Ability.allowed?(current_user, ability, object) diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index dcd0e12cbfc..503b1064b11 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -81,11 +81,15 @@ module Gitlab # We are ignoring connections and built in types for now, # they should be added when queries are generated. def objects - graphql_object_types.select do |object_type| + object_types = graphql_object_types.select do |object_type| !object_type[:name]["Connection"] && !object_type[:name]["Edge"] && !object_type[:name]["__"] end + + object_types.each do |type| + type[:fields] += type[:connections] + end end # We ignore the built-in enum types. diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index ec052943589..97df4233905 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -14,6 +14,8 @@ CAUTION: **Caution:** Fields that are deprecated are marked with **{warning-solid}**. + Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found + in [Removed Items](../removed_items.md). \ :plain diff --git a/lib/gitlab/graphql/lazy.rb b/lib/gitlab/graphql/lazy.rb index a7f7610a041..3cc11047387 100644 --- a/lib/gitlab/graphql/lazy.rb +++ b/lib/gitlab/graphql/lazy.rb @@ -3,17 +3,41 @@ module Gitlab module Graphql class Lazy + include Gitlab::Utils::StrongMemoize + + def initialize(&block) + @proc = block + end + + def force + strong_memoize(:force) { self.class.force(@proc.call) } + end + + def then(&block) + self.class.new { yield force } + end + # Force evaluation of a (possibly) lazy value def self.force(value) case value + when ::Gitlab::Graphql::Lazy + value.force when ::BatchLoader::GraphQL value.sync + when ::GraphQL::Execution::Lazy + value.value # part of the private api, but we can force this as well when ::Concurrent::Promise - value.execute.value + value.execute if value.state == :unscheduled + + value.value # value.value(10.seconds) else value end end + + def self.with_value(unforced, &block) + self.new { unforced }.then(&block) + end end end end diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb index 164fe74148c..9b85ba164d4 100644 --- a/lib/gitlab/graphql/loaders/batch_model_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -12,14 +12,11 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def find - BatchLoader::GraphQL.for({ model: model_class, id: model_id.to_i }).batch do |loader_info, loader| - per_model = loader_info.group_by { |info| info[:model] } - per_model.each do |model, info| - ids = info.map { |i| i[:id] } - results = model.where(id: ids) + BatchLoader::GraphQL.for(model_id.to_i).batch(key: model_class) do |ids, loader, args| + model = args[:key] + results = model.where(id: ids) - results.each { |record| loader.call({ model: model, id: record.id }, record) } - end + results.each { |record| loader.call(record.id, record) } end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index 941a4f434a1..b8535575da5 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -4,6 +4,8 @@ module Gitlab module Graphql module Present class Instrumentation + SAFE_CONTEXT_KEYS = %i[current_user].freeze + def instrument(type, field) return field unless field.metadata[:type_class] @@ -22,7 +24,8 @@ module Gitlab next old_resolver.call(presented_type, args, context) end - presenter = presented_in.presenter_class.new(object, **context.to_h) + attrs = safe_context_values(context) + presenter = presented_in.presenter_class.new(object, **attrs) # we have to use the new `authorized_new` method, as `new` is protected wrapped = presented_type.class.authorized_new(presenter, context) @@ -34,6 +37,12 @@ module Gitlab resolve(resolve_with_presenter) end end + + private + + def safe_context_values(context) + context.to_h.slice(*SAFE_CONTEXT_KEYS) + end end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 5fec50eecd2..dd872caee0e 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -4,10 +4,10 @@ module Gitlab class GroupSearchResults < SearchResults attr_reader :group - def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, sort: nil, filters: {}) + def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, order_by: nil, sort: nil, filters: {}) @group = group - super(current_user, query, limit_projects, default_project_filter: default_project_filter, sort: sort, filters: filters) + super(current_user, query, limit_projects, default_project_filter: default_project_filter, order_by: order_by, sort: sort, filters: filters) end # rubocop:disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/hook_data/release_builder.rb b/lib/gitlab/hook_data/release_builder.rb new file mode 100644 index 00000000000..b15c260f4a8 --- /dev/null +++ b/lib/gitlab/hook_data/release_builder.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class ReleaseBuilder < BaseBuilder + def self.safe_hook_attributes + %i[ + id + created_at + description + name + released_at + tag + ].freeze + end + + alias_method :release, :object + + def build(action) + attrs = { + object_kind: object_kind, + project: release.project.hook_attrs, + description: absolute_image_urls(release.description), + url: Gitlab::UrlBuilder.build(release), + action: action, + assets: { + count: release.assets_count, + links: release.links.map(&:hook_attrs), + sources: release.sources.map(&:hook_attrs) + }, + commit: release.commit.hook_attrs + } + + release.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) + .merge!(attrs) + end + + private + + def object_kind + release.class.name.underscore + end + end + end +end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index e56b88dfce0..33054a5b9bf 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -24,7 +24,9 @@ module Gitlab return 'PO-syntax errors' => [parse_error] end - validate_entries + Gitlab::I18n.with_locale(locale) do + validate_entries + end end def parse_po @@ -156,12 +158,10 @@ module Gitlab end def validate_translation(errors, entry) - Gitlab::I18n.with_locale(locale) do - if entry.has_plural? - translate_plural(entry) - else - translate_singular(entry) - end + if entry.has_plural? + translate_plural(entry) + else + translate_singular(entry) end # `sprintf` could raise an `ArgumentError` when invalid passing something @@ -230,9 +230,7 @@ module Gitlab # This calls the C function that defines the pluralization rule, it can # return a boolean (`false` represents 0, `true` represents 1) or an integer # that specifies the plural form to be used for the given number - pluralization_result = Gitlab::I18n.with_locale(locale) do - FastGettext.pluralisation_rule.call(counter) - end + pluralization_result = FastGettext.pluralisation_rule.call(counter) case pluralization_result when false diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 7b8689069d8..8e78f6e274a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -55,9 +55,17 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user, - shared: shared, - project: project) + @project_tree ||= project_tree_class.new(user: current_user, + shared: shared, + project: project) + end + + def project_tree_class + sample_data_template? ? Gitlab::ImportExport::Project::Sample::TreeRestorer : Gitlab::ImportExport::Project::TreeRestorer + end + + def sample_data_template? + project&.import_data&.data&.dig('sample_data') end def avatar_restorer diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 0d9839b86cf..5c8edd485e5 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -29,9 +29,9 @@ module Gitlab json_decode(data) end - def consume_relation(importable_path, key) + def consume_relation(importable_path, key, mark_as_consumed: true) Enumerator.new do |documents| - next unless @consumed_relations.add?("#{importable_path}/#{key}") + next if mark_as_consumed && !@consumed_relations.add?("#{importable_path}/#{key}") # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") @@ -44,11 +44,6 @@ module Gitlab end end - # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) - def clear_consumed_relations - @consumed_relations.clear - end - private def json_decode(string) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index a0526ba0414..ae7ddbc5eba 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -185,6 +185,7 @@ excluded_attributes: - :secret - :encrypted_secret_token - :encrypted_secret_token_iv + - :repository_read_only merge_request_diff: - :external_diff - :stored_externally @@ -410,8 +411,25 @@ ee: - :deploy_access_levels - :service_desk_setting - :security_setting + - :push_rule included_attributes: issuable_sla: - :issue - :due_at + push_rule: + - :force_push_regex + - :delete_branch_regex + - :commit_message_regex + - :author_email_regex + - :file_name_regex + - :branch_name_regex + - :commit_message_negative_regex + - :max_file_size + - :deny_delete_tag + - :member_check + - :is_sample + - :prevent_secrets + - :reject_unsigned_commits + - :commit_committer_check + - :regexp_uses_re2 diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb index 2d989d21166..543fd25d883 100644 --- a/lib/gitlab/import_export/project/sample/date_calculator.rb +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -9,7 +9,6 @@ module Gitlab def initialize(dates) @dates = dates.dup - @dates.flatten! @dates.compact! @dates.sort! @dates.map! { |date| date.to_time.to_f } diff --git a/lib/gitlab/import_export/project/sample/relation_factory.rb b/lib/gitlab/import_export/project/sample/relation_factory.rb new file mode 100644 index 00000000000..6e59174f9a3 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/relation_factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class RelationFactory < Project::RelationFactory + DATE_MODELS = %i[issues milestones].freeze + + def initialize(date_calculator:, **args) + super(**args) + + @date_calculator = date_calculator + end + + private + + def setup_models + super + + # Override due date attributes in data hash for Sample Data templates + # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import + override_date_attributes + end + + def override_date_attributes + return unless DATE_MODELS.include?(@relation_name) + + @relation_hash['start_date'] = calculate_by_closest_date(@relation_hash['start_date']&.to_time) + @relation_hash['due_date'] = calculate_by_closest_date(@relation_hash['due_date']&.to_time) + end + + def calculate_by_closest_date(date) + return unless date + + @date_calculator.calculate_by_closest_date_to_average(date) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb new file mode 100644 index 00000000000..44ccb67a531 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class RelationTreeRestorer < ImportExport::RelationTreeRestorer + def initialize(*args) + super + + @date_calculator = Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) + end + + private + + def relation_factory_params(*args) + super.merge(date_calculator: @date_calculator) + end + + def dates + return [] if relation_reader.legacy? + + RelationFactory::DATE_MODELS.flat_map do |tag| + relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| + model.first['due_date'] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb deleted file mode 100644 index b0c3940b5f9..00000000000 --- a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Project - module Sample - class SampleDataRelationTreeRestorer < RelationTreeRestorer - DATE_MODELS = %i[issues milestones].freeze - - def initialize(*args) - super - - date_calculator - end - - private - - def build_relation(relation_key, relation_definition, data_hash) - # Override due date attributes in data hash for Sample Data templates - # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import - # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333) - override_date_attributes!(relation_key, data_hash) - super - end - - def override_date_attributes!(relation_key, data_hash) - return unless DATE_MODELS.include?(relation_key.to_sym) - - data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil? - data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil? - end - - # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) - def dates - unless relation_reader.legacy? - DATE_MODELS.map do |tag| - relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do - relation_reader.clear_consumed_relations - end - end - end - end - - def date_calculator - @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) - end - end - end - end - end -end diff --git a/lib/gitlab/import_export/project/sample/tree_restorer.rb b/lib/gitlab/import_export/project/sample/tree_restorer.rb new file mode 100644 index 00000000000..1d4b5328cb9 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/tree_restorer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class TreeRestorer < Project::TreeRestorer + def relation_tree_restorer_class + RelationTreeRestorer + end + + def relation_factory + RelationFactory + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index b1d647281ab..fb9e5be1877 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -85,11 +85,7 @@ module Gitlab end def relation_tree_restorer_class - sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer - end - - def sample_data_template? - @project&.import_data&.data&.dig('sample_data') + RelationTreeRestorer end def members_mapper diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 26e7d2cf765..428bcbe8dc5 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -86,6 +86,10 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) download_or_copy_upload(upload, upload_path) + rescue Errno::ENAMETOOLONG => e + # Do not fail entire project export if downloaded file has filename that exceeds 255 characters. + # Ignore raised exception, skip such upload, log the error and keep going with the export instead. + Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) end end end diff --git a/lib/gitlab/instrumentation/throttle.rb b/lib/gitlab/instrumentation/throttle.rb new file mode 100644 index 00000000000..0b7e990fb2e --- /dev/null +++ b/lib/gitlab/instrumentation/throttle.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + class Throttle + KEY = :instrumentation_throttle_safelist + + def self.safelist + Gitlab::SafeRequestStore[KEY] + end + + def self.safelist=(name) + Gitlab::SafeRequestStore[KEY] = name + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 3a29d2e7efa..d7228099eaf 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -21,6 +21,7 @@ module Gitlab instrument_rugged(payload) instrument_redis(payload) instrument_elasticsearch(payload) + instrument_throttle(payload) end def instrument_gitaly(payload) @@ -56,6 +57,11 @@ module Gitlab payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time end + def instrument_throttle(payload) + safelist = Gitlab::Instrumentation::Throttle.safelist + payload[:throttle_safelist] = safelist if safelist.present? + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 29cfec443e8..8565f664cd4 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -67,15 +67,6 @@ module Gitlab ::JSON.pretty_generate(object, opts) end - # Feature detection for using Oj instead of the `json` gem. - # - # @return [Boolean] - def enable_oj? - return false unless feature_table_exists? - - Feature.enabled?(:oj_json, default_enabled: true) - end - private # Convert JSON string into Ruby through toggleable adapters. @@ -91,11 +82,7 @@ module Gitlab def adapter_load(string, *args, **opts) opts = standardize_opts(opts) - if enable_oj? - Oj.load(string, opts) - else - ::JSON.parse(string, opts) - end + Oj.load(string, opts) rescue Oj::ParseError, Encoding::UndefinedConversionError => ex raise parser_error.new(ex) end @@ -120,11 +107,7 @@ module Gitlab # # @return [String] def adapter_dump(object, *args, **opts) - if enable_oj? - Oj.dump(object, opts) - else - ::JSON.dump(object, *args) - end + Oj.dump(object, opts) end # Generates JSON for an object but with fewer options, using toggleable adapters. @@ -135,11 +118,7 @@ module Gitlab def adapter_generate(object, opts = {}) opts = standardize_opts(opts) - if enable_oj? - Oj.generate(object, opts) - else - ::JSON.generate(object, opts) - end + Oj.generate(object, opts) end # Take a JSON standard options hash and standardize it to work across adapters @@ -149,11 +128,8 @@ module Gitlab # @return [Hash] def standardize_opts(opts) opts ||= {} - - if enable_oj? - opts[:mode] = :rails - opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names] - end + opts[:mode] = :rails + opts[:symbol_keys] = opts[:symbolize_keys] || opts[:symbolize_names] opts end @@ -213,7 +189,7 @@ module Gitlab # @param object [Object] # @return [String] def self.call(object, env = nil) - if Gitlab::Json.enable_oj? && Feature.enabled?(:grape_gitlab_json, default_enabled: true) + if Feature.enabled?(:grape_gitlab_json, default_enabled: true) Gitlab::Json.dump(object) else Grape::Formatter::Json.call(object, env) diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb deleted file mode 100644 index 49d2969f7f3..00000000000 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class BaseCommand - attr_reader :name, :files - - def initialize(rbac:, name:, files:) - @rbac = rbac - @name = name - @files = files - end - - def rbac? - @rbac - end - - def pod_resource - pod_service_account_name = rbac? ? service_account_name : nil - - Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate - end - - def generate_script - <<~HEREDOC - set -xeo pipefail - HEREDOC - end - - def pod_name - "install-#{name}" - end - - def config_map_resource - Gitlab::Kubernetes::ConfigMap.new(name, files).generate - end - - def service_account_resource - return unless rbac? - - Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate - end - - def cluster_role_binding_resource - return unless rbac? - - subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] - - Gitlab::Kubernetes::ClusterRoleBinding.new( - cluster_role_binding_name, - cluster_role_name, - subjects - ).generate - end - - def file_names - files.keys - end - - private - - def files_dir - "/data/helm/#{name}/config" - end - - def namespace - Gitlab::Kubernetes::Helm::NAMESPACE - end - - def service_account_name - Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT - end - - def cluster_role_binding_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING - end - - def cluster_role_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/certificate.rb b/lib/gitlab/kubernetes/helm/certificate.rb deleted file mode 100644 index 598714e0874..00000000000 --- a/lib/gitlab/kubernetes/helm/certificate.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module Kubernetes - module Helm - class Certificate - INFINITE_EXPIRY = 1000.years - SHORT_EXPIRY = 30.minutes - - attr_reader :key, :cert - - def key_string - @key.to_s - end - - def cert_string - @cert.to_pem - end - - def self.from_strings(key_string, cert_string) - key = OpenSSL::PKey::RSA.new(key_string) - cert = OpenSSL::X509::Certificate.new(cert_string) - new(key, cert) - end - - def self.generate_root - _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) - end - - def issue(expires_in: SHORT_EXPIRY) - self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) - end - - private - - def self._issue(signed_by:, expires_in:, certificate_authority:) - key = OpenSSL::PKey::RSA.new(4096) - public_key = key.public_key - - subject = OpenSSL::X509::Name.parse("/C=US") - - cert = OpenSSL::X509::Certificate.new - cert.subject = subject - - cert.issuer = signed_by&.cert&.subject || subject - - cert.not_before = Time.now - cert.not_after = expires_in.from_now - cert.public_key = public_key - cert.serial = 0x0 - cert.version = 2 - - if certificate_authority - extension_factory = OpenSSL::X509::ExtensionFactory.new - extension_factory.subject_certificate = cert - extension_factory.issuer_certificate = cert - cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) - cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) - cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) - end - - cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) - - new(key, cert) - end - - def initialize(key, cert) - @key = key - @cert = cert - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb deleted file mode 100644 index a9e93c0c90e..00000000000 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module ClientCommand - def init_command - <<~SHELL.chomp - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - SHELL - end - - def repository_command - ['helm', 'repo', 'add', name, repository].shelljoin if repository - end - - private - - def repository_update_command - 'helm repo update' - end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb deleted file mode 100644 index f8b9601bc98..00000000000 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class DeleteCommand < BaseCommand - include ClientCommand - - attr_reader :predelete, :postdelete - - def initialize(predelete: nil, postdelete: nil, **args) - super(**args) - @predelete = predelete - @postdelete = postdelete - end - - def generate_script - super + [ - init_command, - predelete, - delete_command, - postdelete - ].compact.join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - def delete_command - ['helm', 'delete', '--purge', name].shelljoin - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb deleted file mode 100644 index e4844e255c5..00000000000 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class InitCommand < BaseCommand - def generate_script - super + [ - init_helm_command - ].join("\n") - end - - private - - def init_helm_command - command = %w[helm init] + init_command_flags - - command.shelljoin - end - - def init_command_flags - tls_flags + optional_service_account_flag - end - - def tls_flags - [ - '--tiller-tls', - '--tiller-tls-verify', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tiller-tls-cert', "#{files_dir}/cert.pem", - '--tiller-tls-key', "#{files_dir}/key.pem" - ] - end - - def optional_service_account_flag - return [] unless rbac? - - ['--service-account', service_account_name] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb deleted file mode 100644 index d166842fce6..00000000000 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class InstallCommand < BaseCommand - include ClientCommand - - attr_reader :chart, :repository, :preinstall, :postinstall - attr_accessor :version - - def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) - super(**args) - @chart = chart - @version = version - @repository = repository - @preinstall = preinstall - @postinstall = postinstall - end - - def generate_script - super + [ - init_command, - repository_command, - repository_update_command, - preinstall, - install_command, - postinstall - ].compact.join("\n") - end - - private - - # Uses `helm upgrade --install` which means we can use this for both - # installation and uprade of applications - def install_command - command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def install_flag - ['--install'] - end - - def reset_values_flag - ['--reset-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def rbac_create_flag - if rbac? - %w[--set rbac.create=true,rbac.enabled=true] - else - %w[--set rbac.create=false,rbac.enabled=false] - end - end - - def optional_version_flag - return [] unless version - - ['--version', version] - end - - def rollback_support_flag - ['--atomic', '--cleanup-on-fail'] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb deleted file mode 100644 index a33dbdac134..00000000000 --- a/lib/gitlab/kubernetes/helm/patch_command.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# PatchCommand is for updating values in installed charts without overwriting -# existing values. -module Gitlab - module Kubernetes - module Helm - class PatchCommand < BaseCommand - include ClientCommand - - attr_reader :chart, :repository - attr_accessor :version - - def initialize(chart:, version:, repository: nil, **args) - super(**args) - - # version is mandatory to prevent chart mismatches - # we do not want our values interpreted in the context of the wrong version - raise ArgumentError, 'version is required' if version.blank? - - @chart = chart - @version = version - @repository = repository - end - - def generate_script - super + [ - init_command, - repository_command, - repository_update_command, - upgrade_command - ].compact.join("\n") - end - - private - - def upgrade_command - command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def reuse_values_flag - ['--reuse-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def version_flag - ['--version', version] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 75484f80070..9d0207e6b1f 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -27,7 +27,7 @@ module Gitlab def container_specification { name: 'helm', - image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}", + image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{command.class::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}-alpine-3.12", env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) @@ -50,11 +50,10 @@ module Gitlab end def generate_pod_env(command) - { - HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, - TILLER_NAMESPACE: namespace_name, + command.env.merge( + HELM_VERSION: command.class::HELM_VERSION, COMMAND_SCRIPT: command.generate_script - }.map { |key, value| { name: key, value: value } } + ).map { |key, value| { name: key, value: value } } end def volumes_specification diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb deleted file mode 100644 index f1f7938039c..00000000000 --- a/lib/gitlab/kubernetes/helm/reset_command.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class ResetCommand < BaseCommand - include ClientCommand - - def generate_script - super + [ - reset_helm_command, - delete_tiller_replicaset, - delete_tiller_clusterrolebinding - ].join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - private - - # This method can be delete once we upgrade Helm to > 12.13.0 - # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900 - # - # Tracking this method to be removed here: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155 - def delete_tiller_replicaset - delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - - def delete_tiller_clusterrolebinding - delete_args = %w[clusterrolebinding tiller-admin] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - - def reset_helm_command - command = %w[helm reset] + optional_tls_flags - - command.shelljoin - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/base_command.rb b/lib/gitlab/kubernetes/helm/v2/base_command.rb new file mode 100644 index 00000000000..931c2248310 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/base_command.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class BaseCommand + attr_reader :name, :files + + HELM_VERSION = '2.16.9' + + def initialize(rbac:, name:, files:) + @rbac = rbac + @name = name + @files = files + end + + def env + { TILLER_NAMESPACE: namespace } + end + + def rbac? + @rbac + end + + def pod_resource + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate + end + + def generate_script + <<~HEREDOC + set -xeo pipefail + HEREDOC + end + + def pod_name + "install-#{name}" + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, files).generate + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + + def file_names + files.keys + end + + private + + def files_dir + "/data/helm/#{name}/config" + end + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end + + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb new file mode 100644 index 00000000000..f603ff44ef3 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/certificate.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true +module Gitlab + module Kubernetes + module Helm + module V2 + class Certificate + INFINITE_EXPIRY = 1000.years + SHORT_EXPIRY = 30.minutes + + attr_reader :key, :cert + + def key_string + @key.to_s + end + + def cert_string + @cert.to_pem + end + + def self.from_strings(key_string, cert_string) + key = OpenSSL::PKey::RSA.new(key_string) + cert = OpenSSL::X509::Certificate.new(cert_string) + new(key, cert) + end + + def self.generate_root + _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) + end + + def issue(expires_in: SHORT_EXPIRY) + self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) + end + + private + + def self._issue(signed_by:, expires_in:, certificate_authority:) + key = OpenSSL::PKey::RSA.new(4096) + public_key = key.public_key + + subject = OpenSSL::X509::Name.parse("/C=US") + + cert = OpenSSL::X509::Certificate.new + cert.subject = subject + + cert.issuer = signed_by&.cert&.subject || subject + + cert.not_before = Time.now.utc + cert.not_after = expires_in.from_now.utc + cert.public_key = public_key + cert.serial = 0x0 + cert.version = 2 + + if certificate_authority + extension_factory = OpenSSL::X509::ExtensionFactory.new + extension_factory.subject_certificate = cert + extension_factory.issuer_certificate = cert + cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) + cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) + cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) + end + + cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) + + new(key, cert) + end + + def initialize(key, cert) + @key = key + @cert = cert + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb new file mode 100644 index 00000000000..88693a28d6c --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/client_command.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + module ClientCommand + def init_command + <<~SHELL.chomp + export HELM_HOST="localhost:44134" + tiller -listen ${HELM_HOST} -alsologtostderr & + helm init --client-only + SHELL + end + + def repository_command + ['helm', 'repo', 'add', name, repository].shelljoin if repository + end + + private + + def repository_update_command + 'helm repo update' + end + + def optional_tls_flags + return [] unless files.key?(:'ca.pem') + + [ + '--tls', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tls-cert', "#{files_dir}/cert.pem", + '--tls-key', "#{files_dir}/key.pem" + ] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/delete_command.rb b/lib/gitlab/kubernetes/helm/v2/delete_command.rb new file mode 100644 index 00000000000..4d52fc1398f --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/delete_command.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class DeleteCommand < BaseCommand + include ClientCommand + + attr_reader :predelete, :postdelete + + def initialize(predelete: nil, postdelete: nil, **args) + super(**args) + @predelete = predelete + @postdelete = postdelete + end + + def generate_script + super + [ + init_command, + predelete, + delete_command, + postdelete + ].compact.join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + def delete_command + ['helm', 'delete', '--purge', name].shelljoin + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/init_command.rb b/lib/gitlab/kubernetes/helm/v2/init_command.rb new file mode 100644 index 00000000000..f8b52feb5b6 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/init_command.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class InitCommand < BaseCommand + def generate_script + super + [ + init_helm_command + ].join("\n") + end + + private + + def init_helm_command + command = %w[helm init] + init_command_flags + + command.shelljoin + end + + def init_command_flags + tls_flags + optional_service_account_flag + end + + def tls_flags + [ + '--tiller-tls', + '--tiller-tls-verify', + '--tls-ca-cert', "#{files_dir}/ca.pem", + '--tiller-tls-cert', "#{files_dir}/cert.pem", + '--tiller-tls-key', "#{files_dir}/key.pem" + ] + end + + def optional_service_account_flag + return [] unless rbac? + + ['--service-account', service_account_name] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb new file mode 100644 index 00000000000..10e16723e45 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/install_command.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class InstallCommand < BaseCommand + include ClientCommand + + attr_reader :chart, :repository, :preinstall, :postinstall + attr_accessor :version + + def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) + super(**args) + @chart = chart + @version = version + @repository = repository + @preinstall = preinstall + @postinstall = postinstall + end + + def generate_script + super + [ + init_command, + repository_command, + repository_update_command, + preinstall, + install_command, + postinstall + ].compact.join("\n") + end + + private + + # Uses `helm upgrade --install` which means we can use this for both + # installation and uprade of applications + def install_command + command = ['helm', 'upgrade', name, chart] + + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def install_flag + ['--install'] + end + + def reset_values_flag + ['--reset-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def rbac_create_flag + if rbac? + %w[--set rbac.create=true,rbac.enabled=true] + else + %w[--set rbac.create=false,rbac.enabled=false] + end + end + + def optional_version_flag + return [] unless version + + ['--version', version] + end + + def rollback_support_flag + ['--atomic', '--cleanup-on-fail'] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb new file mode 100644 index 00000000000..2855e6444b1 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/patch_command.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# PatchCommand is for updating values in installed charts without overwriting +# existing values. +module Gitlab + module Kubernetes + module Helm + module V2 + class PatchCommand < BaseCommand + include ClientCommand + + attr_reader :chart, :repository + attr_accessor :version + + def initialize(chart:, version:, repository: nil, **args) + super(**args) + + # version is mandatory to prevent chart mismatches + # we do not want our values interpreted in the context of the wrong version + raise ArgumentError, 'version is required' if version.blank? + + @chart = chart + @version = version + @repository = repository + end + + def generate_script + super + [ + init_command, + repository_command, + repository_update_command, + upgrade_command + ].compact.join("\n") + end + + private + + def upgrade_command + command = ['helm', 'upgrade', name, chart] + + reuse_values_flag + + version_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def reuse_values_flag + ['--reuse-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def version_flag + ['--version', version] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb new file mode 100644 index 00000000000..172a0884c49 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v2/reset_command.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V2 + class ResetCommand < BaseCommand + include ClientCommand + + def generate_script + super + [ + reset_helm_command, + delete_tiller_replicaset, + delete_tiller_clusterrolebinding + ].join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + private + + # This method can be delete once we upgrade Helm to > 12.13.0 + # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900 + # + # Tracking this method to be removed here: + # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155 + def delete_tiller_replicaset + delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def delete_tiller_clusterrolebinding + delete_args = %w[clusterrolebinding tiller-admin] + + Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) + end + + def reset_helm_command + command = %w[helm reset] + optional_tls_flags + + command.shelljoin + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/base_command.rb b/lib/gitlab/kubernetes/helm/v3/base_command.rb new file mode 100644 index 00000000000..ca1bf5462f0 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/base_command.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V3 + class BaseCommand + attr_reader :name, :files + + HELM_VERSION = '3.2.4' + + def initialize(rbac:, name:, files:) + @rbac = rbac + @name = name + @files = files + end + + def env + {} + end + + def rbac? + @rbac + end + + def pod_resource + pod_service_account_name = rbac? ? service_account_name : nil + + Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate + end + + def generate_script + <<~HEREDOC + set -xeo pipefail + HEREDOC + end + + def pod_name + "install-#{name}" + end + + def config_map_resource + Gitlab::Kubernetes::ConfigMap.new(name, files).generate + end + + def service_account_resource + return unless rbac? + + Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate + end + + def cluster_role_binding_resource + return unless rbac? + + subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] + + Gitlab::Kubernetes::ClusterRoleBinding.new( + cluster_role_binding_name, + cluster_role_name, + subjects + ).generate + end + + def file_names + files.keys + end + + def repository_command + ['helm', 'repo', 'add', name, repository].shelljoin if repository + end + + private + + def repository_update_command + 'helm repo update' + end + + def namespace_flag + ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] + end + + def namespace + Gitlab::Kubernetes::Helm::NAMESPACE + end + + def service_account_name + Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT + end + + def cluster_role_binding_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING + end + + def cluster_role_name + Gitlab::Kubernetes::Helm::CLUSTER_ROLE + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/delete_command.rb b/lib/gitlab/kubernetes/helm/v3/delete_command.rb new file mode 100644 index 00000000000..f628e852f54 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/delete_command.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V3 + class DeleteCommand < BaseCommand + attr_reader :predelete, :postdelete + + def initialize(predelete: nil, postdelete: nil, **args) + super(**args) + @predelete = predelete + @postdelete = postdelete + end + + def generate_script + super + [ + predelete, + delete_command, + postdelete + ].compact.join("\n") + end + + def pod_name + "uninstall-#{name}" + end + + def delete_command + ['helm', 'uninstall', name, *namespace_flag].shelljoin + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb new file mode 100644 index 00000000000..20d17f49115 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/install_command.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Helm + module V3 + class InstallCommand < BaseCommand + attr_reader :chart, :repository, :preinstall, :postinstall + attr_accessor :version + + def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) + super(**args) + @chart = chart + @version = version + @repository = repository + @preinstall = preinstall + @postinstall = postinstall + end + + def generate_script + super + [ + repository_command, + repository_update_command, + preinstall, + install_command, + postinstall + ].compact.join("\n") + end + + private + + # Uses `helm upgrade --install` which means we can use this for both + # installation and uprade of applications + def install_command + command = ['helm', 'upgrade', name, chart] + + install_flag + + rollback_support_flag + + reset_values_flag + + optional_version_flag + + rbac_create_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def install_flag + ['--install'] + end + + def reset_values_flag + ['--reset-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def rbac_create_flag + if rbac? + %w[--set rbac.create=true,rbac.enabled=true] + else + %w[--set rbac.create=false,rbac.enabled=false] + end + end + + def optional_version_flag + return [] unless version + + ['--version', version] + end + + def rollback_support_flag + ['--atomic', '--cleanup-on-fail'] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb new file mode 100644 index 00000000000..00f340591e7 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/v3/patch_command.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# PatchCommand is for updating values in installed charts without overwriting +# existing values. +module Gitlab + module Kubernetes + module Helm + module V3 + class PatchCommand < BaseCommand + attr_reader :chart, :repository + attr_accessor :version + + def initialize(chart:, version:, repository: nil, **args) + super(**args) + + # version is mandatory to prevent chart mismatches + # we do not want our values interpreted in the context of the wrong version + raise ArgumentError, 'version is required' if version.blank? + + @chart = chart + @version = version + @repository = repository + end + + def generate_script + super + [ + repository_command, + repository_update_command, + upgrade_command + ].compact.join("\n") + end + + private + + def upgrade_command + command = ['helm', 'upgrade', name, chart] + + reuse_values_flag + + version_flag + + namespace_flag + + value_flag + + command.shelljoin + end + + def reuse_values_flag + ['--reuse-values'] + end + + def value_flag + ['-f', "/data/helm/#{name}/config/values.yaml"] + end + + def version_flag + ['--version', version] + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index 13cd6dcad3f..a25f005d81e 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -61,18 +61,11 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client delegate :update_cluster_role_binding, - to: :rbac_client - - # RBAC methods delegates to the apis/rbac.authorization.k8s.io api - # group client - delegate :create_role, - :get_role, - :update_role, - to: :rbac_client - - # RBAC methods delegates to the apis/rbac.authorization.k8s.io api - # group client - delegate :update_role_binding, + :create_role, + :get_role, + :update_role, + :delete_role_binding, + :update_role_binding, to: :rbac_client # non-entity methods that can only work with the core client @@ -182,10 +175,21 @@ module Gitlab end end + def patch_ingress(*args) + extensions_client.discover unless extensions_client.discovered + + if extensions_client.respond_to?(:patch_ingress) + extensions_client.patch_ingress(*args) + else + networking_client.patch_ingress(*args) + end + end + def create_or_update_cluster_role_binding(resource) update_cluster_role_binding(resource) end + # Note that we cannot update roleRef as that is immutable def create_or_update_role_binding(resource) update_role_binding(resource) end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index f7eaafeb446..4482610523e 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -24,6 +24,7 @@ module Gitlab @api ||= ::Octokit::Client.new( access_token: access_token, api_endpoint: api_endpoint, + web_endpoint: web_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -85,6 +86,10 @@ module Gitlab end end + def web_endpoint + host.presence || ::Octokit::Default.web_endpoint + end + def config Gitlab::Auth::OAuth::Provider.config_for('github') end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 3f9fd1b1a19..a17e3b1ad5c 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -36,7 +36,7 @@ module Gitlab } end - @client = Client.new(credentials[:user], opts) + @client = Client.new(credentials[:user], **opts) end def execute @@ -303,6 +303,8 @@ module Gitlab end imported!(resource_type) + rescue ::Octokit::NotFound => e + errors << { type: resource_type, errors: e.message } end def imported?(resource_type) diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index f6bda0dbea4..23d7eb67312 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -3,43 +3,70 @@ module Gitlab module Metrics class RequestsRackMiddleware - HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze + HTTP_METHODS = { + "delete" => %w(200 202 204 303 400 401 403 404 500 503), + "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503), + "head" => %w(200 204 301 302 303 401 403 404 410 500), + "options" => %w(200 404), + "patch" => %w(200 202 204 400 403 404 409 416 500), + "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503), + "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) + }.freeze HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category' FEATURE_CATEGORY_DEFAULT = 'unknown' + # These were the top 5 categories at a point in time, chosen as a + # reasonable default. If we initialize every category we'll end up + # with an explosion in unused metric combinations, but we want the + # most common ones to be always present. + FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization', + 'code_review', 'continuous_integration', + 'not_owned', 'source_code_management', + FEATURE_CATEGORY_DEFAULT].freeze + def initialize(app) @app = app end - def self.http_request_total - @http_request_total ||= ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') + def self.http_requests_total + ::Gitlab::Metrics.counter(:http_requests_total, 'Request count') end def self.rack_uncaught_errors_count - @rack_uncaught_errors_count ||= ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') + ::Gitlab::Metrics.counter(:rack_uncaught_errors_total, 'Request handling uncaught errors count') end def self.http_request_duration_seconds - @http_request_duration_seconds ||= ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', - {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) + ::Gitlab::Metrics.histogram(:http_request_duration_seconds, 'Request handling execution time', + {}, [0.05, 0.1, 0.25, 0.5, 0.7, 1, 2.5, 5, 10, 25]) end def self.http_health_requests_total - @http_health_requests_total ||= ::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count') + ::Gitlab::Metrics.counter(:http_health_requests_total, 'Health endpoint request count') end - def self.initialize_http_request_duration_seconds - HTTP_METHODS.each do |method| + def self.initialize_metrics + # This initialization is done to avoid gaps in scraped metrics after + # restarts. It makes sure all counters/histograms are available at + # process start. + # + # For example `rate(http_requests_total{status="500"}[1m])` would return + # no data until the first 500 error would occur. + HTTP_METHODS.each do |method, statuses| http_request_duration_seconds.get({ method: method }) + + statuses.product(FEATURE_CATEGORIES_TO_INITIALIZE) do |status, feature_category| + http_requests_total.get({ method: method, status: status, feature_category: feature_category }) + end end end def call(env) method = env['REQUEST_METHOD'].downcase - method = 'INVALID' unless HTTP_METHODS.include?(method) + method = 'INVALID' unless HTTP_METHODS.key?(method) started = Time.now.to_f health_endpoint = health_endpoint?(env['PATH_INFO']) status = 'undefined' @@ -61,9 +88,13 @@ module Gitlab raise ensure if health_endpoint - RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method) + RequestsRackMiddleware.http_health_requests_total.increment(status: status.to_s, method: method) else - RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT) + RequestsRackMiddleware.http_requests_total.increment( + status: status.to_s, + method: method, + feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT + ) end end end diff --git a/lib/gitlab/middleware/handle_malformed_strings.rb b/lib/gitlab/middleware/handle_malformed_strings.rb new file mode 100644 index 00000000000..84f7e2e1b14 --- /dev/null +++ b/lib/gitlab/middleware/handle_malformed_strings.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + # There is no valid reason for a request to contain a malformed string + # so just return HTTP 400 (Bad Request) if we receive one + class HandleMalformedStrings + include ActionController::HttpAuthentication::Basic + + NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze + + attr_reader :app + + def initialize(app) + @app = app + end + + def call(env) + return [400, { 'Content-Type' => 'text/plain' }, ['Bad Request']] if request_contains_malformed_string?(env) + + app.call(env) + end + + private + + def request_contains_malformed_string?(env) + return false if ENV['DISABLE_REQUEST_VALIDATION'] == '1' + + # Duplicate the env, so it is not modified when accessing the parameters + # https://github.com/rails/rails/blob/34991a6ae2fc68347c01ea7382fa89004159e019/actionpack/lib/action_dispatch/http/parameters.rb#L59 + # The modification causes problems with our multipart middleware + request = ActionDispatch::Request.new(env.dup) + + return true if malformed_path?(request.path) + return true if credentials_malformed?(request) + + request.params.values.any? do |value| + param_has_null_byte?(value) + end + rescue ActionController::BadRequest + # If we can't build an ActionDispatch::Request something's wrong + # This would also happen if `#params` contains invalid UTF-8 + # in this case we'll return a 400 + # + true + end + + def malformed_path?(path) + string_malformed?(Rack::Utils.unescape(path)) + rescue ArgumentError + # Rack::Utils.unescape raised this, path is malformed. + true + end + + def credentials_malformed?(request) + credentials = if has_basic_credentials?(request) + decode_credentials(request).presence + else + request.authorization.presence + end + + return false unless credentials + + string_malformed?(credentials) + end + + def param_has_null_byte?(value, depth = 0) + # Guard against possible attack sending large amounts of nested params + # Should be safe as deeply nested params are highly uncommon. + return false if depth > 2 + + depth += 1 + + if value.respond_to?(:match) + string_malformed?(value) + elsif value.respond_to?(:values) + value.values.any? do |hash_value| + param_has_null_byte?(hash_value, depth) + end + elsif value.is_a?(Array) + value.any? do |array_value| + param_has_null_byte?(array_value, depth) + end + else + false + end + end + + def string_malformed?(string) + # We're using match rather than include, because that will raise an ArgumentError + # when the string contains invalid UTF8 + # + # We try to encode the string from ASCII-8BIT to UTF8. If we failed to do + # so for certain characters in the string, those chars are probably incomplete + # multibyte characters. + string.encode(Encoding::UTF_8).match?(NULL_BYTE_REGEX) + rescue ArgumentError, Encoding::UndefinedConversionError + # If we're here, we caught a malformed string. Return true + true + end + end + end +end diff --git a/lib/gitlab/middleware/handle_null_bytes.rb b/lib/gitlab/middleware/handle_null_bytes.rb deleted file mode 100644 index c88dfb6ee0b..00000000000 --- a/lib/gitlab/middleware/handle_null_bytes.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Middleware - # There is no valid reason for a request to contain a null byte (U+0000) - # so just return HTTP 400 (Bad Request) if we receive one - class HandleNullBytes - NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze - - attr_reader :app - - def initialize(app) - @app = app - end - - def call(env) - return [400, {}, ["Bad Request"]] if request_has_null_byte?(env) - - app.call(env) - end - - private - - def request_has_null_byte?(request) - return false if ENV['REJECT_NULL_BYTES'] == "1" - - request = Rack::Request.new(request) - - request.params.values.any? do |value| - param_has_null_byte?(value) - end - end - - def param_has_null_byte?(value, depth = 0) - # Guard against possible attack sending large amounts of nested params - # Should be safe as deeply nested params are highly uncommon. - return false if depth > 2 - - depth += 1 - - if value.respond_to?(:match) - string_contains_null_byte?(value) - elsif value.respond_to?(:values) - value.values.any? do |hash_value| - param_has_null_byte?(hash_value, depth) - end - elsif value.is_a?(Array) - value.any? do |array_value| - param_has_null_byte?(array_value, depth) - end - else - false - end - end - - def string_contains_null_byte?(string) - string.match?(NULL_BYTE_REGEX) - end - end - end -end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index cfea4aaddf3..101172cdfcc 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -9,20 +9,19 @@ module Gitlab APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance' - WHITELISTED_GIT_ROUTES = { - 'repositories/git_http' => %w{git_upload_pack git_receive_pack} + ALLOWLISTED_GIT_ROUTES = { + 'repositories/git_http' => %w{git_upload_pack} }.freeze - WHITELISTED_GIT_LFS_ROUTES = { - 'repositories/lfs_api' => %w{batch}, - 'repositories/lfs_locks_api' => %w{verify create unlock} + ALLOWLISTED_GIT_LFS_BATCH_ROUTES = { + 'repositories/lfs_api' => %w{batch} }.freeze - WHITELISTED_GIT_REVISION_ROUTES = { + ALLOWLISTED_GIT_REVISION_ROUTES = { 'projects/compare' => %w{create} }.freeze - WHITELISTED_SESSION_ROUTES = { + ALLOWLISTED_SESSION_ROUTES = { 'sessions' => %w{destroy}, 'admin/sessions' => %w{create destroy} }.freeze @@ -55,7 +54,7 @@ module Gitlab def disallowed_request? DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && - !whitelisted_routes + !allowlisted_routes end def json_request? @@ -87,8 +86,8 @@ module Gitlab end # Overridden in EE module - def whitelisted_routes - workhorse_passthrough_route? || internal_route? || lfs_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query? + def allowlisted_routes + workhorse_passthrough_route? || internal_route? || lfs_batch_route? || compare_git_revisions_route? || sidekiq_route? || session_route? || graphql_query? end # URL for requests passed through gitlab-workhorse to rails-web @@ -96,9 +95,9 @@ module Gitlab def workhorse_passthrough_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match return false unless request.post? && - request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') + request.path.end_with?('.git/git-upload-pack') - WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def internal_route? @@ -109,18 +108,16 @@ module Gitlab # Calling route_hash may be expensive. Only do it if we think there's a possible match return false unless request.post? && request.path.end_with?('compare') - WHITELISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_GIT_REVISION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end - def lfs_route? + # Batch upload requests are blocked in: + # https://gitlab.com/gitlab-org/gitlab/blob/master/app/controllers/repositories/lfs_api_controller.rb#L106 + def lfs_batch_route? # Calling route_hash may be expensive. Only do it if we think there's a possible match - unless request.path.end_with?('/info/lfs/objects/batch', - '/info/lfs/locks', '/info/lfs/locks/verify') || - %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) - return false - end + return unless request.path.end_with?('/info/lfs/objects/batch') - WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_GIT_LFS_BATCH_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def session_route? @@ -128,7 +125,7 @@ module Gitlab return false unless request.post? && request.path.end_with?('/users/sign_out', '/admin/session', '/admin/session/destroy') - WHITELISTED_SESSION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + ALLOWLISTED_SESSION_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def sidekiq_route? diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index 2dd7d08a58b..a3c0fdcf467 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -8,7 +8,7 @@ module Gitlab end def call(env) - Gitlab::UrlBlocker.validate!(env[:url], { allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests? }) + Gitlab::UrlBlocker.validate!(env[:url], allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests?) @app.call(env) end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index b60ecb6631b..541f9b06842 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -96,16 +96,6 @@ module Gitlab args[:strategy_class] = args[:strategy_class].constantize end - # Providers that are known to depend on rack-oauth2, like those using - # Omniauth::Strategies::OpenIDConnect, need to be quirked so the - # client_auth_method argument value is passed as a symbol. - if (args[:strategy_class] == OmniAuth::Strategies::OpenIDConnect || - args[:name] == 'openid_connect') && - args[:client_auth_method].is_a?(String) - - args[:client_auth_method] = args[:client_auth_method].to_sym - end - args end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 706c16f6149..ad0a5c80604 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -49,6 +49,9 @@ module Gitlab s search sent_notifications + sitemap + sitemap.xml + sitemap.xml.gz slash-command-logo.png snippets unsubscribes @@ -251,6 +254,14 @@ module Gitlab %r{#{personal_snippet_path_regex}|#{project_snippet_path_regex}} end + def container_image_regex + @container_image_regex ||= %r{([\w\.-]+\/){0,1}[\w\.-]+}.freeze + end + + def container_image_blob_sha_regex + @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze + end + private def personal_snippet_path_regex diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 333564bee01..6719dc8362b 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -4,11 +4,11 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, query, project:, repository_ref: nil, sort: nil, filters: {}) + def initialize(current_user, query, project:, repository_ref: nil, order_by: nil, sort: nil, filters: {}) @project = project @repository_ref = repository_ref.presence - super(current_user, query, [project], sort: sort, filters: filters) + super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters) end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) @@ -75,15 +75,6 @@ module Gitlab @commits_count ||= commits(limit: count_limit).count end - def single_commit_result? - return false if commits_count != 1 - - counts = %i(limited_milestones_count limited_notes_count - limited_merge_requests_count limited_issues_count - limited_blobs_count wiki_blobs_count) - counts.all? { |count_method| public_send(count_method) == 0 } # rubocop:disable GitlabSecurity/PublicSend - end - private def paginated_commits(page, per_page) diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index dd7a27ead01..1294e475145 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,9 +29,9 @@ module Gitlab # Anything, including `/cmd arg` which are ignored by this filter # ` - ^.*`\n* + `\n* .+? - \n*`$ + \n*` ) }mix.freeze diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index c8c949a9363..1986b7a1789 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -56,21 +56,21 @@ module Gitlab @updates[:merge] = params[:merge_request_diff_head_sha] end - desc 'Toggle the Work In Progress status' + desc 'Toggle the Draft status' explanation do noun = quick_action_target.to_ability_name.humanize(capitalize: false) if quick_action_target.work_in_progress? - _("Unmarks this %{noun} as Work In Progress.") + _("Unmarks this %{noun} as a draft.") else - _("Marks this %{noun} as Work In Progress.") + _("Marks this %{noun} as a draft.") end % { noun: noun } end execution_message do noun = quick_action_target.to_ability_name.humanize(capitalize: false) if quick_action_target.work_in_progress? - _("Unmarked this %{noun} as Work In Progress.") + _("Unmarked this %{noun} as a draft.") else - _("Marked this %{noun} as Work In Progress.") + _("Marked this %{noun} as a draft.") end % { noun: noun } end @@ -80,7 +80,7 @@ module Gitlab # Allow it to mark as WIP on MR creation page _or_ through MR notes. (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end - command :wip do + command :draft, :wip do @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 5584323789b..6f80c7d439f 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -18,6 +18,10 @@ module Gitlab pool.with { |redis| yield redis } end + def version + with { |redis| redis.info['redis_version'] } + end + def pool @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 01aff48b08b..d7501fc7068 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,7 +4,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project - merge_request snippet commit commit_range directly_addressed_user epic iteration).freeze + merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability).freeze attr_accessor :project, :current_user, :author # This counter is increased by a number of references filtered out by # banzai reference exctractor. Note that this counter is stateful and @@ -38,7 +38,7 @@ module Gitlab end REFERABLES.each do |type| - define_method("#{type}s") do + define_method(type.to_s.pluralize) do @references[type] ||= references(type) end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 1b169b6186b..4ae6297f6f5 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @composer_package_version_regex ||= %r{^v?(\d+(\.(\d+|x))*(-.+)?)}.freeze end + def composer_dev_version_regex + @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze + end + def package_name_regex @package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze end diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb index 03d9f961dd9..0ed31176dd8 100644 --- a/lib/gitlab/repository_size_checker.rb +++ b/lib/gitlab/repository_size_checker.rb @@ -32,18 +32,24 @@ module Gitlab def changes_will_exceed_size_limit?(change_size) return false unless enabled? - change_size > limit || exceeded_size(change_size) > 0 + above_size_limit? || exceeded_size(change_size) > 0 end # @param change_size [int] in bytes def exceeded_size(change_size = 0) - current_size + change_size - limit + size = current_size + change_size - limit + + [size, 0].max end def error_message @error_message_object ||= ::Gitlab::RepositorySizeErrorMessage.new(self) end + def additional_repo_storage_available? + false + end + private attr_reader :namespace diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb index 556190453de..8da840779c9 100644 --- a/lib/gitlab/repository_size_error_message.rb +++ b/lib/gitlab/repository_size_error_message.rb @@ -4,7 +4,7 @@ module Gitlab class RepositorySizeErrorMessage include ActiveSupport::NumberHelper - delegate :current_size, :limit, :exceeded_size, to: :@checker + delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker # @param checher [RepositorySizeChecker] def initialize(checker) @@ -24,7 +24,11 @@ module Gitlab end def new_changes_error - "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}" + if additional_repo_storage_available? + "Your push to this repository has been rejected because it would exceed storage limits. Please contact your GitLab administrator for more information." + else + "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}" + end end def more_info_message diff --git a/lib/gitlab/repository_url_builder.rb b/lib/gitlab/repository_url_builder.rb index 2b88af1f77c..a2d0d50d20b 100644 --- a/lib/gitlab/repository_url_builder.rb +++ b/lib/gitlab/repository_url_builder.rb @@ -4,9 +4,6 @@ module Gitlab module RepositoryUrlBuilder class << self def build(path, protocol: :ssh) - # TODO: See https://gitlab.com/gitlab-org/gitlab/-/issues/213021 - path = path.sub('@snippets', 'snippets') - case protocol when :ssh ssh_url(path) diff --git a/lib/gitlab/robots_txt/parser.rb b/lib/gitlab/robots_txt/parser.rb index b9a3837e468..604d2f9b35b 100644 --- a/lib/gitlab/robots_txt/parser.rb +++ b/lib/gitlab/robots_txt/parser.rb @@ -3,34 +3,68 @@ module Gitlab module RobotsTxt class Parser - attr_reader :disallow_rules + DISALLOW_REGEX = /^disallow: /i.freeze + ALLOW_REGEX = /^allow: /i.freeze + + attr_reader :disallow_rules, :allow_rules def initialize(content) @raw_content = content - @disallow_rules = parse_raw_content! + @disallow_rules, @allow_rules = parse_raw_content! end def disallowed?(path) + return false if allow_rules.any? { |rule| path =~ rule } + disallow_rules.any? { |rule| path =~ rule } end private - # This parser is very basic as it only knows about `Disallow:` lines, - # and simply ignores all other lines. + # This parser is very basic as it only knows about `Disallow:` + # and `Allow:` lines, and simply ignores all other lines. # - # Order of predecence, 'Allow:`, etc are ignored for now. + # Patterns ending in `$`, and `*` for 0 or more characters are recognized. + # + # It is case insensitive and `Allow` rules takes precedence + # over `Disallow`. def parse_raw_content! - @raw_content.each_line.map do |line| - if line.start_with?('Disallow:') - value = line.sub('Disallow:', '').strip - value = Regexp.escape(value).gsub('\*', '.*') - Regexp.new("^#{value}") - else - nil + disallowed = [] + allowed = [] + + @raw_content.each_line.each do |line| + if disallow_rule?(line) + disallowed << get_disallow_pattern(line) + elsif allow_rule?(line) + allowed << get_allow_pattern(line) end - end.compact + end + + [disallowed, allowed] + end + + def disallow_rule?(line) + line =~ DISALLOW_REGEX + end + + def get_disallow_pattern(line) + get_pattern(line, DISALLOW_REGEX) + end + + def allow_rule?(line) + line =~ ALLOW_REGEX + end + + def get_allow_pattern(line) + get_pattern(line, ALLOW_REGEX) + end + + def get_pattern(line, rule_regex) + value = line.sub(rule_regex, '').strip + value = Regexp.escape(value).gsub('\*', '.*') + value = value.sub(/\\\$$/, '$') + Regexp.new("^#{value}") end end end diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index fc1abc064c7..183e582925d 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include BlobActiveModel - attr_reader :project, :content_match, :blob_path + attr_reader :project, :content_match, :blob_path, :highlight_line PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze @@ -26,6 +26,7 @@ module Gitlab @binary_basename = opts.fetch(:basename, nil) @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) + @highlight_line = opts.fetch(:highlight_line, nil) @binary_data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) @project = opts.fetch(:project, nil) diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb new file mode 100644 index 00000000000..3395c34d171 --- /dev/null +++ b/lib/gitlab/search/sort_options.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Search + module SortOptions + def sort_and_direction(order_by, sort) + # Due to different uses of sort param in web vs. API requests we prefer + # order_by when present + case [order_by, sort] + when %w[created_at asc], [nil, 'created_asc'] + :created_at_asc + when %w[created_at desc], [nil, 'created_desc'] + :created_at_desc + else + :unknown + end + end + module_function :sort_and_direction # rubocop: disable Style/AccessModifierDeclarations + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index b81264c5d0c..0091ae1e8ce 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query, :sort, :filters + attr_reader :current_user, :query, :order_by, :sort, :filters # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -19,11 +19,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {}) + def initialize(current_user, query, limit_projects = nil, order_by: nil, sort: nil, default_project_filter: false, filters: {}) @current_user = current_user @query = query @limit_projects = limit_projects || Project.all @default_project_filter = default_project_filter + @order_by = order_by @sort = sort @filters = filters end @@ -94,10 +95,6 @@ module Gitlab @limited_users_count ||= limited_count(users) end - def single_commit_result? - false - end - def count_limit COUNT_LIMIT end @@ -132,13 +129,15 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_sort(scope) - case sort - when 'oldest' + # Due to different uses of sort param we prefer order_by when + # present + case ::Gitlab::Search::SortOptions.sort_and_direction(order_by, sort) + when :created_at_asc scope.reorder('created_at ASC') - when 'newest' + when :created_at_desc scope.reorder('created_at DESC') else - scope + scope.reorder('created_at DESC') end end # rubocop: enable CodeReuse/ActiveRecord @@ -219,7 +218,7 @@ module Gitlab params[:state] = filters[:state] if filters.key?(:state) - if [true, false].include?(filters[:confidential]) && Feature.enabled?(:search_filter_by_confidential) + if [true, false].include?(filters[:confidential]) params[:confidential] = filters[:confidential] end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 4df6a50c8dd..259d3e300b6 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -99,6 +99,7 @@ module Gitlab config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path config[:gitlab] = { url: Gitlab.config.gitlab.url } + config[:logging] = { dir: Rails.root.join('log').to_s } TomlRB.dump(config) end diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index 1e5d23a8405..e471517c50a 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -47,16 +47,24 @@ module Gitlab option_parser.parse!(argv) + # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 + if @queue_selector && @experimental_queue_selector + raise CommandError, + 'You cannot specify --queue-selector and --experimental-queue-selector together' + end + all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path) queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path) queue_groups = argv.map do |queues| next queue_names if queues == '*' - # When using the experimental queue query syntax, we treat - # each queue group as a worker attribute query, and resolve - # the queues for the queue group using this query. - if @experimental_queue_selector + # When using the queue query syntax, we treat each queue group + # as a worker attribute query, and resolve the queues for the + # queue group using this query. + + # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 + if @queue_selector || @experimental_queue_selector SidekiqConfig::CliMethods.query_workers(queues, all_queues) else SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names) @@ -182,7 +190,12 @@ module Gitlab @rails_path = path end - opt.on('--experimental-queue-selector', 'EXPERIMENTAL: Run workers based on the provided selector') do |experimental_queue_selector| + opt.on('--queue-selector', 'Run workers based on the provided selector') do |queue_selector| + @queue_selector = queue_selector + end + + # Remove with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 + opt.on('--experimental-queue-selector', 'DEPRECATED: use --queue-selector-instead') do |experimental_queue_selector| @experimental_queue_selector = experimental_queue_selector end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index 326dfdae661..dc81c34c4d0 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -16,7 +16,7 @@ module Gitlab # Add process id params job['pid'] = ::Process.pid - job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + job.delete('args') unless SidekiqLogArguments.enabled? job end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb index 6fdef4c354e..63e8bee4443 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb @@ -8,6 +8,7 @@ module Gitlab STRATEGIES = { until_executing: UntilExecuting, + until_executed: UntilExecuted, none: None }.freeze diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb new file mode 100644 index 00000000000..df5df590281 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/base.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + class Base + def initialize(duplicate_job) + @duplicate_job = duplicate_job + end + + def schedule(job) + raise NotImplementedError + end + + def perform(_job) + raise NotImplementedError + end + + private + + attr_reader :duplicate_job + + def strategy_name + self.class.name.to_s.demodulize.underscore.humanize.downcase + end + + def check! + # The default expiry time is the DuplicateJob::DUPLICATE_KEY_TTL already + # Only the strategies de-duplicating when scheduling + duplicate_job.check! + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb new file mode 100644 index 00000000000..59b0e7e29da --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + module DeduplicatesWhenScheduling + def initialize(duplicate_job) + @duplicate_job = duplicate_job + end + + def schedule(job) + if deduplicatable_job? && check! && duplicate_job.duplicate? + job['duplicate-of'] = duplicate_job.existing_jid + + if duplicate_job.droppable? + Gitlab::SidekiqLogging::DeduplicationLogger.instance.log( + job, "dropped #{strategy_name}", duplicate_job.options) + return false + end + end + + yield + end + + private + + def deduplicatable_job? + !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled] + end + + def check! + duplicate_job.check!(expiry) + end + + def expiry + return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled? + + time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i + + time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb index cd101cd16b6..acbe0efaafa 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/none.rb @@ -5,10 +5,7 @@ module Gitlab module DuplicateJobs module Strategies # This strategy will never deduplicate a job - class None - def initialize(_duplicate_job) - end - + class None < Base def schedule(_job) yield end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb new file mode 100644 index 00000000000..738efa36fc8 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module DuplicateJobs + module Strategies + # This strategy takes a lock before scheduling the job in a queue and + # removes the lock after the job has executed preventing a new job to be queued + # while a job is still executing. + class UntilExecuted < Base + include DeduplicatesWhenScheduling + + def perform(_job) + yield + + duplicate_job.delete! + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb index 46ce0eb4a91..68d66383b2b 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -7,50 +7,14 @@ module Gitlab # This strategy takes a lock before scheduling the job in a queue and # removes the lock before the job starts allowing a new job to be queued # while a job is still executing. - class UntilExecuting - def initialize(duplicate_job) - @duplicate_job = duplicate_job - end - - def schedule(job) - if deduplicatable_job? && check! && duplicate_job.duplicate? - job['duplicate-of'] = duplicate_job.existing_jid - - if duplicate_job.droppable? - Gitlab::SidekiqLogging::DeduplicationLogger.instance.log( - job, "dropped until executing", duplicate_job.options) - return false - end - end - - yield - end + class UntilExecuting < Base + include DeduplicatesWhenScheduling def perform(_job) duplicate_job.delete! yield end - - private - - attr_reader :duplicate_job - - def deduplicatable_job? - !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled] - end - - def check! - duplicate_job.check!(expiry) - end - - def expiry - return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled? - - time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i - - time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL - end end end end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index ff24ec69ab0..0a2cee75af7 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -34,7 +34,7 @@ module Gitlab delegate :project, to: :repository def supported_extensions - %w[.md].freeze + %w[.md .md.erb].freeze end def commit_id @@ -50,8 +50,6 @@ module Gitlab end def extension_supported? - return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) - supported_extensions.any? { |ext| path.end_with?(ext) } end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 50e09bdcdd6..e84937ec4ad 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -105,6 +105,20 @@ module Gitlab files.map { |t| { name: t.name } } end end + + def template_subsets(project = nil) + return [] if project && !project.repository.exists? + + if categories.any? + categories.keys.map do |category| + files = self.by_category(category, project) + [category, files.map { |t| { key: t.key, name: t.name, content: t.content } }] + end.to_h + else + files = self.all(project) + files.map { |t| { key: t.key, name: t.name, content: t.content } } + end + end end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 02d354ec43a..19be468e3d5 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'snowplow-tracker' - module Gitlab module Tracking SNOWPLOW_NAMESPACE = 'gl' @@ -27,16 +25,11 @@ module Gitlab end def event(category, action, label: nil, property: nil, value: nil, context: nil) - return unless enabled? - - snowplow.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + snowplow.event(category, action, label: label, property: property, value: value, context: context) end def self_describing_event(schema_url, event_data_json, context: nil) - return unless enabled? - - event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json) - snowplow.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) + snowplow.self_describing_event(schema_url, event_data_json, context: context) end def snowplow_options(group) @@ -54,19 +47,7 @@ module Gitlab private def snowplow - @snowplow ||= SnowplowTracker::Tracker.new( - emitter, - SnowplowTracker::Subject.new, - SNOWPLOW_NAMESPACE, - Gitlab::CurrentSettings.snowplow_app_id - ) - end - - def emitter - SnowplowTracker::AsyncEmitter.new( - Gitlab::CurrentSettings.snowplow_collector_hostname, - protocol: 'https' - ) + @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new end end end diff --git a/lib/gitlab/tracking/destinations/base.rb b/lib/gitlab/tracking/destinations/base.rb new file mode 100644 index 00000000000..00e92e0bd57 --- /dev/null +++ b/lib/gitlab/tracking/destinations/base.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Destinations + class Base + def event(category, action, label: nil, property: nil, value: nil, context: nil) + raise NotImplementedError, "#{self} does not implement #{__method__}" + end + end + end + end +end diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb new file mode 100644 index 00000000000..9cebcfe5ee1 --- /dev/null +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'snowplow-tracker' + +module Gitlab + module Tracking + module Destinations + class Snowplow < Base + extend ::Gitlab::Utils::Override + + override :event + def event(category, action, label: nil, property: nil, value: nil, context: nil) + return unless enabled? + + tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + end + + def self_describing_event(schema_url, event_data_json, context: nil) + return unless enabled? + + event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, event_data_json) + tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) + end + + private + + def enabled? + Gitlab::CurrentSettings.snowplow_enabled? + end + + def tracker + @tracker ||= SnowplowTracker::Tracker.new( + emitter, + SnowplowTracker::Subject.new, + Gitlab::Tracking::SNOWPLOW_NAMESPACE, + Gitlab::CurrentSettings.snowplow_app_id + ) + end + + def emitter + SnowplowTracker::AsyncEmitter.new( + Gitlab::CurrentSettings.snowplow_collector_hostname, + protocol: 'https' + ) + end + end + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 9213b5ebab2..eece2c343d2 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -49,7 +49,7 @@ module Gitlab return [uri, nil] unless address_info ip_address = ip_address(address_info) - return [uri, nil] if domain_whitelisted?(uri) || ip_whitelisted?(ip_address, port: get_port(uri)) + return [uri, nil] if domain_allowed?(uri) || ip_allowed?(ip_address, port: get_port(uri)) protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection) @@ -65,8 +65,8 @@ module Gitlab protected_uri_with_hostname end - def blocked_url?(*args) - validate!(*args) + def blocked_url?(url, **kwargs) + validate!(url, **kwargs) false rescue BlockedUrlError @@ -113,8 +113,8 @@ module Gitlab end rescue SocketError # If the dns rebinding protection is not enabled or the domain - # is whitelisted we avoid the dns rebinding checks - return if domain_whitelisted?(uri) || !dns_rebind_protection + # is allowed we avoid the dns rebinding checks + return if domain_allowed?(uri) || !dns_rebind_protection # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories @@ -253,12 +253,12 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end - def domain_whitelisted?(uri) - Gitlab::UrlBlockers::UrlWhitelist.domain_whitelisted?(uri.normalized_host, port: get_port(uri)) + def domain_allowed?(uri) + Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri)) end - def ip_whitelisted?(ip_address, port: nil) - Gitlab::UrlBlockers::UrlWhitelist.ip_whitelisted?(ip_address, port: port) + def ip_allowed?(ip_address, port: nil) + Gitlab::UrlBlockers::UrlAllowlist.ip_allowed?(ip_address, port: port) end def config diff --git a/lib/gitlab/url_blockers/domain_whitelist_entry.rb b/lib/gitlab/url_blockers/domain_allowlist_entry.rb index b94e8ee3f69..b65bd9e1a92 100644 --- a/lib/gitlab/url_blockers/domain_whitelist_entry.rb +++ b/lib/gitlab/url_blockers/domain_allowlist_entry.rb @@ -2,7 +2,7 @@ module Gitlab module UrlBlockers - class DomainWhitelistEntry + class DomainAllowlistEntry attr_reader :domain, :port def initialize(domain, port: nil) diff --git a/lib/gitlab/url_blockers/ip_whitelist_entry.rb b/lib/gitlab/url_blockers/ip_allowlist_entry.rb index 88c76574d3d..b293afe166c 100644 --- a/lib/gitlab/url_blockers/ip_whitelist_entry.rb +++ b/lib/gitlab/url_blockers/ip_allowlist_entry.rb @@ -2,7 +2,7 @@ module Gitlab module UrlBlockers - class IpWhitelistEntry + class IpAllowlistEntry attr_reader :ip, :port # Argument ip should be an IPAddr object diff --git a/lib/gitlab/url_blockers/url_whitelist.rb b/lib/gitlab/url_blockers/url_allowlist.rb index 59f74dde7fc..60238bea75a 100644 --- a/lib/gitlab/url_blockers/url_whitelist.rb +++ b/lib/gitlab/url_blockers/url_allowlist.rb @@ -2,43 +2,41 @@ module Gitlab module UrlBlockers - class UrlWhitelist + class UrlAllowlist class << self - def ip_whitelisted?(ip_string, port: nil) + def ip_allowed?(ip_string, port: nil) return false if ip_string.blank? - ip_whitelist, _ = outbound_local_requests_whitelist_arrays + ip_allowlist, _ = outbound_local_requests_allowlist_arrays ip_obj = Gitlab::Utils.string_to_ip_object(ip_string) - ip_whitelist.any? do |ip_whitelist_entry| - ip_whitelist_entry.match?(ip_obj, port) + ip_allowlist.any? do |ip_allowlist_entry| + ip_allowlist_entry.match?(ip_obj, port) end end - def domain_whitelisted?(domain_string, port: nil) + def domain_allowed?(domain_string, port: nil) return false if domain_string.blank? - _, domain_whitelist = outbound_local_requests_whitelist_arrays + _, domain_allowlist = outbound_local_requests_allowlist_arrays - domain_whitelist.any? do |domain_whitelist_entry| - domain_whitelist_entry.match?(domain_string, port) + domain_allowlist.any? do |domain_allowlist_entry| + domain_allowlist_entry.match?(domain_string, port) end end private - attr_reader :ip_whitelist, :domain_whitelist - # We cannot use Gitlab::CurrentSettings as ApplicationSetting itself # calls this class. This ends up in a cycle where # Gitlab::CurrentSettings creates an ApplicationSetting which then # calls this method. # # See https://gitlab.com/gitlab-org/gitlab/issues/9833 - def outbound_local_requests_whitelist_arrays + def outbound_local_requests_allowlist_arrays return [[], []] unless ApplicationSetting.current - ApplicationSetting.current.outbound_local_requests_whitelist_arrays + ApplicationSetting.current.outbound_local_requests_allowlist_arrays end end end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 1e522ae63b6..ce59e10241e 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -32,6 +32,8 @@ module Gitlab instance.milestone_url(object, **options) when Note note_url(object, **options) + when Release + instance.release_url(object, **options) when Project instance.project_url(object, **options) when Snippet diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 68f24559b1f..4b0dd54683b 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -40,8 +40,11 @@ module Gitlab with_finished_at(:recording_ce_finished_at) do license_usage_data + .merge(system_usage_data_license) + .merge(system_usage_data_settings) .merge(system_usage_data) .merge(system_usage_data_monthly) + .merge(system_usage_data_weekly) .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) @@ -157,6 +160,8 @@ module Gitlab projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_service_enabled: count(AlertsService.active), + projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), + projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), @@ -212,9 +217,11 @@ module Gitlab # rubocop: enable UsageData/LargeTable: packages: count(::Packages::Package.where(last_28_days_time_period)), personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)), - project_snippets: count(ProjectSnippet.where(last_28_days_time_period)) + project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), + projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) }.merge( - snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) + snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)), + aggregated_metrics_monthly ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -222,6 +229,27 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def system_usage_data_license + { + license: {} + } + end + + def system_usage_data_settings + { + settings: {} + } + end + + def system_usage_data_weekly + { + counts_weekly: { + }.merge( + aggregated_metrics_weekly + ) + } + end + def cycle_analytics_usage_data Gitlab::CycleAnalytics::UsageData.new.to_json rescue ActiveRecord::StatementInvalid @@ -500,6 +528,7 @@ module Gitlab key => { configure: usage_activity_by_stage_configure(time_period), create: usage_activity_by_stage_create(time_period), + enablement: usage_activity_by_stage_enablement(time_period), manage: usage_activity_by_stage_manage(time_period), monitor: usage_activity_by_stage_monitor(time_period), package: usage_activity_by_stage_package(time_period), @@ -555,6 +584,11 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Empty placeholder allows this to match the pattern used by other sections + def usage_activity_by_stage_enablement(time_period) + {} + end + # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_manage(time_period) @@ -564,7 +598,11 @@ module Gitlab users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), + bulk_imports: { + gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id) + }, projects_imported: { + total: count(Project.where(time_period).where.not(import_type: nil)), gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), github: projects_imported_count('github', time_period), @@ -577,7 +615,8 @@ module Gitlab issues_imported: { jira: distinct_count(::JiraImportState.where(time_period), :user_id), fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period) + phabricator: projects_imported_count('phabricator', time_period), + csv: distinct_count(Issues::CsvImport.where(time_period), :user_id) }, groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) } @@ -592,7 +631,10 @@ module Gitlab operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: user_minimum_id, finish: user_maximum_id), - projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id) + projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), + projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), + projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), + projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -664,6 +706,22 @@ module Gitlab { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end + def aggregated_metrics_monthly + return {} unless Feature.enabled?(:product_analytics_aggregated_metrics) + + { + aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_monthly_data + } + end + + def aggregated_metrics_weekly + return {} unless Feature.enabled?(:product_analytics_aggregated_metrics) + + { + aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_weekly_data + } + end + def analytics_unique_visits_data results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash| hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } @@ -709,7 +767,8 @@ module Gitlab data = { action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, action_monthly_active_users_design_management: Gitlab::UsageDataCounters::TrackUniqueEvents::DESIGN_ACTION, - action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION + action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION, + action_monthly_active_users_git_write: Gitlab::UsageDataCounters::TrackUniqueEvents::GIT_WRITE_ACTION } data.each do |key, event| diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml new file mode 100644 index 00000000000..97ec8423b95 --- /dev/null +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -0,0 +1,17 @@ +#- name: unique name of aggregated metric +# operator: aggregation operator. Valid values are: +# - "OR": counts unique elements that were observed triggering any of following events +# - "AND": counts unique elements that were observed triggering all of following events +# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes +# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. +# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. +# Corresponding feature flag should have `default_enabled` attribute set to `false`. +# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. +--- +- name: product_analytics_test_metrics_union + operator: OR + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] + feature_flag: product_analytics_aggregated_metrics +- name: product_analytics_test_metrics_intersection + operator: AND + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb index 22188b555d2..07e1963f9fb 100644 --- a/lib/gitlab/usage_data_counters/designs_counter.rb +++ b/lib/gitlab/usage_data_counters/designs_counter.rb @@ -1,42 +1,8 @@ # frozen_string_literal: true module Gitlab::UsageDataCounters - class DesignsCounter - extend Gitlab::UsageDataCounters::RedisCounter - + class DesignsCounter < BaseCounter KNOWN_EVENTS = %w[create update delete].freeze - - UnknownEvent = Class.new(StandardError) - - class << self - # Each event gets a unique Redis key - def redis_key(event) - raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s) - - "USAGE_DESIGN_MANAGEMENT_DESIGNS_#{event}".upcase - end - - def count(event) - increment(redis_key(event)) - end - - def read(event) - total_count(redis_key(event)) - end - - def totals - KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h - end - - def fallback_totals - KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h - end - - private - - def counter_key(event) - "design_management_designs_#{event}".to_sym - end - end + PREFIX = 'design_management_designs' end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index eb132ef0967..573ad1dce35 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -5,18 +5,28 @@ module Gitlab module HLLRedisCounter DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - DEFAULT_REDIS_SLOT = ''.freeze - - UnknownEvent = Class.new(StandardError) - UnknownAggregation = Class.new(StandardError) - - KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze + DEFAULT_REDIS_SLOT = '' + + EventError = Class.new(StandardError) + UnknownEvent = Class.new(EventError) + UnknownAggregation = Class.new(EventError) + AggregationMismatch = Class.new(EventError) + SlotMismatch = Class.new(EventError) + CategoryMismatch = Class.new(EventError) + UnknownAggregationOperator = Class.new(EventError) + InvalidContext = Class.new(EventError) + + KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # - # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml + # All events should be added to known_events yml files lib/gitlab/usage_data_counters/known_events/ # # Event example: # @@ -25,6 +35,7 @@ module Gitlab # category: compliance # Group events in categories # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly # aggregation: daily # Aggregation level, keys are stored daily or weekly + # feature_flag: # The event feature flag # # Usage: # @@ -33,28 +44,24 @@ module Gitlab class << self include Gitlab::Utils::UsageData - def track_event(entity_id, event_name, time = Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled? - - event = event_for(event_name) - - raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? - - Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event)) + def track_event(value, event_name, time = Time.zone.now) + track(value, event_name, time: time) end - def unique_events(event_names:, start_date:, end_date:) - events = events_for(Array(event_names)) - - raise 'Events should be in same slot' unless events_in_same_slot?(events) - raise 'Events should be in same category' unless events_in_same_category?(events) - raise 'Events should have same aggregation level' unless events_same_aggregation?(events) - - aggregation = events.first[:aggregation] + def track_event_in_context(value, event_name, context, time = Time.zone.now) + return if context.blank? + return unless context.in?(valid_context_list) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) + track(value, event_name, context: context, time: time) + end - redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + def unique_events(event_names:, start_date:, end_date:, context: '') + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise CategoryMismatch, events unless events_in_same_category?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + raise InvalidContext if context.present? && !context.in?(valid_context_list) + end end def categories @@ -72,8 +79,8 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) - hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current) end if eligible_for_totals?(events_names) @@ -89,8 +96,136 @@ module Gitlab event_for(event_name).present? end + def aggregated_metrics_monthly_data + aggregated_metrics_data(4.weeks.ago.to_date) + end + + def aggregated_metrics_weekly_data + aggregated_metrics_data(7.days.ago.to_date) + end + + def known_events + @known_events ||= load_events(KNOWN_EVENTS_PATH) + end + + def aggregated_metrics + @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH) + end + private + def track(value, event_name, context: '', time: Time.zone.now) + return unless Gitlab::CurrentSettings.usage_ping_enabled? + + event = event_for(event_name) + raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: value, expiry: expiry(event)) + end + + # The aray of valid context on which we allow tracking + def valid_context_list + Plan.all_plans + end + + def aggregated_metrics_data(start_date) + aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) + + weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current) + end + end + + def calculate_count_for_aggregation(aggregation, start_date:, end_date:) + case aggregation[:operator] + when UNION_OF_AGGREGATED_METRICS + calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + when INTERSECTION_OF_AGGREGATED_METRICS + calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + else + raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}" + end + end + + # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle + # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 + def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_events = begin + subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ + calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + + def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + subset_sizes = (1..(event_names.size - 1)) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + event_names.combination(subset_size).sum do |events_subset| + subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ + calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + event_names.sum do |event| + subset_powers_cache[subset_size][event] ||= \ + unique_events(event_names: event, start_date: start_date, end_date: end_date) + end + end + end + end + + def calculate_events_union(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + end + end + + def count_unique_events(event_names:, start_date:, end_date:, context: '') + events = events_for(Array(event_names).map(&:to_s)) + + yield events if block_given? + + aggregation = events.first[:aggregation] + + keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + end + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level # and if there are more than 1 event def eligible_for_totals?(events_names) @@ -100,16 +235,22 @@ module Gitlab events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:) + def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date) + daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date) + weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) end end - def known_events - @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access) + def load_events(wildcard) + Dir[wildcard].each_with_object([]) do |path, events| + events.push(*load_yaml_from_path(path)) + end + end + + def load_yaml_from_path(path) + YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) end def known_events_names @@ -141,7 +282,7 @@ module Gitlab end def event_for(event_name) - known_events.find { |event| event[:name] == event_name } + known_events.find { |event| event[:name] == event_name.to_s } end def events_for(event_names) @@ -153,17 +294,26 @@ module Gitlab end # Compose the key in order to store events daily or weekly - def redis_key(event, time) + def redis_key(event, time, context = '') raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + key = apply_slot(event) + key = apply_time_aggregation(key, time, event) + key = "#{context}_#{key}" if context.present? + key + end + + def apply_slot(event) slot = redis_slot(event) - key = if slot.present? - event[:name].to_s.gsub(slot, "{#{slot}}") - else - "{#{event[:name]}}" - end + if slot.present? + event[:name].to_s.gsub(slot, "{#{slot}}") + else + "{#{event[:name]}}" + end + end + def apply_time_aggregation(key, time, event) if event[:aggregation].to_sym == :daily year_day = time.strftime('%G-%j') "#{year_day}-#{key}" @@ -173,21 +323,29 @@ module Gitlab end end - def daily_redis_keys(events:, start_date:, end_date:) + def daily_redis_keys(events:, start_date:, end_date:, context: '') (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date) } + events.map { |event| redis_key(event, date, context) } end.flatten end - def weekly_redis_keys(events:, start_date:, end_date:) + def validate_aggregation_operator!(operator) + return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator) + + raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") + end + + def weekly_redis_keys(events:, start_date:, end_date:, context: '') weeks = end_date.to_date.cweek - start_date.to_date.cweek weeks = 1 if weeks == 0 (0..(weeks - 1)).map do |week_increment| - events.map { |event| redis_key(event, start_date + week_increment * 7.days) } + events.map { |event| redis_key(event, start_date + week_increment * 7.days, context) } end.flatten end end end end end + +Gitlab::UsageDataCounters::HLLRedisCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::HLLRedisCounter') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index e8839875109..da013a06777 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -9,14 +9,12 @@ module Gitlab ISSUE_CREATED = 'g_project_management_issue_created' ISSUE_CLOSED = 'g_project_management_issue_closed' ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' - ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed' ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed' ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed' ISSUE_REOPENED = 'g_project_management_issue_reopened' ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' - ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed' ISSUE_CROSS_REFERENCED = 'g_project_management_issue_cross_referenced' ISSUE_MOVED = 'g_project_management_issue_moved' ISSUE_RELATED = 'g_project_management_issue_related' @@ -24,15 +22,15 @@ module Gitlab ISSUE_MARKED_AS_DUPLICATE = 'g_project_management_issue_marked_as_duplicate' ISSUE_LOCKED = 'g_project_management_issue_locked' ISSUE_UNLOCKED = 'g_project_management_issue_unlocked' - ISSUE_ADDED_TO_EPIC = 'g_project_management_issue_added_to_epic' - ISSUE_REMOVED_FROM_EPIC = 'g_project_management_issue_removed_from_epic' - ISSUE_CHANGED_EPIC = 'g_project_management_issue_changed_epic' ISSUE_DESIGNS_ADDED = 'g_project_management_issue_designs_added' ISSUE_DESIGNS_MODIFIED = 'g_project_management_issue_designs_modified' ISSUE_DESIGNS_REMOVED = 'g_project_management_issue_designs_removed' ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed' ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed' ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed' + ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added' + ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited' + ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self def track_issue_created_action(author:, time: Time.zone.now) @@ -75,14 +73,6 @@ module Gitlab track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) end - def track_issue_iteration_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ITERATION_CHANGED, author, time) - end - - def track_issue_weight_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_WEIGHT_CHANGED, author, time) - end - def track_issue_cross_referenced_action(author:, time: Time.zone.now) track_unique_action(ISSUE_CROSS_REFERENCED, author, time) end @@ -111,18 +101,6 @@ module Gitlab track_unique_action(ISSUE_UNLOCKED, author, time) end - def track_issue_added_to_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ADDED_TO_EPIC, author, time) - end - - def track_issue_removed_from_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_REMOVED_FROM_EPIC, author, time) - end - - def track_issue_changed_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CHANGED_EPIC, author, time) - end - def track_issue_designs_added_action(author:, time: Time.zone.now) track_unique_action(ISSUE_DESIGNS_ADDED, author, time) end @@ -147,6 +125,18 @@ module Gitlab track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) end + def track_issue_comment_added_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_ADDED, author, time) + end + + def track_issue_comment_edited_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_EDITED, author, time) + end + + def track_issue_comment_removed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_REMOVED, author, time) + end + private def track_unique_action(action, author, time) @@ -159,3 +149,5 @@ module Gitlab end end end + +Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::IssueActivityUniqueCounter') diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index bc56c5d6d9b..85f16ea807b 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -4,114 +4,141 @@ redis_slot: compliance category: compliance aggregation: weekly + feature_flag: track_unique_visits - name: g_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: i_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: i_compliance_credential_inventory category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: a_compliance_audit_events_api category: compliance redis_slot: compliance aggregation: weekly + feature_flag: usage_data_a_compliance_audit_events_api # Analytics category - name: g_analytics_contribution category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_insights category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_issues category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_productivity category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_pipelines category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_code_reviews category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_insights category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_issues category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_repo category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_cohorts category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_dev_ops_score category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_instance_statistics category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_edit_by_web_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: g_edit_by_sfe category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: i_search_total category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: i_search_advanced category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: i_search_paid category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: wiki_action category: source_code aggregation: daily @@ -121,6 +148,9 @@ - name: project_action category: source_code aggregation: daily +- name: git_write_action + category: source_code + aggregation: daily - name: merge_request_action category: source_code aggregation: daily @@ -133,173 +163,242 @@ redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_status_changed - name: incident_management_alert_assigned redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_assigned - name: incident_management_alert_todo redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_todo - name: incident_management_incident_created redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_created - name: incident_management_incident_reopened redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_reopened - name: incident_management_incident_closed redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_closed - name: incident_management_incident_assigned redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_assigned - name: incident_management_incident_todo redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_todo - name: incident_management_incident_comment redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_comment - name: incident_management_incident_zoom_meeting redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_zoom_meeting - name: incident_management_incident_published redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_published - name: incident_management_incident_relate redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_relate - name: incident_management_incident_unrelate redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_unrelate - name: incident_management_incident_change_confidential redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_change_confidential # Testing category - name: i_testing_test_case_parsed category: testing redis_slot: testing aggregation: weekly + feature_flag: usage_data_i_testing_test_case_parsed # Project Management group - name: g_project_management_issue_title_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_description_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_assignee_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_confidential category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_visible category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_created category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_closed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_reopened category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_label_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_milestone_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_iteration_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_weight_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_moved category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_related category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_unrelated category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_marked_as_duplicate category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_locked category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_unlocked category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_added_to_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_removed_from_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_changed_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_modified category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_removed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_due_date_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_spent_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_added + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_edited + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_removed + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_health_status_changed + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +# Secrets Management +- name: i_ci_secrets_management_vault_build_created + category: ci_secrets_management + redis_slot: ci_secrets_management + aggregation: weekly + feature_flag: usage_data_i_ci_secrets_management_vault_build_created diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml new file mode 100644 index 00000000000..7ed02aa2a85 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -0,0 +1,265 @@ +--- +- name: i_package_maven_user_push + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_push + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_user_delete + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_delete + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_user_pull + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_pull + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_push + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_push + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_delete + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_delete + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_pull + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_pull + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_push + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_push + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_delete + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_delete + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_pull + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_pull + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_push + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_push + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_delete + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_delete + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_pull + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_pull + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_push + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_push + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_delete + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_delete + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_pull + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_pull + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_push + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_push + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_delete + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_delete + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_pull + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_pull + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_push + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_push + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_delete + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_delete + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_pull + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_pull + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_push + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_push + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_delete + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_delete + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_pull + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_pull + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_push + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_push + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_delete + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_delete + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_pull + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_pull + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_push + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_push + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_delete + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_delete + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_pull + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_pull + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_push + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_push + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_delete + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_delete + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_pull + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_pull + category: tag_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb index 8886a106da8..3c5989d1e11 100644 --- a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb +++ b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb @@ -3,7 +3,7 @@ module Gitlab module UsageDataCounters class StaticSiteEditorCounter < BaseCounter - KNOWN_EVENTS = %w[views].freeze + KNOWN_EVENTS = %w[views commits merge_requests].freeze PREFIX = 'static_site_editor' class << self diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 7053744b665..95380ae0b1d 100644 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -8,6 +8,9 @@ module Gitlab PUSH_ACTION = :project_action MERGE_REQUEST_ACTION = :merge_request_action + GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze + GIT_WRITE_ACTION = :git_write_action + ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ wiki: { created: WIKI_ACTION, @@ -41,6 +44,8 @@ module Gitlab return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + + track_git_write_action(author_id, transformed_action, time) end def count_unique_events(event_action:, date_from:, date_to:) @@ -64,6 +69,12 @@ module Gitlab def valid_action?(action) Event.actions.key?(action) end + + def track_git_write_action(author_id, transformed_action, time) + return unless GIT_WRITE_ACTIONS.include?(transformed_action) + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, GIT_WRITE_ACTION, time) + end end end end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index 00fcd42a9af..9f2f4ac3971 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -2,54 +2,43 @@ module Gitlab module UsageDataCounters - class WebIdeCounter - extend RedisCounter - KNOWN_EVENTS = %i[commits views merge_requests previews terminals pipelines].freeze + class WebIdeCounter < BaseCounter + KNOWN_EVENTS = %w[commits views merge_requests previews terminals pipelines].freeze PREFIX = 'web_ide' class << self def increment_commits_count - increment(redis_key('commits')) + count('commits') end def increment_merge_requests_count - increment(redis_key('merge_requests')) + count('merge_requests') end def increment_views_count - increment(redis_key('views')) + count('views') end def increment_terminals_count - increment(redis_key('terminals')) + count('terminals') end def increment_pipelines_count - increment(redis_key('pipelines')) + count('pipelines') end def increment_previews_count return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - increment(redis_key('previews')) - end - - def totals - KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h - end - - def fallback_totals - KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h + count('previews') end private def redis_key(event) - "#{PREFIX}_#{event}_count".upcase - end + require_known_event(event) - def counter_key(event) - "#{PREFIX}_#{event}".to_sym + "#{prefix}_#{event}_count".upcase end end end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 1c6ddc2e70f..eec89e1ab72 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -90,9 +90,13 @@ module Gitlab def can_collaborate?(ref) assert_project! + can_push? || branch_allows_collaboration_for?(ref) + end + + def branch_allows_collaboration_for?(ref) # Checking for an internal project or group to prevent an infinite loop: # https://gitlab.com/gitlab-org/gitlab/issues/36805 - can_push? || (!project.internal? && project.branch_allows_collaboration?(user, ref)) + (!project.internal? && project.branch_allows_collaboration?(user, ref)) end def permission_cache diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb index 069e68e8d29..88f2a4455c6 100644 --- a/lib/gitlab/webpack/dev_server_middleware.rb +++ b/lib/gitlab/webpack/dev_server_middleware.rb @@ -16,14 +16,14 @@ module Gitlab super(app, backend: "#{@proxy_scheme}://#{@proxy_host}:#{@proxy_port}", **opts) end - # disable SSL check since any cert used here will likely be self-signed - def rewrite_env(env) - env["rack.ssl_verify_none"] = true - env - end - def perform_request(env) if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + # disable SSL check since any cert used here will likely be self-signed + env['rack.ssl_verify_none'] = true + + # ensure we pass the expected Host header so webpack-dev-server doesn't complain + env['HTTP_HOST'] = "#{@proxy_host}:#{@proxy_port}" + if relative_url_root = Rails.application.config.relative_url_root env['SCRIPT_NAME'] = "" env['REQUEST_PATH'].sub!(/\A#{Regexp.escape(relative_url_root)}/, '') diff --git a/lib/gitlab/whats_new.rb b/lib/gitlab/whats_new.rb index e95ace2c475..69ccb48c544 100644 --- a/lib/gitlab/whats_new.rb +++ b/lib/gitlab/whats_new.rb @@ -2,27 +2,39 @@ module Gitlab module WhatsNew - CACHE_DURATION = 1.day + CACHE_DURATION = 1.hour WHATS_NEW_FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml') private - def whats_new_most_recent_release_items - Rails.cache.fetch('whats_new:release_items', expires_in: CACHE_DURATION) do - file = File.read(most_recent_release_file_path) + def whats_new_release_items(page: 1) + Rails.cache.fetch(whats_new_items_cache_key(page), expires_in: CACHE_DURATION) do + index = page - 1 + file_path = whats_new_file_paths[index] + + next if file_path.nil? + + file = File.read(file_path) items = YAML.safe_load(file, permitted_classes: [Date]) items if items.is_a?(Array) end rescue => e - Gitlab::ErrorTracking.track_exception(e, yaml_file_path: most_recent_release_file_path) + Gitlab::ErrorTracking.track_exception(e, page: page) nil end - def most_recent_release_file_path - @most_recent_release_file_path ||= Dir.glob(WHATS_NEW_FILES_PATH).max + def whats_new_file_paths + @whats_new_file_paths ||= Rails.cache.fetch('whats_new:file_paths', expires_in: CACHE_DURATION) do + Dir.glob(WHATS_NEW_FILES_PATH).sort.reverse + end + end + + def whats_new_items_cache_key(page) + filename = /\d*\_\d*\_\d*/.match(whats_new_file_paths&.first) + "whats_new:release_items:file-#{filename}:page-#{page}" end end end diff --git a/lib/gitlab/with_feature_category.rb b/lib/gitlab/with_feature_category.rb new file mode 100644 index 00000000000..65d21daf78a --- /dev/null +++ b/lib/gitlab/with_feature_category.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module WithFeatureCategory + extend ActiveSupport::Concern + include Gitlab::ClassAttributes + + class_methods do + def feature_category(category, actions = []) + feature_category_configuration[category] ||= [] + feature_category_configuration[category] += actions.map(&:to_s) + + validate_config!(feature_category_configuration) + end + + def feature_category_for_action(action) + category_config = feature_category_configuration.find do |_, actions| + actions.empty? || actions.include?(action) + end + + category_config&.first || superclass_feature_category_for_action(action) + end + + private + + def validate_config!(config) + empty = config.find { |_, actions| actions.empty? } + duplicate_actions = config.values.map(&:uniq).flatten.group_by(&:itself).select { |_, v| v.count > 1 }.keys + + if config.length > 1 && empty + raise ArgumentError, "#{empty.first} is defined for all actions, but other categories are set" + end + + if duplicate_actions.any? + raise ArgumentError, "Actions have multiple feature categories: #{duplicate_actions.join(', ')}" + end + end + + def feature_category_configuration + class_attributes[:feature_category_config] ||= {} + end + + def superclass_feature_category_for_action(action) + return unless superclass.respond_to?(:feature_category_for_action) + + superclass.feature_category_for_action(action) + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index eb780a2f7f6..8e7af8876a4 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -270,7 +270,7 @@ module Gitlab prefix: metadata['ArchivePrefix'], format: format, path: path.presence || "", - include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive) + include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive, default_enabled: true) ).to_proto ) } diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index cd94efddc1e..b239b6812ca 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -12,6 +12,9 @@ module Quality lib/gitlab/background_migration lib/ee/gitlab/background_migration ], + frontend_fixture: %w[ + frontend/fixtures + ], unit: %w[ bin channels @@ -63,7 +66,7 @@ module Quality end def pattern(level) - @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*_spec.rb" + @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}" end def regexp(level) @@ -76,6 +79,9 @@ module Quality # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default when regexp(:migration), regexp(:background_migration) :migration + # Detect frontend fixture before matching other unit tests + when regexp(:frontend_fixture) + :frontend_fixture when regexp(:unit) :unit when regexp(:integration) @@ -93,6 +99,15 @@ module Quality private + def suffix(level) + case level + when :frontend_fixture + ".rb" + else + "_spec.rb" + end + end + def migration_and_background_migration_folders TEST_LEVEL_FOLDERS.fetch(:migration) + TEST_LEVEL_FOLDERS.fetch(:background_migration) end diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index e2a7d3ef5ba..e7e0d4e471f 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -20,7 +20,7 @@ module Rouge is_first = false yield %(<span id="LC#{@line_number}" class="line" lang="#{@tag}">) - line.each { |token, value| yield span(token, value.chomp) } + line.each { |token, value| yield span(token, value.chomp! || value) } yield %(</span>) @line_number += 1 diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb index b86ec82bde6..acbfb411873 100644 --- a/lib/rspec_flaky/flaky_examples_collection.rb +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'active_support/hash_with_indifferent_access' +require 'delegate' require_relative 'flaky_example' diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index fc984d737d5..9eafa5ef008 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -18,7 +18,7 @@ upstream gitlab-workhorse { # GitLab socket file, - # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket + # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/sockets/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index ba01e250bbb..ae5c88455e4 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -22,7 +22,7 @@ upstream gitlab-workhorse { # GitLab socket file, - # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket + # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/sockets/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 0709f783f09..0f4fbe4fba5 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.parse('2.24.0') + @required_version ||= Gitlab::VersionInfo.parse('2.29.0') end def self.current_version diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 74cf3aad951..49d2d3d62a2 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -22,7 +22,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") if Rails.env.test? command.push( 'BUNDLE_FLAGS=--no-deployment', - "BUNDLE_PATH=#{Bundler.bundle_path}") + "GEM_HOME=#{Bundler.bundle_path}") end storage_paths = { 'default' => args.storage_path } diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake new file mode 100644 index 00000000000..3484b9b6072 --- /dev/null +++ b/lib/tasks/gitlab/packages/events.rake @@ -0,0 +1,40 @@ +require 'logger' + +desc "GitLab | Packages | Events | Generate hll counter events file for packages" +namespace :gitlab do + namespace :packages do + namespace :events do + task generate: :environment do + logger = Logger.new(STDOUT) + logger.info('Building list of package events...') + + path = File.join(File.dirname(::Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml') + + File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml } + + logger.info("Events file `#{path}` generated successfully") + rescue => e + logger.error("Error building events list: #{e}") + end + + def event_pairs + ::Packages::Event.event_types.keys.product(::Packages::Event.originator_types.keys) + end + + def generate_unique_events_list + ::Packages::Event::EVENT_SCOPES.keys.each_with_object([]) do |event_scope, events| + event_pairs.each do |event_type, originator| + if name = ::Packages::Event.allowed_event_name(event_scope, event_type, originator) + events << { + "name" => name, + "category" => "#{event_scope}_packages", + "aggregation" => "weekly", + "redis_slot" => "package" + } + end + end + end + end + end + end +end |