diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-20 23:50:22 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-20 23:50:22 +0000 |
| commit | 9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch) | |
| tree | 70467ae3692a0e35e5ea56bcb803eb512a10bedb /lib | |
| parent | 4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff) | |
| download | gitlab-ce-9dc93a4519d9d5d7be48ff274127136236a3adb3.tar.gz | |
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'lib')
456 files changed, 7496 insertions, 4352 deletions
diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb index 7e561783685..d91d4a0d4d5 100644 --- a/lib/api/admin/sidekiq.rb +++ b/lib/api/admin/sidekiq.rb @@ -12,11 +12,11 @@ module API namespace 'queues' do desc 'Drop jobs matching the given metadata from the Sidekiq queue' params do - Labkit::Context::KNOWN_KEYS.each do |key| + Gitlab::ApplicationContext::KNOWN_KEYS.each do |key| optional key, type: String, allow_blank: false end - at_least_one_of(*Labkit::Context::KNOWN_KEYS) + at_least_one_of(*Gitlab::ApplicationContext::KNOWN_KEYS) end delete ':queue_name' do result = diff --git a/lib/api/api.rb b/lib/api/api.rb index f83a36068dd..a287ffbfcd8 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -59,7 +59,7 @@ module API project: -> { @project }, namespace: -> { @group }, runner: -> { @current_runner || @runner }, - caller_id: route.origin, + caller_id: api_endpoint.endpoint_id, remote_ip: request.ip, feature_category: feature_category ) @@ -293,6 +293,8 @@ module API mount ::API::Triggers mount ::API::Unleash mount ::API::UsageData + mount ::API::UsageDataQueries + mount ::API::UsageDataNonSqlMetrics mount ::API::UserCounts mount ::API::Users mount ::API::Variables diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8641271f2df..8822a30d4a1 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -55,7 +55,7 @@ module API user = find_user_from_sources return unless user - if user.is_a?(User) && Feature.enabled?(:user_mode_in_session) + if user.is_a?(User) && Gitlab::CurrentSettings.admin_mode # Sessions are enforced to be unavailable for API calls, so ignore them for admin mode Gitlab::Auth::CurrentUserMode.bypass_session!(user.id) end @@ -236,7 +236,7 @@ module API def after # Use a Grape middleware since the Grape `after` blocks might run # before we are finished rendering the `Grape::Entity` classes - Gitlab::Auth::CurrentUserMode.reset_bypass_session! if Feature.enabled?(:user_mode_in_session) + Gitlab::Auth::CurrentUserMode.reset_bypass_session! if Gitlab::CurrentSettings.admin_mode # Explicit nil is needed or the api call return value will be overwritten nil diff --git a/lib/api/applications.rb b/lib/api/applications.rb index b883f83cc19..be482272b20 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -41,6 +41,8 @@ module API desc 'Delete an application' delete ':id' do application = ApplicationsFinder.new(params).execute + break not_found!('Application') unless application + application.destroy no_content! diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index fa75d012613..339c0e779f9 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -70,7 +70,7 @@ module API optional :variables, Array, desc: 'Array of variables available in the pipeline' end post ':id/pipeline' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42124') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711') authorize! :create_pipeline, user_project diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 80d5e80e21e..c5249f1377b 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -245,7 +245,7 @@ module API job = authenticate_job! - result = ::Ci::CreateJobArtifactsService.new(job).authorize(artifact_type: params[:artifact_type], filesize: params[:filesize]) + result = ::Ci::JobArtifacts::CreateService.new(job).authorize(artifact_type: params[:artifact_type], filesize: params[:filesize]) if result[:status] == :success content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE @@ -284,7 +284,7 @@ module API artifacts = params[:file] metadata = params[:metadata] - result = ::Ci::CreateJobArtifactsService.new(job).execute(artifacts, params, metadata_file: metadata) + result = ::Ci::JobArtifacts::CreateService.new(job).execute(artifacts, params, metadata_file: metadata) if result[:status] == :success status :created diff --git a/lib/api/commits.rb b/lib/api/commits.rb index a24848082a9..bd9f83ac24c 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -186,16 +186,14 @@ module API use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' end - # rubocop: disable CodeReuse/ActiveRecord get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = commit.notes.order(:created_at) + notes = commit.notes.with_api_entity_associations.fresh present paginate(notes), with: Entities::CommitNote end - # rubocop: enable CodeReuse/ActiveRecord desc 'Cherry pick commit into a branch' do detail 'This feature was introduced in GitLab 8.15' @@ -372,7 +370,7 @@ module API current_user, project_id: user_project.id, commit_sha: commit.sha - ).execute + ).execute.with_api_entity_associations present paginate(commit_merge_requests), with: Entities::MergeRequestBasic end diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index bd8d9b68858..115a6b8ac4f 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -161,6 +161,8 @@ module API not_found! unless metadata + track_package_event('pull_package', :composer) + send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true end end diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index 53b778875fc..5364eeb1880 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -95,7 +95,7 @@ module API # https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource params do - requires :q, type: String, desc: 'The search term' + optional :q, type: String, desc: 'The search term' optional :skip, type: Integer, desc: 'The number of results to skip', default: 0, regexp: NON_NEGATIVE_INTEGER_REGEX optional :take, type: Integer, desc: 'The number of results to return', default: Kaminari.config.default_per_page, regexp: POSITIVE_INTEGER_REGEX optional :prerelease, type: ::Grape::API::Boolean, desc: 'Include prerelease versions', default: true diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 0a541620c3a..9f0f569b711 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -44,7 +44,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ":id/deploy_keys" do - keys = user_project.deploy_keys_projects.preload(:deploy_key) + keys = user_project.deploy_keys_projects.preload(deploy_key: :user) present paginate(keys), with: Entities::DeployKeysProject end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index d0c842bb19d..0a6ecf2919c 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -36,7 +36,9 @@ module API get ':id/deployments' do authorize! :read_deployment, user_project - deployments = DeploymentsFinder.new(params.merge(project: user_project)).execute + deployments = + DeploymentsFinder.new(params.merge(project: user_project)) + .execute.with_api_entity_associations present paginate(deployments), with: Entities::Deployment end diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index cf0b32bed26..2de49d6ed40 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -8,11 +8,10 @@ module API expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) } # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770 expose :tag_list do |project| - # project.tags.order(:name).pluck(:name) is the most suitable option - # to avoid loading all the ActiveRecord objects but, if we use it here - # it override the preloaded associations and makes a query - # (fixed in https://github.com/rails/rails/pull/25976). - project.tags.map(&:name).sort + # Tags is a preloaded association. If we perform then sorting + # through the database, it will trigger a new query, ending up + # in an N+1 if we have several projects + project.tags.pluck(:name).sort # rubocop:disable CodeReuse/ActiveRecord end expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url diff --git a/lib/api/entities/clusters/agent.rb b/lib/api/entities/clusters/agent.rb new file mode 100644 index 00000000000..3b4538b81c2 --- /dev/null +++ b/lib/api/entities/clusters/agent.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + module Clusters + class Agent < Grape::Entity + expose :id + expose :project, with: Entities::ProjectIdentity, as: :config_project + end + end + end +end diff --git a/lib/api/entities/email.rb b/lib/api/entities/email.rb index 5ba425def3d..46ebc458bcd 100644 --- a/lib/api/entities/email.rb +++ b/lib/api/entities/email.rb @@ -3,7 +3,7 @@ module API module Entities class Email < Grape::Entity - expose :id, :email + expose :id, :email, :confirmed_at end end end diff --git a/lib/api/entities/job_request/job_info.rb b/lib/api/entities/job_request/job_info.rb index 09c13aa8471..a4bcc9726d0 100644 --- a/lib/api/entities/job_request/job_info.rb +++ b/lib/api/entities/job_request/job_info.rb @@ -4,7 +4,7 @@ module API module Entities module JobRequest class JobInfo < Grape::Entity - expose :name, :stage + expose :id, :name, :stage expose :project_id, :project_name end end diff --git a/lib/api/entities/namespace_existence.rb b/lib/api/entities/namespace_existence.rb new file mode 100644 index 00000000000..d93078ecdac --- /dev/null +++ b/lib/api/entities/namespace_existence.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class NamespaceExistence < Grape::Entity + expose :exists, :suggests + end + end +end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index e332e5e40fa..690bc5d419d 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -127,15 +127,16 @@ module API # as `:tags` are defined as: `has_many :tags, through: :taggings` # N+1 is solved then by using `subject.tags.map(&:name)` # MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555 - super(projects_relation).preload(:group) + super(projects_relation).preload(group: :namespace_settings) .preload(:ci_cd_settings) .preload(:project_setting) .preload(:container_expiration_policy) .preload(:auto_devops) + .preload(:service_desk_setting) .preload(project_group_links: { group: :route }, fork_network: :root_project, fork_network_member: :forked_from_project, - forked_from_project: [:route, :forks, :tags, namespace: :route]) + forked_from_project: [:route, :forks, :tags, :group, :project_feature, namespace: [:route, :owner]]) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/entities/project_import_failed_relation.rb b/lib/api/entities/project_import_failed_relation.rb index 16b26ad0efa..b8f842c1646 100644 --- a/lib/api/entities/project_import_failed_relation.rb +++ b/lib/api/entities/project_import_failed_relation.rb @@ -3,7 +3,11 @@ module API module Entities class ProjectImportFailedRelation < Grape::Entity - expose :id, :created_at, :exception_class, :exception_message, :source + expose :id, :created_at, :exception_class, :source + + expose :exception_message do |_| + nil + end expose :relation_key, as: :relation_name end diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 248a86751d2..3ce6d03e236 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -11,10 +11,10 @@ module API work_information(user) end expose :followers, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user| - user.followers.count + user.followers.size end expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user| - user.followees.count + user.followees.size end end end diff --git a/lib/api/entities/user_preferences.rb b/lib/api/entities/user_preferences.rb new file mode 100644 index 00000000000..7a6df9b6c59 --- /dev/null +++ b/lib/api/entities/user_preferences.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class UserPreferences < Grape::Entity + expose :id, :user_id, :view_diffs_file_by_file + end + end +end diff --git a/lib/api/entities/user_public.rb b/lib/api/entities/user_public.rb index 15e9b905bef..685adb1dd10 100644 --- a/lib/api/entities/user_public.rb +++ b/lib/api/entities/user_public.rb @@ -14,6 +14,7 @@ module API expose :two_factor_enabled?, as: :two_factor_enabled expose :external expose :private_profile + expose :commit_email end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 3e1e430c2f9..b606b2e814d 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -26,7 +26,7 @@ module API get ':id/environments' do authorize! :read_environment, user_project - environments = ::EnvironmentsFinder.new(user_project, current_user, params).find + environments = ::EnvironmentsFinder.new(user_project, current_user, params).execute present paginate(environments), with: Entities::Environment, current_user: current_user end diff --git a/lib/api/files.rb b/lib/api/files.rb index cb73bde73f5..f3de7fbe96b 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -113,7 +113,7 @@ module API desc 'Get raw file metadata from repository' params do requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false + optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -124,7 +124,7 @@ module API desc 'Get raw file contents from the repository' params do requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false + optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 3d0ba97b51a..cce55fa92d9 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -62,7 +62,7 @@ module API authorize_upload!(project) bad_request!('File is too large') if max_file_size_exceeded? - track_event('push_package') + ::Gitlab::Tracking.event(self.options[:for].name, 'push_package') create_package_file_params = declared_params.merge(build: current_authenticated_job) ::Packages::Generic::CreatePackageFileService @@ -94,7 +94,7 @@ module API package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version]) package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! - track_event('pull_package') + ::Gitlab::Tracking.event(self.options[:for].name, 'pull_package') present_carrierwave_file!(package_file.file) end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 09744fbeda2..8d52a0a5b4e 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -8,6 +8,8 @@ module API before { authorize! :admin_group, user_group } feature_category :continuous_integration + helpers Helpers::VariablesHelpers + params do requires :id, type: String, desc: 'The ID of a group' end @@ -30,16 +32,13 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end - # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key' do - key = params[:key] - variable = user_group.variables.find_by(key: key) + variable = find_variable(user_group, params) break not_found!('GroupVariable') unless variable present variable, with: Entities::Ci::Variable end - # rubocop: enable CodeReuse/ActiveRecord desc 'Create a new variable in a group' do success Entities::Ci::Variable @@ -50,12 +49,19 @@ module API optional :protected, type: String, desc: 'Whether the variable is protected' optional :masked, type: String, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' + + use :optional_group_variable_params_ee end post ':id/variables' do + filtered_params = filter_variable_parameters( + user_group, + declared_params(include_missing: false) + ) + variable = ::Ci::ChangeVariableService.new( container: user_group, current_user: current_user, - params: { action: :create, variable_params: declared_params(include_missing: false) } + params: { action: :create, variable_params: filtered_params } ).execute if variable.valid? @@ -74,13 +80,19 @@ module API optional :protected, type: String, desc: 'Whether the variable is protected' optional :masked, type: String, desc: 'Whether the variable is masked' optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' + + use :optional_group_variable_params_ee end - # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do + filtered_params = filter_variable_parameters( + user_group, + declared_params(include_missing: false) + ) + variable = ::Ci::ChangeVariableService.new( container: user_group, current_user: current_user, - params: { action: :update, variable_params: declared_params(include_missing: false) } + params: { action: :update, variable_params: filtered_params } ).execute if variable.valid? @@ -91,7 +103,6 @@ module API rescue ::ActiveRecord::RecordNotFound not_found!('GroupVariable') end - # rubocop: enable CodeReuse/ActiveRecord desc 'Delete an existing variable from a group' do success Entities::Ci::Variable @@ -99,21 +110,18 @@ module API params do requires :key, type: String, desc: 'The key of the variable' end - # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do - variable = user_group.variables.find_by!(key: params[:key]) + variable = find_variable(user_group, params) + break not_found!('GroupVariable') unless variable destroy_conditionally!(variable) do |target_variable| ::Ci::ChangeVariableService.new( container: user_group, current_user: current_user, - params: { action: :destroy, variable_params: declared_params(include_missing: false) } + params: { action: :destroy, variable: variable } ).execute end - rescue ::ActiveRecord::RecordNotFound - not_found!('GroupVariable') end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 26fa00d6186..912813d5bb7 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -52,9 +52,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? order_options = { params[:order_by] => params[:sort] } order_options["id"] ||= "asc" - groups = groups.reorder(order_options) - - groups + groups.reorder(order_options) end # rubocop: enable CodeReuse/ActiveRecord @@ -112,7 +110,6 @@ module API end def delete_group(group) - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/46285') destroy_conditionally!(group) do |group| ::Groups::DestroyService.new(group, current_user).async_execute end @@ -141,6 +138,10 @@ module API def authorize_group_creation! authorize! :create_group end + + def check_subscription!(group) + render_api_error!("This group can't be removed because it is linked to a subscription.", :bad_request) if group.paid? + end end resource :groups do @@ -239,6 +240,7 @@ module API delete ":id" do group = find_group!(params[:id]) authorize! :admin_group, group + check_subscription! group delete_group(group) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9db4a03c5b9..2d8a4f60e2a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -3,6 +3,7 @@ module API module Helpers include Gitlab::Utils + include Helpers::Caching include Helpers::Pagination include Helpers::PaginationStrategies @@ -48,7 +49,11 @@ module API # Returns the job associated with the token provided for # authentication, if any def current_authenticated_job - @current_authenticated_job + if try(:namespace_inheritable, :authentication) + ci_build_from_namespace_inheritable + else + @current_authenticated_job # rubocop:disable Gitlab/ModuleWithInstanceVariables + end end # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -539,17 +544,6 @@ module API end end - def track_event(action = action_name, **args) - category = args.delete(:category) || self.options[:for].name - raise "invalid category" unless category - - ::Gitlab::Tracking.event(category, action.to_s, **args) - rescue => error - Gitlab::AppLogger.warn( - "Tracking event failed for action: #{action}, category: #{category}, message: #{error.message}" - ) - end - def increment_counter(event_name) feature_name = "usage_data_#{event_name}" return unless Feature.enabled?(feature_name) @@ -564,10 +558,6 @@ module API def increment_unique_values(event_name, values) return unless values.present? - feature_flag = "usage_data_#{event_name}" - - return unless Feature.enabled?(feature_flag, default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: values) rescue => error Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb index a6cfe930190..da11f07485b 100644 --- a/lib/api/helpers/authentication.rb +++ b/lib/api/helpers/authentication.rb @@ -52,6 +52,11 @@ module API token&.user end + def ci_build_from_namespace_inheritable + token = token_from_namespace_inheritable + token if token.is_a?(::Ci::Build) + end + private def find_token_from_raw_credentials(token_types, raw) diff --git a/lib/api/helpers/caching.rb b/lib/api/helpers/caching.rb new file mode 100644 index 00000000000..d0f22109879 --- /dev/null +++ b/lib/api/helpers/caching.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +# Grape helpers for caching. +# +# This module helps introduce standardised caching into the Grape API +# in a similar manner to the standard Grape DSL. + +module API + module Helpers + module Caching + # @return [ActiveSupport::Duration] + DEFAULT_EXPIRY = 1.day + + # @return [ActiveSupport::Cache::Store] + def cache + Rails.cache + end + + # This is functionally equivalent to the standard `#present` used in + # Grape endpoints, but the JSON for the object, or for each object of + # a collection, will be cached. + # + # With a collection all the keys will be fetched in a single call and the + # Entity rendered for those missing from the cache, which are then written + # back into it. + # + # Both the single object, and all objects inside a collection, must respond + # to `#cache_key`. + # + # To override the Grape formatter we return a custom wrapper in + # `Gitlab::Json::PrecompiledJson` which tells the `Gitlab::Json::GrapeFormatter` + # to export the string without conversion. + # + # A cache context can be supplied to add more context to the cache key. This + # defaults to including the `current_user` in every key for safety, unless overridden. + # + # @param obj_or_collection [Object, Enumerable<Object>] the object or objects to render + # @param with [Grape::Entity] the entity to use for rendering + # @param cache_context [Proc] a proc to call for each object to provide more context to the cache key + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @return [Gitlab::Json::PrecompiledJson] + def present_cached(obj_or_collection, with:, cache_context: -> (_) { current_user.cache_key }, expires_in: DEFAULT_EXPIRY, **presenter_args) + json = + if obj_or_collection.is_a?(Enumerable) + cached_collection( + obj_or_collection, + presenter: with, + presenter_args: presenter_args, + context: cache_context, + expires_in: expires_in + ) + else + cached_object( + obj_or_collection, + presenter: with, + presenter_args: presenter_args, + context: cache_context, + expires_in: expires_in + ) + end + + body Gitlab::Json::PrecompiledJson.new(json) + end + + private + + # Optionally uses a `Proc` to add context to a cache key + # + # @param object [Object] must respond to #cache_key + # @param context [Proc] a proc that will be called with the object as an argument, and which should return a + # string or array of strings to be combined into the cache key + # @return [String] + def contextual_cache_key(object, context) + return object.cache_key if context.nil? + + [object.cache_key, context.call(object)].flatten.join(":") + end + + # Used for fetching or rendering a single object + # + # @param object [Object] the object to render + # @param presenter [Grape::Entity] + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @param context [Proc] + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [String] + def cached_object(object, presenter:, presenter_args:, context:, expires_in:) + cache.fetch(contextual_cache_key(object, context), expires_in: expires_in) do + Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + end + end + + # Used for fetching or rendering multiple objects + # + # @param objects [Enumerable<Object>] the objects to render + # @param presenter [Grape::Entity] + # @param presenter_args [Hash] keyword arguments to be passed to the entity + # @param context [Proc] + # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry + # @return [Array<String>] + def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:) + json = fetch_multi(collection, context: context, expires_in: expires_in) do |obj| + Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + end + + json.values + end + + # An adapted version of ActiveSupport::Cache::Store#fetch_multi. + # + # The original method only provides the missing key to the block, + # not the missing object, so we have to create a map of cache keys + # to the objects to allow us to pass the object to the missing value + # block. + # + # The result is that this is functionally identical to `#fetch`. + def fetch_multi(*objs, context:, **kwargs) + objs.flatten! + map = multi_key_map(objs, context: context) + + cache.fetch_multi(*map.keys, **kwargs) do |key| + yield map[key] + end + end + + # @param objects [Enumerable<Object>] objects which _must_ respond to `#cache_key` + # @param context [Proc] a proc that can be called to help generate each cache key + # @return [Hash] + def multi_key_map(objects, context:) + objects.index_by do |object| + contextual_cache_key(object, context) + end + end + end + end +end diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb index a44fd4b0a5b..8940cf87f82 100644 --- a/lib/api/helpers/common_helpers.rb +++ b/lib/api/helpers/common_helpers.rb @@ -32,6 +32,10 @@ module API end end.compact.to_set end + + def endpoint_id + "#{request.request_method} #{route.origin}" + end end end end diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb index 3ddef0c16b3..4f7f85bd69d 100644 --- a/lib/api/helpers/graphql_helpers.rb +++ b/lib/api/helpers/graphql_helpers.rb @@ -6,8 +6,8 @@ module API # against the graphql API. Helper code for the graphql server implementation # should be in app/graphql/ or lib/gitlab/graphql/ module GraphqlHelpers - def run_graphql!(query:, context: {}, transform: nil) - result = GitlabSchema.execute(query, context: context) + def run_graphql!(query:, context: {}, variables: nil, transform: nil) + result = GitlabSchema.execute(query, variables: variables, context: context) if transform transform.call(result) diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 71a18524104..cb938bc8a14 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -116,7 +116,7 @@ module API end def create_note(noteable, opts) - whitelist_query_limiting + disable_query_limiting authorize!(:create_note, noteable) parent = noteable_parent(noteable) @@ -144,8 +144,8 @@ module API present discussion, with: Entities::Discussion end - def whitelist_query_limiting - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/-/issues/211538') + def disable_query_limiting + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/211538') end end end diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index d5f5448fd42..b18f52b5be6 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -14,7 +14,8 @@ module API package, current_user, project, - conan_package_reference: params[:conan_package_reference] + conan_package_reference: params[:conan_package_reference], + id: params[:id] ) render_api_error!("No recipe manifest found", 404) if yield(presenter).empty? @@ -31,19 +32,15 @@ module API end def recipe_upload_urls - { upload_urls: Hash[ - file_names.select(&method(:recipe_file?)).map do |file_name| - [file_name, build_recipe_file_upload_url(file_name)] - end - ] } + { upload_urls: file_names.select(&method(:recipe_file?)).to_h do |file_name| + [file_name, build_recipe_file_upload_url(file_name)] + end } end def package_upload_urls - { upload_urls: Hash[ - file_names.select(&method(:package_file?)).map do |file_name| - [file_name, build_package_file_upload_url(file_name)] - end - ] } + { upload_urls: file_names.select(&method(:package_file?)).to_h do |file_name| + [file_name, build_package_file_upload_url(file_name)] + end } end def recipe_file?(file_name) @@ -212,10 +209,8 @@ module API end def find_personal_access_token - personal_access_token = find_personal_access_token_from_conan_jwt || + find_personal_access_token_from_conan_jwt || find_personal_access_token_from_http_basic_auth - - personal_access_token end def find_user_from_job_token diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index 577ba97d68a..989c4e1761b 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -10,7 +10,7 @@ module API def redirect_registry_request(forward_to_registry, package_type, options) if forward_to_registry && redirect_registry_request_available? - track_event("#{package_type}_request_forward") + ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward") redirect(registry_url(package_type, options)) else yield diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb index e1898d28ef7..2221eec0f82 100644 --- a/lib/api/helpers/packages_helpers.rb +++ b/lib/api/helpers/packages_helpers.rb @@ -50,7 +50,8 @@ module API def track_package_event(event_name, scope, **args) ::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute - track_event(event_name, **args) + category = args.delete(:category) || self.options[:for].name + ::Gitlab::Tracking.event(category, event_name.to_s, **args) end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 39586483990..688cd2da994 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -38,10 +38,17 @@ module API end end + # HTTP status codes to terminate the job on GitLab Runner: + # - 403 def authenticate_job!(require_running: true) job = current_job - not_found! unless job + # 404 is not returned here because we want to terminate the job if it's + # running. A 404 can be returned from anywhere in the networking stack which is why + # we are explicit about a 403, we should improve this in + # https://gitlab.com/gitlab-org/gitlab/-/issues/327703 + forbidden! unless job + forbidden! unless job_token_valid?(job) forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete? diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index ed3d694f006..2f2ad88c942 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -394,7 +394,7 @@ module API required: true, name: :external_wiki_url, type: String, - desc: 'The URL of the external Wiki' + desc: 'The URL of the external wiki' } ], 'flowdock' => [ @@ -543,9 +543,15 @@ module API }, { required: false, + name: :jira_issue_transition_automatic, + type: Boolean, + desc: 'Enable automatic issue transitions' + }, + { + required: false, name: :jira_issue_transition_id, type: String, - desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' + desc: 'The ID of one or more transitions for custom issue transitions' }, { required: false, diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb new file mode 100644 index 00000000000..e2b3372fc33 --- /dev/null +++ b/lib/api/helpers/variables_helpers.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + module Helpers + module VariablesHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + params :optional_group_variable_params_ee do + end + + def filter_variable_parameters(_, params) + params # Overridden in EE + end + + def find_variable(owner, params) + variables = ::Ci::VariablesFinder.new(owner, params).execute.to_a + + return variables.first unless variables.many? # rubocop: disable CodeReuse/ActiveRecord + + conflict!("There are multiple variables with provided parameters. Please use 'filter[environment_scope]'") + end + end + end +end + +API::Helpers::VariablesHelpers.prepend_if_ee('EE::API::Helpers::VariablesHelpers') diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index a3fee49cd8f..4dcfc0cf7eb 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -15,7 +15,7 @@ module API Gitlab::ApplicationContext.push( user: -> { actor&.user }, project: -> { project }, - caller_id: route.origin, + caller_id: api_endpoint.endpoint_id, remote_ip: request.ip, feature_category: feature_category ) @@ -23,7 +23,7 @@ module API helpers ::API::Helpers::InternalHelpers - UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze + UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result' VALID_PAT_SCOPES = Set.new( Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth::REGISTRY_SCOPES @@ -52,20 +52,20 @@ module API actor.update_last_used_at! check_result = begin - Gitlab::Auth::CurrentUserMode.bypass_session!(actor.user&.id) do - access_check!(actor, params) - end - rescue Gitlab::GitAccess::ForbiddenError => e - # The return code needs to be 401. If we return 403 - # the custom message we return won't be shown to the user - # and, instead, the default message 'GitLab: API is not accessible' - # will be displayed - return response_with_status(code: 401, success: false, message: e.message) - rescue Gitlab::GitAccess::TimeoutError => e - return response_with_status(code: 503, success: false, message: e.message) - rescue Gitlab::GitAccess::NotFoundError => e - return response_with_status(code: 404, success: false, message: e.message) - end + with_admin_mode_bypass!(actor.user&.id) do + access_check!(actor, params) + end + rescue Gitlab::GitAccess::ForbiddenError => e + # The return code needs to be 401. If we return 403 + # the custom message we return won't be shown to the user + # and, instead, the default message 'GitLab: API is not accessible' + # will be displayed + return response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::TimeoutError => e + return response_with_status(code: 503, success: false, message: e.message) + rescue Gitlab::GitAccess::NotFoundError => e + return response_with_status(code: 404, success: false, message: e.message) + end log_user_activity(actor.user) @@ -109,9 +109,7 @@ module API end end - def validate_actor_key(actor, key_id) - return 'Could not find a user without a key' unless key_id - + def validate_actor(actor) return 'Could not find the given key' unless actor.key 'Could not find a user for the given key' unless actor.user @@ -120,6 +118,19 @@ module API def two_factor_otp_check { success: false, message: 'Feature is not available' } end + + def with_admin_mode_bypass!(actor_id) + return yield unless Gitlab::CurrentSettings.admin_mode + + Gitlab::Auth::CurrentUserMode.bypass_session!(actor_id) do + yield + end + end + + # Overridden in EE + def geo_proxy + {} + end end namespace 'internal' do @@ -193,7 +204,7 @@ module API actor.update_last_used_at! user = actor.user - error_message = validate_actor_key(actor, params[:key_id]) + error_message = validate_actor(actor) if params[:user_id] && user.nil? break { success: false, message: 'Could not find the given user' } @@ -222,7 +233,7 @@ module API actor.update_last_used_at! user = actor.user - error_message = validate_actor_key(actor, params[:key_id]) + error_message = validate_actor(actor) break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' } if actor.key.is_a?(DeployKey) @@ -295,7 +306,7 @@ module API actor.update_last_used_at! user = actor.user - error_message = validate_actor_key(actor, params[:key_id]) + error_message = validate_actor(actor) if error_message { success: false, message: error_message } @@ -314,6 +325,12 @@ module API two_factor_otp_check end + + # Workhorse calls this to determine if it is a Geo secondary site + # that should proxy requests. FOSS can quickly return empty data. + get '/geo_proxy', feature_category: :geo_replication do + geo_proxy + end end end end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 87ad79d601f..af2c53dd778 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -13,7 +13,7 @@ module API helpers do def authenticate_gitlab_kas_request! - unauthorized! unless Gitlab::Kas.verify_api_request(headers) + render_api_error!('KAS JWT authentication invalid', 401) unless Gitlab::Kas.verify_api_request(headers) end def agent_token @@ -51,9 +51,11 @@ module API end def check_agent_token - forbidden! unless agent_token + unauthorized! unless agent_token forbidden! unless Gitlab::Kas.included_in_gitlab_com_rollout?(agent.project) + + agent_token.track_usage end end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 52c32b4d1cf..0d562cc18f8 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -25,11 +25,11 @@ module API optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end post ":id/invitations" do - source = find_source(source_type, params[:id]) + params[:source] = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) + authorize_admin_source!(source_type, params[:source]) - ::Members::InviteService.new(current_user, params).execute(source) + ::Members::InviteService.new(current_user, params).execute end desc 'Get a list of group or project invitations viewable by the authenticated user' do diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index e938dbbae87..1cd5bde224b 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -18,7 +18,10 @@ module API end get ':id/issues/:issue_iid/links' do source_issue = find_project_issue(params[:issue_iid]) - related_issues = source_issue.related_issues(current_user) + related_issues = source_issue.related_issues(current_user) do |issues| + issues.with_api_entity_associations.preload_awardable + end + related_issues.each { |issue| issue.lazy_subscription(current_user, user_project) } # preload subscriptions present related_issues, with: Entities::RelatedIssue, diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 13dac1c174c..4f2ac73c0d3 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -242,7 +242,7 @@ module API use :issue_params end post ':id/issues' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42320') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21140') check_rate_limit! :issues_create, [current_user] @@ -288,7 +288,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord put ':id/issues/:issue_iid' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42322') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20775') issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue @@ -346,7 +346,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord post ':id/issues/:issue_iid/move' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42323') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20776') issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 7390219b60e..54951f9bd01 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -6,8 +6,6 @@ module API before { authenticate! } - feature_category :continuous_integration - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do params do requires :id, type: String, desc: 'The ID of a project' @@ -40,7 +38,7 @@ module API use :pagination end # rubocop: disable CodeReuse/ActiveRecord - get ':id/jobs' do + get ':id/jobs', feature_category: :continuous_integration do authorize_read_builds! builds = user_project.builds.order('id DESC') @@ -57,7 +55,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/jobs/:job_id' do + get ':id/jobs/:job_id', feature_category: :continuous_integration do authorize_read_builds! build = find_build!(params[:job_id]) @@ -72,7 +70,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/jobs/:job_id/trace' do + get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do authorize_read_builds! build = find_build!(params[:job_id]) @@ -94,7 +92,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a job' end - post ':id/jobs/:job_id/cancel' do + post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do authorize_update_builds! build = find_build!(params[:job_id]) @@ -111,7 +109,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/jobs/:job_id/retry' do + post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do authorize_update_builds! build = find_build!(params[:job_id]) @@ -129,7 +127,7 @@ module API params do requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/jobs/:job_id/erase' do + post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do authorize_update_builds! build = find_build!(params[:job_id]) @@ -148,7 +146,7 @@ module API requires :job_id, type: Integer, desc: 'The ID of a Job' end - post ":id/jobs/:job_id/play" do + post ":id/jobs/:job_id/play", feature_category: :continuous_integration do authorize_read_builds! job = find_job!(params[:job_id]) @@ -174,10 +172,8 @@ module API success Entities::Ci::Job end route_setting :authentication, job_token_allowed: true - get do - # current_authenticated_job will be nil if user is using - # a valid authentication that is not CI_JOB_TOKEN - not_found!('Job') unless current_authenticated_job + get '', feature_category: :continuous_integration do + validate_current_authenticated_job present current_authenticated_job, with: Entities::Ci::Job end @@ -196,6 +192,14 @@ module API builds.where(status: available_statuses && scope) end # rubocop: enable CodeReuse/ActiveRecord + + def validate_current_authenticated_job + # current_authenticated_job will be nil if user is using + # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN + not_found!('Job') unless current_authenticated_job + end end end end + +API::Jobs.prepend_if_ee('EE::API::Jobs') diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index 4a5b2ead163..bd1d984719e 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -23,6 +23,15 @@ module API helpers ::API::Helpers::PackagesHelpers helpers do + def path_exists?(path) + # return true when FF disabled so that processing the request is not stopped + return true unless Feature.enabled?(:check_maven_path_first) + return false if path.blank? + + Packages::Maven::Metadatum.with_path(path) + .exists? + end + def extract_format(file_name) name, _, format = file_name.rpartition('.') @@ -77,6 +86,22 @@ module API request.head? && file.fog_credentials[:provider] == 'AWS' end + + def fetch_package(file_name:, project: nil, group: nil) + order_by_package_file = false + if Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml) + order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) && + !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM) + end + + ::Packages::Maven::PackageFinder.new( + params[:path], + current_user, + project: project, + group: group, + order_by_package_file: order_by_package_file + ).execute! + end end desc 'Download the maven package file at instance level' do @@ -88,6 +113,9 @@ module API end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + # return a similar failure to authorize_read_package!(project) + forbidden! unless path_exists?(params[:path]) + file_name, format = extract_format(params[:file_name]) # To avoid name collision we require project path and project package be the same. @@ -97,8 +125,7 @@ module API authorize_read_package!(project) - package = ::Packages::Maven::PackageFinder - .new(params[:path], current_user, project: project).execute! + package = fetch_package(file_name: file_name, project: project) package_file = ::Packages::PackageFileFinder .new(package, file_name).execute! @@ -127,14 +154,16 @@ module API end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + # return a similar failure to group = find_group(params[:id]) + not_found!('Group') unless path_exists?(params[:path]) + file_name, format = extract_format(params[:file_name]) group = find_group(params[:id]) not_found!('Group') unless can?(current_user, :read_group, group) - package = ::Packages::Maven::PackageFinder - .new(params[:path], current_user, group: group).execute! + package = fetch_package(file_name: file_name, group: group) authorize_read_package!(package.project) @@ -167,12 +196,14 @@ module API end route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + # return a similar failure to user_project + not_found!('Project') unless path_exists?(params[:path]) + authorize_read_package!(user_project) file_name, format = extract_format(params[:file_name]) - package = ::Packages::Maven::PackageFinder - .new(params[:path], current_user, project: user_project).execute! + package = fetch_package(file_name: file_name, project: user_project) package_file = ::Packages::PackageFileFinder .new(package, file_name).execute! diff --git a/lib/api/members.rb b/lib/api/members.rb index 42f608102b3..aaf0e3e1927 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -100,9 +100,9 @@ module API authorize_admin_source!(source_type, source) if params[:user_id].to_s.include?(',') - create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id] }) + create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id], source: source }) - ::Members::CreateService.new(current_user, create_service_params).execute(source) + ::Members::CreateService.new(current_user, create_service_params).execute elsif params[:user_id].present? member = source.members.find_by(user_id: params[:user_id]) conflict!('Member already exists') if member @@ -155,6 +155,8 @@ module API desc 'Removes a user from a group or project.' params do requires :user_id, type: Integer, desc: 'The user ID of the member' + optional :skip_subresources, type: Boolean, default: false, + desc: 'Flag indicating if the deletion of direct memberships of the removed member in subgroups and projects should be skipped' optional :unassign_issuables, type: Boolean, default: false, desc: 'Flag indicating if the removed member should be unassigned from any issues or merge requests within given group or project' end @@ -164,7 +166,7 @@ module API member = source_members(source).find_by!(user_id: params[:user_id]) destroy_conditionally!(member) do - ::Members::DestroyService.new(current_user).execute(member, unassign_issuables: params[:unassign_issuables]) + ::Members::DestroyService.new(current_user).execute(member, skip_subresources: params[:skip_subresources], unassign_issuables: params[:unassign_issuables]) end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 97a6c7075b3..470f78a7dc2 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -45,7 +45,7 @@ module API merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull + present_cached merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull, cache_context: nil end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 5051c1a5529..613de514ffa 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -8,11 +8,20 @@ module API before { authenticate_non_get! } - feature_category :code_review - helpers Helpers::MergeRequestsHelpers helpers Helpers::SSEHelpers + # These endpoints are defined in `TimeTrackingEndpoints` and is shared by + # API::Issues. In order to be able to define the feature category of these + # endpoints, we need to define them at the top-level by route. + feature_category :code_review, [ + '/projects/:id/merge_requests/:merge_request_iid/time_estimate', + '/projects/:id/merge_requests/:merge_request_iid/reset_time_estimate', + '/projects/:id/merge_requests/:merge_request_iid/add_spent_time', + '/projects/:id/merge_requests/:merge_request_iid/reset_spent_time', + '/projects/:id/merge_requests/:merge_request_iid/time_stats' + ] + # EE::API::MergeRequests would override the following helpers helpers do params :optional_params_ee do @@ -125,7 +134,7 @@ module API use :merge_requests_params use :optional_scope_param end - get do + get feature_category: :code_review do authenticate! unless params[:scope] == 'all' merge_requests = find_merge_requests @@ -145,7 +154,7 @@ module API optional :non_archived, type: Boolean, desc: 'Return merge requests from non archived projects', default: true end - get ":id/merge_requests" do + get ":id/merge_requests", feature_category: :code_review do merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true) present merge_requests, serializer_options_for(merge_requests).merge(group: user_group) @@ -184,7 +193,7 @@ module API use :merge_requests_params optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of merge requests' end - get ":id/merge_requests" do + get ":id/merge_requests", feature_category: :code_review do authorize! :read_merge_request, user_project merge_requests = find_merge_requests(project_id: user_project.id) @@ -206,8 +215,8 @@ module API desc: 'The target project of the merge request defaults to the :id of the project' use :optional_params end - post ":id/merge_requests" do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42316') + post ":id/merge_requests", feature_category: :code_review do + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20770') authorize! :create_merge_request_from, user_project @@ -228,7 +237,7 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' end - delete ":id/merge_requests/:merge_request_iid" do + delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review do merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) @@ -247,7 +256,7 @@ module API desc 'Get a single merge request' do success Entities::MergeRequest end - get ':id/merge_requests/:merge_request_iid' do + get ':id/merge_requests/:merge_request_iid', feature_category: :code_review do not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project) merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -265,7 +274,7 @@ module API desc 'Get the participants of a merge request' do success Entities::UserBasic end - get ':id/merge_requests/:merge_request_iid/participants' do + get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review do not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project) merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -278,7 +287,7 @@ module API desc 'Get the commits of a merge request' do success Entities::Commit end - get ':id/merge_requests/:merge_request_iid/commits' do + get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review do not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project) merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -293,7 +302,7 @@ module API desc 'Get the context commits of a merge request' do success Entities::Commit end - get ':id/merge_requests/:merge_request_iid/context_commits' do + get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do merge_request = find_merge_request_with_access(params[:merge_request_iid]) project = merge_request.project @@ -311,7 +320,7 @@ module API desc 'create context commits of merge request' do success Entities::Commit end - post ':id/merge_requests/:merge_request_iid/context_commits' do + post ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do commit_ids = params[:commits] if commit_ids.size > CONTEXT_COMMITS_POST_LIMIT @@ -339,7 +348,7 @@ module API requires :commits, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, allow_blank: false, desc: 'List of context commits sha' end desc 'remove context commits of merge request' - delete ':id/merge_requests/:merge_request_iid/context_commits' do + delete ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do commit_ids = params[:commits] merge_request = find_merge_request_with_access(params[:merge_request_iid]) project = merge_request.project @@ -361,7 +370,7 @@ module API desc 'Show the merge request changes' do success Entities::MergeRequestChanges end - get ':id/merge_requests/:merge_request_iid/changes' do + get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review do not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project) merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -376,7 +385,7 @@ module API desc 'Get the merge request pipelines' do success Entities::Ci::PipelineBasic end - get ':id/merge_requests/:merge_request_iid/pipelines' do + get ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do pipelines = merge_request_pipelines_with_access not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project) @@ -387,7 +396,7 @@ module API desc 'Create a pipeline for merge request' do success ::API::Entities::Ci::Pipeline end - post ':id/merge_requests/:merge_request_iid/pipelines' do + post ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do pipeline = ::MergeRequests::CreatePipelineService .new(user_project, current_user, allow_duplicate: true) .execute(find_merge_request_with_access(params[:merge_request_iid])) @@ -415,8 +424,8 @@ module API use :optional_params at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of) end - put ':id/merge_requests/:merge_request_iid' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42318') + put ':id/merge_requests/:merge_request_iid', feature_category: :code_review do + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20772') merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) @@ -424,7 +433,13 @@ module API mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params.has_key?(:remove_source_branch) mr_params = convert_parameters_from_legacy_format(mr_params) - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) + service = if mr_params.one? && (mr_params.keys & %i[assignee_id assignee_ids]).one? + ::MergeRequests::UpdateAssigneesService + else + ::MergeRequests::UpdateService + end + + merge_request = service.new(user_project, current_user, mr_params).execute(merge_request) handle_merge_request_errors!(merge_request) @@ -444,8 +459,8 @@ module API optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge' end - put ':id/merge_requests/:merge_request_iid/merge' do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317') + put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review do + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796') merge_request = find_project_merge_request(params[:merge_request_iid]) @@ -485,7 +500,7 @@ module API end desc 'Returns the up to date merge-ref HEAD commit' - get ':id/merge_requests/:merge_request_iid/merge_ref' do + get ':id/merge_requests/:merge_request_iid/merge_ref', feature_category: :code_review do merge_request = find_project_merge_request(params[:merge_request_iid]) result = ::MergeRequests::MergeabilityCheckService.new(merge_request).execute(recheck: true) @@ -500,7 +515,7 @@ module API desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do success Entities::MergeRequest end - post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do + post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds', feature_category: :code_review do merge_request = find_project_merge_request(params[:merge_request_iid]) unauthorized! unless merge_request.can_cancel_auto_merge?(current_user) @@ -514,7 +529,7 @@ module API params do optional :skip_ci, type: Boolean, desc: 'Do not create CI pipeline' end - put ':id/merge_requests/:merge_request_iid/rebase' do + put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review do merge_request = find_project_merge_request(params[:merge_request_iid]) authorize_push_to_merge_request!(merge_request) @@ -533,7 +548,7 @@ module API params do use :pagination end - get ':id/merge_requests/:merge_request_iid/closes_issues' do + get ':id/merge_requests/:merge_request_iid/closes_issues', feature_category: :code_review do merge_request = find_merge_request_with_access(params[:merge_request_iid]) issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user)) issues = paginate(issues) diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index b337b992841..d75ed3a48d7 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -80,7 +80,7 @@ module API params = build_finder_params(milestone, parent) - issuables = finder_klass.new(current_user, params).execute + issuables = finder_klass.new(current_user, params).execute.with_api_entity_associations present paginate(issuables), with: entity, current_user: current_user end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 25a901c18b6..465d2f23e9d 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -56,6 +56,23 @@ module API present user_namespace, with: Entities::Namespace, current_user: current_user end + + desc 'Get existence of a namespace including alternative suggestions' do + success Entities::NamespaceExistence + end + params do + requires :namespace, type: String, desc: "Namespace's path" + optional :parent_id, type: Integer, desc: "The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered." + end + get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace_path = params[:namespace] + + exists = Namespace.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists? + suggestions = exists ? [Namespace.clean_path(namespace_path)] : [] + + present :exists, exists + present :suggests, suggestions + end end end end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 15b06cea385..5f3a574eeee 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -67,7 +67,7 @@ module API check_rate_limit! :project_import, [current_user, :project_import] - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823') validate_file! diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 19b63c28f89..92f6970e6fc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -13,6 +13,8 @@ module API feature_category :projects, ['/projects/:id/custom_attributes', '/projects/:id/custom_attributes/:key'] + PROJECT_ATTACHMENT_SIZE_EXEMPT = 1.gigabyte + helpers do # EE::API::Projects would override this method def apply_filters(projects) @@ -52,6 +54,29 @@ module API accepted! end + + def exempt_from_global_attachment_size?(user_project) + list = ::Gitlab::RackAttack::UserAllowlist.new(ENV['GITLAB_UPLOAD_API_ALLOWLIST']) + list.include?(user_project.id) + end + + # Temporarily introduced for upload API: https://gitlab.com/gitlab-org/gitlab/-/issues/325788 + def project_attachment_size(user_project) + return PROJECT_ATTACHMENT_SIZE_EXEMPT if exempt_from_global_attachment_size?(user_project) + return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project) + + PROJECT_ATTACHMENT_SIZE_EXEMPT + end + + # This is to help determine which projects to use in https://gitlab.com/gitlab-org/gitlab/-/issues/325788 + def log_if_upload_exceed_max_size(user_project, file) + return if file.size <= user_project.max_attachment_size + + if file.size > user_project.max_attachment_size + allowed = exempt_from_global_attachment_size?(user_project) + Gitlab::AppLogger.info({ message: "File exceeds maximum size", file_bytes: file.size, project_id: user_project.id, project_path: user_project.full_path, upload_allowed: allowed }) + end + end end helpers do @@ -215,7 +240,7 @@ module API use :create_params end post do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/issues/21139') attrs = declared_params(include_missing: false) attrs = translate_params_for_compatibility(attrs) filter_attributes_using_license!(attrs) @@ -248,7 +273,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord post "user/:user_id", feature_category: :projects do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/issues/21139') authenticated_as_admin! user = User.find_by(id: params.delete(:user_id)) not_found!('User') unless user @@ -310,7 +335,7 @@ module API optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' end post ':id/fork', feature_category: :source_code_management do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759') not_found! unless can?(current_user, :fork_project, user_project) @@ -460,7 +485,7 @@ module API get ':id/languages', feature_category: :source_code_management do ::Projects::RepositoryLanguagesService .new(user_project, current_user) - .execute.map { |lang| [lang.name, lang.share] }.to_h + .execute.to_h { |lang| [lang.name, lang.share] } end desc 'Delete a project' @@ -545,13 +570,27 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Workhorse authorize the file upload' do + detail 'This feature was introduced in GitLab 13.11' + end + post ':id/uploads/authorize', feature_category: :not_owned do + require_gitlab_workhorse! + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + FileUploader.workhorse_authorize(has_length: false, maximum_size: project_attachment_size(user_project)) + end + desc 'Upload a file' params do - # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 - requires :file, type: File, desc: 'The file to be uploaded' # rubocop:disable Scalability/FileUploads + requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded' end post ":id/uploads", feature_category: :not_owned do - upload = UploadService.new(user_project, params[:file]).execute + log_if_upload_exceed_max_size(user_project, params[:file]) + + service = UploadService.new(user_project, params[:file]) + service.override_max_attachment_size = project_attachment_size(user_project) + upload = service.execute present upload, with: Entities::ProjectUpload end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index f6ffeeea829..033cc6744b0 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -116,10 +116,23 @@ module API params do requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + optional :from_project_id, type: String, desc: 'The project to compare from' optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare' do - compare = CompareService.new(user_project, params[:to]).execute(user_project, params[:from], straight: params[:straight]) + if params[:from_project_id].present? + target_project = MergeRequestTargetProjectFinder + .new(current_user: current_user, source_project: user_project, project_feature: :repository) + .execute(include_routes: true).find_by_id(params[:from_project_id]) + + if target_project.blank? + render_api_error!("Target project id:#{params[:from_project_id]} is not a fork of project id:#{params[:id]}", 400) + end + else + target_project = user_project + end + + compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight]) if compare present compare, with: Entities::Compare diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index 99c278be8e7..705e4778c83 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -19,7 +19,7 @@ module API get ":id/access_tokens" do resource = find_source(source_type, params[:id]) - next unauthorized! unless has_permission_to_read?(resource) + next unauthorized! unless current_user.can?(:read_resource_access_tokens, resource) tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute @@ -85,10 +85,6 @@ module API def find_token(resource, token_id) PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).find_by_id(token_id) end - - def has_permission_to_read?(resource) - can?(current_user, :project_bot_access, resource) || can?(current_user, :admin_resource_access_tokens, resource) - end end end end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 8d2d4586d8d..1d17148e0df 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -99,6 +99,8 @@ module API track_package_event('push_package', :rubygems) + package_file = nil + ActiveRecord::Base.transaction do package = ::Packages::CreateTemporaryPackageService.new( user_project, current_user, declared_params.merge(build: current_authenticated_job) @@ -109,12 +111,18 @@ module API file_name: PACKAGE_FILENAME } - ::Packages::CreatePackageFileService.new( + package_file = ::Packages::CreatePackageFileService.new( package, file_params.merge(build: current_authenticated_job) ).execute end - created! + if package_file + ::Packages::Rubygems::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker + + created! + else + bad_request!('Package creation failed') + end rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: user_project.id }) diff --git a/lib/api/search.rb b/lib/api/search.rb index f0ffe6ba443..8fabf379d49 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -22,13 +22,15 @@ module API users: Entities::UserBasic }.freeze - SCOPE_PRELOAD_METHOD = { - merge_requests: :with_api_entity_associations, - projects: :with_api_entity_associations, - issues: :with_api_entity_associations, - milestones: :with_api_entity_associations, - commits: :with_api_commit_entity_associations - }.freeze + def scope_preload_method + { + merge_requests: :with_api_entity_associations, + projects: :with_api_entity_associations, + issues: :with_api_entity_associations, + milestones: :with_api_entity_associations, + commits: :with_api_commit_entity_associations + }.freeze + end def search(additional_params = {}) search_params = { @@ -60,7 +62,7 @@ module API end def preload_method - SCOPE_PRELOAD_METHOD[params[:scope].to_sym] + scope_preload_method[params[:scope].to_sym] end def verify_search_scope!(resource:) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 64a72b4cb7f..95d0c525ced 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -30,6 +30,7 @@ module API success Entities::ApplicationSetting end params do + optional :admin_mode, type: Boolean, desc: 'Require admin users to re-authenticate for administrative (i.e. potentially dangerous) operations' optional :admin_notification_email, type: String, desc: 'Deprecated: Use :abuse_notification_email instead. Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' optional :abuse_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 7636c45bdac..e77d7e34de3 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -28,7 +28,13 @@ module API sort: "#{params[:order_by]}_#{params[:sort]}", search: params[:search]).execute - present paginate(::Kaminari.paginate_array(tags)), with: Entities::Tag, project: user_project + paginated_tags = paginate(::Kaminari.paginate_array(tags)) + + if Feature.enabled?(:api_caching_tags, user_project, type: :development) + present_cached paginated_tags, with: Entities::Tag, project: user_project, cache_context: -> (_tag) { user_project.cache_key } + else + present paginated_tags, with: Entities::Tag, project: user_project + end end desc 'Get a single repository tag' do diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index aebbc95cbea..84c51e5aeac 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -21,7 +21,7 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42283') + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758') forbidden! if gitlab_pipeline_hook_request? diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index c7d63f8d6ac..7deec15dcac 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -2,24 +2,22 @@ module API class UsageData < ::API::Base - before { authenticate! } + before { authenticate_non_get! } feature_category :usage_ping namespace 'usage_data' do before do - not_found! unless Feature.enabled?(:usage_data_api, default_enabled: true) + not_found! unless Feature.enabled?(:usage_data_api, default_enabled: :yaml, type: :ops) forbidden!('Invalid CSRF token is provided') unless verified_request? end desc 'Track usage data events' do detail 'This feature was introduced in GitLab 13.4.' end - params do requires :event, type: String, desc: 'The event name that should be tracked' end - post 'increment_counter' do event_name = params[:event] @@ -31,7 +29,6 @@ module API params do requires :event, type: String, desc: 'The event name that should be tracked' end - post 'increment_unique_users' do event_name = params[:event] @@ -39,6 +36,16 @@ module API status :ok end + + desc 'Get a list of all metric definitions' do + detail 'This feature was introduced in GitLab 13.11.' + end + get 'metric_definitions' do + content_type 'application/yaml' + env['api.format'] = :binary + + Gitlab::Usage::MetricDefinition.dump_metrics_yaml + end end end end diff --git a/lib/api/usage_data_non_sql_metrics.rb b/lib/api/usage_data_non_sql_metrics.rb new file mode 100644 index 00000000000..63a14a223f5 --- /dev/null +++ b/lib/api/usage_data_non_sql_metrics.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + class UsageDataNonSqlMetrics < ::API::Base + before { authenticated_as_admin! } + + feature_category :usage_ping + + namespace 'usage_data' do + before do + not_found! unless Feature.enabled?(:usage_data_non_sql_metrics, default_enabled: :yaml, type: :ops) + end + + desc 'Get Non SQL usage ping metrics' do + detail 'This feature was introduced in GitLab 13.11.' + end + + get 'non_sql_metrics' do + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534') + + data = Gitlab::UsageDataNonSqlMetrics.uncached_data + + present data + end + end + end +end diff --git a/lib/api/usage_data_queries.rb b/lib/api/usage_data_queries.rb new file mode 100644 index 00000000000..0ad9ad7650c --- /dev/null +++ b/lib/api/usage_data_queries.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module API + class UsageDataQueries < ::API::Base + before { authenticated_as_admin! } + + feature_category :usage_ping + + namespace 'usage_data' do + before do + not_found! unless Feature.enabled?(:usage_data_queries_api, default_enabled: :yaml, type: :ops) + end + + desc 'Get raw SQL queries for usage data SQL metrics' do + detail 'This feature was introduced in GitLab 13.11.' + end + + get 'queries' do + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534') + + queries = Gitlab::UsageDataQueries.uncached_data + + present queries + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index b2f99bb18dc..078ba7542a3 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -231,7 +231,7 @@ module API optional :password, type: String, desc: 'The password of the new user' optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' optional :skip_confirmation, type: Boolean, desc: 'Flag indicating the account is confirmed' - at_least_one_of :password, :reset_password + at_least_one_of :password, :reset_password, :force_random_password requires :name, type: String, desc: 'The name of the user' requires :username, type: String, desc: 'The username of the user' optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set' @@ -571,8 +571,6 @@ module API end # rubocop: disable CodeReuse/ActiveRecord delete ":id", feature_category: :users do - Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/20757') - authenticated_as_admin! user = User.find_by(id: params[:id]) @@ -998,6 +996,29 @@ module API present paginate(current_user.emails), with: Entities::Email end + desc "Update the current user's preferences" do + success Entities::UserPreferences + detail 'This feature was introduced in GitLab 13.10.' + end + params do + requires :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' + end + put "preferences", feature_category: :users do + authenticate! + + preferences = current_user.user_preference + + attrs = declared_params(include_missing: false) + + service = ::UserPreferences::UpdateService.new(current_user, attrs).execute + + if service.success? + present preferences, with: Entities::UserPreferences + else + render_api_error!('400 Bad Request', 400) + end + end + desc 'Get a single email address owned by the currently authenticated user' do success Entities::Email end diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 2d25e76626a..29e4a79110f 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -18,7 +18,7 @@ module API # Used to differentiate Jira Cloud requests from Jira Server requests # Jira Cloud user agent format: Jira DVCS Connector Vertigo/version # Jira Server user agent format: Jira DVCS Connector/version - JIRA_DVCS_CLOUD_USER_AGENT = 'Jira DVCS Connector Vertigo'.freeze + JIRA_DVCS_CLOUD_USER_AGENT = 'Jira DVCS Connector Vertigo' include PaginationParams @@ -75,11 +75,14 @@ module API # rubocop: enable CodeReuse/ActiveRecord def authorized_merge_requests - MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?).execute + MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?) + .execute.with_jira_integration_associations end def authorized_merge_requests_for_project(project) - MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?, project_id: project.id).execute + MergeRequestsFinder + .new(current_user, authorized_only: !current_user.admin?, project_id: project.id) + .execute.with_jira_integration_associations end # rubocop: disable CodeReuse/ActiveRecord @@ -194,16 +197,13 @@ module API # Self-hosted Jira (tested on 7.11.1) requests this endpoint right # after fetching branches. - # rubocop: disable CodeReuse/ActiveRecord get ':namespace/:project/events' do user_project = find_project_with_access(params) merge_requests = authorized_merge_requests_for_project(user_project) - merge_requests = merge_requests.preload(:author, :assignees, :metrics, source_project: :namespace, target_project: :namespace) present paginate(merge_requests), with: ::API::Github::Entities::PullRequestEvent end - # rubocop: enable CodeReuse/ActiveRecord params do use :project_full_path diff --git a/lib/api/variables.rb b/lib/api/variables.rb index 94fa98b7a14..8b0745c6b5b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -9,21 +9,7 @@ module API feature_category :continuous_integration - helpers do - def filter_variable_parameters(params) - # This method exists so that EE can more easily filter out certain - # parameters, without having to modify the source code directly. - params - end - - def find_variable(params) - variables = ::Ci::VariablesFinder.new(user_project, params).execute.to_a - - return variables.first unless variables.many? # rubocop: disable CodeReuse/ActiveRecord - - conflict!("There are multiple variables with provided parameters. Please use 'filter[environment_scope]'") - end - end + helpers Helpers::VariablesHelpers params do requires :id, type: String, desc: 'The ID of a project' @@ -49,7 +35,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get ':id/variables/:key' do - variable = find_variable(params) + variable = find_variable(user_project, params) not_found!('Variable') unless variable present variable, with: Entities::Ci::Variable @@ -71,7 +57,7 @@ module API variable = ::Ci::ChangeVariableService.new( container: user_project, current_user: current_user, - params: { action: :create, variable_params: filter_variable_parameters(declared_params(include_missing: false)) } + params: { action: :create, variable_params: declared_params(include_missing: false) } ).execute if variable.valid? @@ -95,17 +81,13 @@ module API end # rubocop: disable CodeReuse/ActiveRecord put ':id/variables/:key' do - variable = find_variable(params) + variable = find_variable(user_project, params) not_found!('Variable') unless variable - variable_params = filter_variable_parameters( - declared_params(include_missing: false) - .except(:key, :filter) - ) variable = ::Ci::ChangeVariableService.new( container: user_project, current_user: current_user, - params: { action: :update, variable: variable, variable_params: variable_params } + params: { action: :update, variable: variable, variable_params: declared_params(include_missing: false).except(:key, :filter) } ).execute if variable.valid? @@ -125,7 +107,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord delete ':id/variables/:key' do - variable = find_variable(params) + variable = find_variable(user_project, params) not_found!('Variable') unless variable ::Ci::ChangeVariableService.new( diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index 6f87b7b7d3c..ea83076c49b 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -141,9 +141,9 @@ module Atlassian def user_notes_count(merge_requests) return unless merge_requests - Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').map do |count_group| + Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').to_h do |count_group| [count_group.noteable_id, count_group.count] - end.to_h + end end def jwt_token(http_method, uri) diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb index 8372d2a62da..a3434c529a4 100644 --- a/lib/atlassian/jira_connect/serializers/build_entity.rb +++ b/lib/atlassian/jira_connect/serializers/build_entity.rb @@ -25,11 +25,11 @@ module Atlassian # extract Jira issue keys from either the source branch/ref or the # merge request title. @issue_keys ||= begin - pipeline.all_merge_requests.flat_map do |mr| - src = "#{mr.source_branch} #{mr.title}" - JiraIssueKeyExtractor.new(src).issue_keys - end.uniq - end + pipeline.all_merge_requests.flat_map do |mr| + src = "#{mr.source_branch} #{mr.title}" + JiraIssueKeyExtractor.new(src).issue_keys + end.uniq + end end private diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 79b7b2c61f2..627bb44331b 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -249,7 +249,7 @@ module Backup progress.puts " * #{display_repo_path} ... " if repository.empty? - progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan) + progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan) return end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb deleted file mode 100644 index 2448c2c2bb2..00000000000 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ /dev/null @@ -1,446 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # Issues, Merge Requests, Snippets, Commits and Commit Ranges share - # similar functionality in reference filtering. - class AbstractReferenceFilter < ReferenceFilter - include CrossProjectReference - - # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found - # reference (which we replace with placeholder during re-scaping). The - # random number helps ensure it's pretty close to unique. Since it's a - # transitory value (it never gets saved) we can initialize once, and it - # doesn't matter if it changes on a restart. - REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_" - REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze - - def self.object_class - # Implement in child class - # Example: MergeRequest - end - - def self.object_name - @object_name ||= object_class.name.underscore - end - - def self.object_sym - @object_sym ||= object_name.to_sym - end - - # Public: Find references in text (like `!123` for merge requests) - # - # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| - # object = find_object(project_ref, id) - # "<a href=...>#{object.to_reference}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, the Integer referenced object ID, an optional String - # of the external project reference, and all of the matchdata. - # - # Returns a String replaced with the return of the block. - def self.references_in(text, pattern = object_class.reference_pattern) - text.gsub(pattern) do |match| - if ident = identifier($~) - yield match, ident, $~[:project], $~[:namespace], $~ - else - match - end - end - end - - def self.identifier(match_data) - symbol = symbol_from_match(match_data) - - parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol) - end - - def identifier(match_data) - self.class.identifier(match_data) - end - - def self.symbol_from_match(match) - key = object_sym - match[key] if match.names.include?(key.to_s) - end - - # Transform a symbol extracted from the text to a meaningful value - # In most cases these will be integers, so we call #to_i by default - # - # This method has the contract that if a string `ref` refers to a - # record `record`, then `parse_symbol(ref) == record_identifier(record)`. - def self.parse_symbol(symbol, match_data) - symbol.to_i - end - - # We assume that most classes are identifying records by ID. - # - # This method has the contract that if a string `ref` refers to a - # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`. - def record_identifier(record) - record.id - end - - def object_class - self.class.object_class - end - - def object_sym - self.class.object_sym - end - - def references_in(*args, &block) - self.class.references_in(*args, &block) - end - - # Implement in child class - # Example: project.merge_requests.find - def find_object(parent_object, id) - end - - # Override if the link reference pattern produces a different ID (global - # ID vs internal ID, for instance) to the regular reference pattern. - def find_object_from_link(parent_object, id) - find_object(parent_object, id) - end - - # Implement in child class - # Example: project_merge_request_url - def url_for_object(object, parent_object) - end - - def find_object_cached(parent_object, id) - cached_call(:banzai_find_object, id, path: [object_class, parent_object.id]) do - find_object(parent_object, id) - end - end - - def find_object_from_link_cached(parent_object, id) - cached_call(:banzai_find_object_from_link, id, path: [object_class, parent_object.id]) do - find_object_from_link(parent_object, id) - end - end - - def from_ref_cached(ref) - cached_call("banzai_#{parent_type}_refs".to_sym, ref) do - parent_from_ref(ref) - end - end - - def url_for_object_cached(object, parent_object) - cached_call(:banzai_url_for_object, object, path: [object_class, parent_object.id]) do - url_for_object(object, parent_object) - end - end - - def call - return doc unless project || group || user - - ref_pattern = object_class.reference_pattern - link_pattern = object_class.link_reference_pattern - - # Compile often used regexps only once outside of the loop - ref_pattern_anchor = /\A#{ref_pattern}\z/ - link_pattern_start = /\A#{link_pattern}/ - link_pattern_anchor = /\A#{link_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) && ref_pattern - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - object_link_filter(content, ref_pattern) - end - - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if ref_pattern && link =~ ref_pattern_anchor - replace_link_node_with_href(node, index, link) do - object_link_filter(link, ref_pattern, link_content: inner_html) - end - - next - end - - next unless link_pattern - - if link == inner_html && inner_html =~ link_pattern_start - replace_link_node_with_text(node, index) do - object_link_filter(inner_html, link_pattern, link_reference: true) - end - - next - end - - if link =~ link_pattern_anchor - replace_link_node_with_href(node, index, link) do - object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true) - end - - next - end - end - end - end - - doc - end - - # Replace references (like `!123` for merge requests) in text with links - # to the referenced object's details page. - # - # text - String text to replace references in. - # pattern - Reference pattern to match against. - # link_content - Original content of the link being replaced. - # link_reference - True if this was using the link reference pattern, - # false otherwise. - # - # Returns a String with references replaced with links. All links - # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. - def object_link_filter(text, pattern, link_content: nil, link_reference: false) - references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| - parent_path = if parent_type == :group - full_group_path(namespace_ref) - else - full_project_path(namespace_ref, project_ref) - end - - parent = from_ref_cached(parent_path) - - if parent - object = - if link_reference - find_object_from_link_cached(parent, id) - else - find_object_cached(parent, id) - end - end - - if object - title = object_link_title(object, matches) - klass = reference_class(object_sym) - - data_attributes = data_attributes_for(link_content || match, parent, object, - link_content: !!link_content, - link_reference: link_reference) - data = data_attribute(data_attributes) - - url = - if matches.names.include?("url") && matches[:url] - matches[:url] - else - url_for_object_cached(object, parent) - end - - content = link_content || object_link_text(object, matches) - - link = %(<a href="#{url}" #{data} - title="#{escape_once(title)}" - class="#{klass}">#{content}</a>) - - wrap_link(link, object) - else - match - end - end - end - - def wrap_link(link, object) - link - end - - def data_attributes_for(text, parent, object, link_content: false, link_reference: false) - object_parent_type = parent.is_a?(Group) ? :group : :project - - { - original: escape_html_entities(text), - link: link_content, - link_reference: link_reference, - object_parent_type => parent.id, - object_sym => object.id - } - end - - def object_link_text_extras(object, matches) - extras = [] - - if matches.names.include?("anchor") && matches[:anchor] && matches[:anchor] =~ /\A\#note_(\d+)\z/ - extras << "comment #{Regexp.last_match(1)}" - end - - extension = matches[:extension] if matches.names.include?("extension") - - extras << extension if extension - - extras - end - - def object_link_title(object, matches) - object.title - end - - def object_link_text(object, matches) - parent = project || group || user - text = object.reference_link_text(parent) - - extras = object_link_text_extras(object, matches) - text += " (#{extras.join(", ")})" if extras.any? - - text - end - - # Returns a Hash containing all object references (e.g. issue IDs) per the - # project they belong to. - def references_per_parent - @references_per ||= {} - - @references_per[parent_type] ||= begin - refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = [ - object_class.link_reference_pattern, - object_class.reference_pattern - ].compact.reduce { |a, b| Regexp.union(a, b) } - - nodes.each do |node| - node.to_html.scan(regex) do - path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end - - if ident = identifier($~) - refs[path] << ident - end - end - end - - refs - end - end - - # Returns a Hash containing referenced projects grouped per their full - # path. - def parent_per_reference - @per_reference ||= {} - - @per_reference[parent_type] ||= begin - refs = Set.new - - references_per_parent.each do |ref, _| - refs << ref - end - - find_for_paths(refs.to_a).index_by(&:full_path) - end - end - - def relation_for_paths(paths) - klass = parent_type.to_s.camelize.constantize - result = klass.where_full_path_in(paths) - return result if parent_type == :group - - result.includes(:namespace) if parent_type == :project - end - - # Returns projects for the given paths. - def find_for_paths(paths) - if Gitlab::SafeRequestStore.active? - cache = refs_cache - to_query = paths - cache.keys - - unless to_query.empty? - records = relation_for_paths(to_query) - - found = [] - records.each do |record| - ref = record.full_path - get_or_set_cache(cache, ref) { record } - found << ref - end - - not_found = to_query - found - not_found.each do |ref| - get_or_set_cache(cache, ref) { nil } - end - end - - cache.slice(*paths).values.compact - else - relation_for_paths(paths) - end - end - - def current_parent_path - @current_parent_path ||= parent&.full_path - end - - def current_project_namespace_path - @current_project_namespace_path ||= project&.namespace&.full_path - end - - def records_per_parent - @_records_per_project ||= {} - - @_records_per_project[object_class.to_s.underscore] ||= begin - hash = Hash.new { |h, k| h[k] = {} } - - parent_per_reference.each do |path, parent| - record_ids = references_per_parent[path] - - parent_records(parent, record_ids).each do |record| - hash[parent][record_identifier(record)] = record - end - end - - hash - end - end - - private - - def full_project_path(namespace, project_ref) - return current_parent_path unless project_ref - - namespace_ref = namespace || current_project_namespace_path - "#{namespace_ref}/#{project_ref}" - end - - def refs_cache - Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} - end - - def parent_type - :project - end - - def parent - parent_type == :project ? project : group - end - - def full_group_path(group_ref) - return current_parent_path unless group_ref - - group_ref - end - - def unescape_html_entities(text) - CGI.unescapeHTML(text.to_s) - end - - def escape_html_entities(text) - CGI.escapeHTML(text.to_s) - end - - def escape_with_placeholders(text, placeholder_data) - escaped = escape_html_entities(text) - - escaped.gsub(REFERENCE_PLACEHOLDER_PATTERN) do |match| - placeholder_data[Regexp.last_match(1).to_i] - end - end - end - end -end - -Banzai::Filter::AbstractReferenceFilter.prepend_if_ee('EE::Banzai::Filter::AbstractReferenceFilter') diff --git a/lib/banzai/filter/alert_reference_filter.rb b/lib/banzai/filter/alert_reference_filter.rb deleted file mode 100644 index 228a4159c99..00000000000 --- a/lib/banzai/filter/alert_reference_filter.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class AlertReferenceFilter < IssuableReferenceFilter - self.reference_type = :alert - - def self.object_class - AlertManagement::Alert - end - - def self.object_sym - :alert - end - - def parent_records(parent, ids) - parent.alert_management_alerts.where(iid: ids.to_a) - end - - def url_for_object(alert, project) - ::Gitlab::Routing.url_helpers.details_project_alert_management_url( - project, - alert.iid, - only_path: context[:only_path] - ) - end - end - end -end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index d569711431c..a86c1bb2892 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -43,7 +43,7 @@ module Banzai TEXT_QUERY = %Q(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., '://') - ]).freeze + ]) PUNCTUATION_PAIRS = { "'" => "'", diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb deleted file mode 100644 index d6b46236a49..00000000000 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces commit range references with links. - # - # This filter supports cross-project references. - class CommitRangeReferenceFilter < AbstractReferenceFilter - self.reference_type = :commit_range - - def self.object_class - CommitRange - end - - def self.references_in(text, pattern = CommitRange.reference_pattern) - text.gsub(pattern) do |match| - yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ - end - end - - def initialize(*args) - super - - @commit_map = {} - end - - def find_object(project, id) - return unless project.is_a?(Project) - - range = CommitRange.new(id, project) - - range.valid_commits? ? range : nil - end - - def url_for_object(range, project) - h = Gitlab::Routing.url_helpers - h.project_compare_url(project, - range.to_param.merge(only_path: context[:only_path])) - end - - def object_link_title(range, matches) - nil - end - end - end -end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb deleted file mode 100644 index 3df003a88fa..00000000000 --- a/lib/banzai/filter/commit_reference_filter.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces commit references with links. - # - # This filter supports cross-project references. - class CommitReferenceFilter < AbstractReferenceFilter - self.reference_type = :commit - - def self.object_class - Commit - end - - def self.references_in(text, pattern = Commit.reference_pattern) - text.gsub(pattern) do |match| - yield match, $~[:commit], $~[:project], $~[:namespace], $~ - end - end - - def find_object(project, id) - return unless project.is_a?(Project) && project.valid_repo? - - _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } - - record - end - - def referenced_merge_request_commit_shas - return [] unless noteable.is_a?(MergeRequest) - - @referenced_merge_request_commit_shas ||= begin - referenced_shas = references_per_parent.values.reduce(:|).to_a - noteable.all_commit_shas.select do |sha| - referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } - end - end - end - - # The default behaviour is `#to_i` - we just pass the hash through. - def self.parse_symbol(sha_hash, _match) - sha_hash - end - - def url_for_object(commit, project) - h = Gitlab::Routing.url_helpers - - if referenced_merge_request_commit_shas.include?(commit.id) - h.diffs_project_merge_request_url(project, - noteable, - commit_id: commit.id, - only_path: only_path?) - else - h.project_commit_url(project, - commit, - only_path: only_path?) - end - end - - def object_link_text_extras(object, matches) - extras = super - - path = matches[:path] if matches.names.include?("path") - if path == '/builds' - extras.unshift "builds" - end - - extras - end - - private - - def parent_records(parent, ids) - parent.commits_by(oids: ids.to_a) - end - - def noteable - context[:noteable] - end - - def only_path? - context[:only_path] - end - end - end -end diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb index 5288db3b0cb..a615abc1989 100644 --- a/lib/banzai/filter/commit_trailers_filter.rb +++ b/lib/banzai/filter/commit_trailers_filter.rb @@ -36,7 +36,7 @@ module Banzai next if html == content - node.replace(html) + node.replace("\n\n#{html}") end doc diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb deleted file mode 100644 index 1754fec93d4..00000000000 --- a/lib/banzai/filter/design_reference_filter.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class DesignReferenceFilter < AbstractReferenceFilter - class Identifier - include Comparable - attr_reader :issue_iid, :filename - - def initialize(issue_iid:, filename:) - @issue_iid = issue_iid - @filename = filename - end - - def as_composite_id(id_for_iid) - id = id_for_iid[issue_iid] - return unless id - - { issue_id: id, filename: filename } - end - - def <=>(other) - return unless other.is_a?(Identifier) - - [issue_iid, filename] <=> [other.issue_iid, other.filename] - end - alias_method :eql?, :== - - def hash - [issue_iid, filename].hash - end - end - - self.reference_type = :design - - def find_object(project, identifier) - records_per_parent[project][identifier] - end - - def parent_records(project, identifiers) - return [] unless project.design_management_enabled? - - iids = identifiers.map(&:issue_iid).to_set - issues = project.issues.where(iid: iids) - id_for_iid = issues.index_by(&:iid).transform_values(&:id) - issue_by_id = issues.index_by(&:id) - - designs(identifiers, id_for_iid).each do |d| - issue = issue_by_id[d.issue_id] - # optimisation: assign values we have already fetched - d.project = project - d.issue = issue - end - end - - def relation_for_paths(paths) - super.includes(:route, :namespace, :group) - end - - def parent_type - :project - end - - # optimisation to reuse the parent_per_reference query information - def parent_from_ref(ref) - parent_per_reference[ref || current_parent_path] - end - - def url_for_object(design, project) - path_options = { vueroute: design.filename } - Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options) - end - - def data_attributes_for(_text, _project, design, **_kwargs) - super.merge(issue: design.issue_id) - end - - def self.object_class - ::DesignManagement::Design - end - - def self.object_sym - :design - end - - def self.parse_symbol(raw, match_data) - filename = match_data[:url_filename] - iid = match_data[:issue].to_i - Identifier.new(filename: CGI.unescape(filename), issue_iid: iid) - end - - def record_identifier(design) - Identifier.new(filename: design.filename, issue_iid: design.issue.iid) - end - - private - - def designs(identifiers, id_for_iid) - identifiers - .map { |identifier| identifier.as_composite_id(id_for_iid) } - .compact - .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch - .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) } - end - end - end -end diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb deleted file mode 100644 index 70a6cb0a6dc..00000000000 --- a/lib/banzai/filter/epic_reference_filter.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # The actual filter is implemented in the EE mixin - class EpicReferenceFilter < IssuableReferenceFilter - self.reference_type = :epic - - def self.object_class - Epic - end - - private - - def group - context[:group] || context[:project]&.group - end - end - end -end - -Banzai::Filter::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::EpicReferenceFilter') diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb deleted file mode 100644 index fcf4863ab4f..00000000000 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces external issue tracker references with links. - # References are ignored if the project doesn't use an external issue - # tracker. - # - # This filter does not support cross-project references. - class ExternalIssueReferenceFilter < ReferenceFilter - self.reference_type = :external_issue - - # Public: Find `JIRA-123` issue references in text - # - # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| - # "<a href=...>##{issue}</a>" - # end - # - # text - String text to search. - # - # Yields the String match and the String issue reference. - # - # Returns a String replaced with the return of the block. - def self.references_in(text, pattern) - text.gsub(pattern) do |match| - yield match, $~[:issue] - end - end - - def call - # Early return if the project isn't using an external tracker - return doc if project.nil? || default_issues_tracker? - - ref_pattern = issue_reference_pattern - ref_start_pattern = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - issue_link_filter(content) - end - - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_start_pattern - replace_link_node_with_href(node, index, link) do - issue_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc - end - - private - - # Replace `JIRA-123` issue references in text with links to the referenced - # issue's details page. - # - # text - String text to replace references in. - # link_content - Original content of the link being replaced. - # - # Returns a String with `JIRA-123` references replaced with links. All - # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text, link_content: nil) - self.class.references_in(text, issue_reference_pattern) do |match, id| - url = url_for_issue(id) - klass = reference_class(:issue) - data = data_attribute(project: project.id, external_issue: id) - content = link_content || match - - %(<a href="#{url}" #{data} - title="#{escape_once(issue_title)}" - class="#{klass}">#{content}</a>) - end - end - - def url_for_issue(issue_id) - return '' if project.nil? - - url = if only_path? - project.external_issue_tracker.issue_path(issue_id) - else - project.external_issue_tracker.issue_url(issue_id) - end - - # Ensure we return a valid URL to prevent possible XSS. - URI.parse(url).to_s - rescue URI::InvalidURIError - '' - end - - def default_issues_tracker? - external_issues_cached(:default_issues_tracker?) - end - - def issue_reference_pattern - external_issues_cached(:external_issue_reference_pattern) - end - - def project - context[:project] - end - - def issue_title - "Issue in #{project.external_issue_tracker.title}" - end - - def external_issues_cached(attribute) - cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } - cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend - cached_attributes[project.id][attribute] - end - end - end -end diff --git a/lib/banzai/filter/feature_flag_reference_filter.rb b/lib/banzai/filter/feature_flag_reference_filter.rb deleted file mode 100644 index c11576901ce..00000000000 --- a/lib/banzai/filter/feature_flag_reference_filter.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class FeatureFlagReferenceFilter < IssuableReferenceFilter - self.reference_type = :feature_flag - - def self.object_class - Operations::FeatureFlag - end - - def self.object_sym - :feature_flag - end - - def parent_records(parent, ids) - parent.operations_feature_flags.where(iid: ids.to_a) - end - - def url_for_object(feature_flag, project) - ::Gitlab::Routing.url_helpers.edit_project_feature_flag_url( - project, - feature_flag.iid, - only_path: context[:only_path] - ) - end - - def object_link_title(object, matches) - object.name - end - end - end -end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 8a7d3c49ffb..6de9f2b86f6 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -98,14 +98,15 @@ module Banzai return unless image?(content) - if url?(content) - path = content - elsif file = wiki.find_file(content, load_content: false) - path = ::File.join(wiki_base_path, file.path) - end + path = + if url?(content) + content + elsif file = wiki.find_file(content, load_content: false) + file.path + end if path - content_tag(:img, nil, data: { src: path }, class: 'gfm') + content_tag(:img, nil, src: path, class: 'gfm') end end diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb deleted file mode 100644 index b91ba9f7256..00000000000 --- a/lib/banzai/filter/issuable_reference_filter.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class IssuableReferenceFilter < AbstractReferenceFilter - def record_identifier(record) - record.iid.to_i - end - - def find_object(parent, iid) - records_per_parent[parent][iid] - end - - def parent_from_ref(ref) - parent_per_reference[ref || current_parent_path] - end - end - end -end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb deleted file mode 100644 index 216418ee5fa..00000000000 --- a/lib/banzai/filter/issue_reference_filter.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces issue references with links. References to - # issues that do not exist are ignored. - # - # This filter supports cross-project references. - # - # When external issues tracker like Jira is activated we should not - # use issue reference pattern, but we should still be able - # to reference issues from other GitLab projects. - class IssueReferenceFilter < IssuableReferenceFilter - self.reference_type = :issue - - def self.object_class - Issue - end - - def url_for_object(issue, project) - return issue_path(issue, project) if only_path? - - issue_url(issue, project) - end - - def parent_records(parent, ids) - parent.issues.where(iid: ids.to_a) - end - - def object_link_text_extras(issue, matches) - super + design_link_extras(issue, matches.named_captures['path']) - end - - private - - def issue_path(issue, project) - Gitlab::Routing.url_helpers.namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: issue.iid) - end - - def issue_url(issue, project) - Gitlab::Routing.url_helpers.namespace_project_issue_url(namespace_id: project.namespace, project_id: project, id: issue.iid) - end - - def design_link_extras(issue, path) - if path == '/designs' && read_designs?(issue) - ['designs'] - else - [] - end - end - - def read_designs?(issue) - issue.project.design_management_enabled? - end - end - end -end diff --git a/lib/banzai/filter/iteration_reference_filter.rb b/lib/banzai/filter/iteration_reference_filter.rb deleted file mode 100644 index 9d2b533e6da..00000000000 --- a/lib/banzai/filter/iteration_reference_filter.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # The actual filter is implemented in the EE mixin - class IterationReferenceFilter < AbstractReferenceFilter - self.reference_type = :iteration - - def self.object_class - Iteration - end - end - end -end - -Banzai::Filter::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::IterationReferenceFilter') diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb deleted file mode 100644 index a4d3e352051..00000000000 --- a/lib/banzai/filter/label_reference_filter.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces label references with links. - class LabelReferenceFilter < AbstractReferenceFilter - self.reference_type = :label - - def self.object_class - Label - end - - def find_object(parent_object, id) - find_labels(parent_object).find(id) - end - - def references_in(text, pattern = Label.reference_pattern) - labels = {} - unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| - namespace, project = $~[:namespace], $~[:project] - project_path = full_project_path(namespace, project) - label = find_label_cached(project_path, $~[:label_id], $~[:label_name]) - - if label - labels[label.id] = yield match, label.id, project, namespace, $~ - "#{REFERENCE_PLACEHOLDER}#{label.id}" - else - match - end - end - - return text if labels.empty? - - escape_with_placeholders(unescaped_html, labels) - end - - def find_label_cached(parent_ref, label_id, label_name) - cached_call(:banzai_find_label_cached, label_name&.tr('"', '') || label_id, path: [object_class, parent_ref]) do - find_label(parent_ref, label_id, label_name) - end - end - - def find_label(parent_ref, label_id, label_name) - parent = parent_from_ref(parent_ref) - return unless parent - - label_params = label_params(label_id, label_name) - find_labels(parent).find_by(label_params) - end - - def find_labels(parent) - params = if parent.is_a?(Group) - { group_id: parent.id, - include_ancestor_groups: true, - only_group_labels: true } - else - { project: parent, - include_ancestor_groups: true } - end - - LabelsFinder.new(nil, params).execute(skip_authorization: true) - end - - # Parameters to pass to `Label.find_by` based on the given arguments - # - # id - Integer ID to pass. If present, returns {id: id} - # name - String name to pass. If `id` is absent, finds by name without - # surrounding quotes. - # - # Returns a Hash. - def label_params(id, name) - if name - { name: name.tr('"', '') } - else - { id: id.to_i } - end - end - - def url_for_object(label, parent) - label_url_method = - if context[:label_url_method] - context[:label_url_method] - elsif parent.is_a?(Project) - :project_issues_url - end - - return unless label_url_method - - Gitlab::Routing.url_helpers.public_send(label_url_method, parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend - end - - def object_link_text(object, matches) - label_suffix = '' - parent = project || group - - if project || full_path_ref?(matches) - project_path = full_project_path(matches[:namespace], matches[:project]) - parent_from_ref = from_ref_cached(project_path) - reference = parent_from_ref.to_human_reference(parent) - - label_suffix = " <i>in #{ERB::Util.html_escape(reference)}</i>" if reference.present? - end - - presenter = object.present(issuable_subject: parent) - LabelsHelper.render_colored_label(presenter, suffix: label_suffix) - end - - def wrap_link(link, label) - presenter = label.present(issuable_subject: project || group) - LabelsHelper.wrap_label_html(link, small: true, label: presenter) - end - - def full_path_ref?(matches) - matches[:namespace] && matches[:project] - end - - def reference_class(type, tooltip: true) - super + ' gl-link gl-label-link' - end - - def object_link_title(object, matches) - presenter = object.present(issuable_subject: project || group) - LabelsHelper.label_tooltip_title(presenter) - end - end - end -end - -Banzai::Filter::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::LabelReferenceFilter') diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index c915f0ee35b..2247984b86d 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -39,7 +39,7 @@ module Banzai end end - doc.css('pre.code.math').each do |el| + doc.css('pre.code.language-math').each do |el| el[STYLE_ATTRIBUTE] = 'display' el[:class] += " #{TAG_CLASS}" end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb deleted file mode 100644 index 0b8bd17a71b..00000000000 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces merge request references with links. References - # to merge requests that do not exist are ignored. - # - # This filter supports cross-project references. - class MergeRequestReferenceFilter < IssuableReferenceFilter - self.reference_type = :merge_request - - def self.object_class - MergeRequest - end - - def url_for_object(mr, project) - h = Gitlab::Routing.url_helpers - h.project_merge_request_url(project, mr, - only_path: context[:only_path]) - end - - def object_link_title(object, matches) - # The method will return `nil` if object is not a commit - # allowing for properly handling the extended MR Tooltip - object_link_commit_title(object, matches) - end - - def object_link_text_extras(object, matches) - extras = super - - if commit_ref = object_link_commit_ref(object, matches) - klass = reference_class(:commit, tooltip: false) - commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>) - - return extras.unshift(commit_ref_tag) - end - - path = matches[:path] if matches.names.include?("path") - - case path - when '/diffs' - extras.unshift "diffs" - when '/commits' - extras.unshift "commits" - when '/builds' - extras.unshift "builds" - end - - extras - end - - def parent_records(parent, ids) - parent.merge_requests - .where(iid: ids.to_a) - .includes(target_project: :namespace) - end - - def reference_class(object_sym, options = {}) - super(object_sym, tooltip: false) - end - - def data_attributes_for(text, parent, object, **data) - super.merge(project_path: parent.full_path, iid: object.iid, mr_title: object.title) - end - - private - - def object_link_commit_title(object, matches) - object_link_commit(object, matches)&.title - end - - def object_link_commit_ref(object, matches) - object_link_commit(object, matches)&.short_id - end - - def object_link_commit(object, matches) - return unless matches.names.include?('query') && query = matches[:query] - - # Removes leading "?". CGI.parse expects "arg1&arg2&arg3" - params = CGI.parse(query.sub(/^\?/, '')) - - return unless commit_sha = params['commit_id']&.first - - if commit = find_commit_by_sha(object, commit_sha) - Commit.from_hash(commit.to_hash, object.project) - end - end - - def find_commit_by_sha(object, commit_sha) - @all_commits ||= {} - @all_commits[object.id] ||= object.all_commits - - @all_commits[object.id].find { |commit| commit.sha == commit_sha } - end - end - end -end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb deleted file mode 100644 index 126208db935..00000000000 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ /dev/null @@ -1,138 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces milestone references with links. - class MilestoneReferenceFilter < AbstractReferenceFilter - include Gitlab::Utils::StrongMemoize - - self.reference_type = :milestone - - def self.object_class - Milestone - end - - # Links to project milestones contain the IID, but when we're handling - # 'regular' references, we need to use the global ID to disambiguate - # between group and project milestones. - def find_object(parent, id) - return unless valid_context?(parent) - - find_milestone_with_finder(parent, id: id) - end - - def find_object_from_link(parent, iid) - return unless valid_context?(parent) - - find_milestone_with_finder(parent, iid: iid) - end - - def valid_context?(parent) - strong_memoize(:valid_context) do - group_context?(parent) || project_context?(parent) - end - end - - def group_context?(parent) - strong_memoize(:group_context) do - parent.is_a?(Group) - end - end - - def project_context?(parent) - strong_memoize(:project_context) do - parent.is_a?(Project) - end - end - - def references_in(text, pattern = Milestone.reference_pattern) - # We'll handle here the references that follow the `reference_pattern`. - # Other patterns (for example, the link pattern) are handled by the - # default implementation. - return super(text, pattern) if pattern != Milestone.reference_pattern - - milestones = {} - unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| - milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) - - if milestone - milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~ - "#{REFERENCE_PLACEHOLDER}#{milestone.id}" - else - match - end - end - - return text if milestones.empty? - - escape_with_placeholders(unescaped_html, milestones) - end - - def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) - project_path = full_project_path(namespace_ref, project_ref) - - # Returns group if project is not found by path - parent = parent_from_ref(project_path) - - return unless parent - - milestone_params = milestone_params(milestone_id, milestone_name) - - find_milestone_with_finder(parent, milestone_params) - end - - def milestone_params(iid, name) - if name - { name: name.tr('"', '') } - else - { iid: iid.to_i } - end - end - - def find_milestone_with_finder(parent, params) - finder_params = milestone_finder_params(parent, params[:iid].present?) - - MilestonesFinder.new(finder_params).find_by(params) - end - - def milestone_finder_params(parent, find_by_iid) - { order: nil, state: 'all' }.tap do |params| - params[:project_ids] = parent.id if project_context?(parent) - - # We don't support IID lookups because IIDs can clash between - # group/project milestones and group/subgroup milestones. - params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid - end - end - - def self_and_ancestors_ids(parent) - if group_context?(parent) - parent.self_and_ancestors.select(:id) - elsif project_context?(parent) - parent.group&.self_and_ancestors&.select(:id) - end - end - - def url_for_object(milestone, project) - Gitlab::Routing - .url_helpers - .milestone_url(milestone, only_path: context[:only_path]) - end - - def object_link_text(object, matches) - milestone_link = escape_once(super) - reference = object.project&.to_reference_base(project) - - if reference.present? - "#{milestone_link} <i>in #{reference}</i>".html_safe - else - milestone_link - end - end - - def object_link_title(object, matches) - nil - end - end - end -end diff --git a/lib/banzai/filter/project_reference_filter.rb b/lib/banzai/filter/project_reference_filter.rb deleted file mode 100644 index 50e23460cb8..00000000000 --- a/lib/banzai/filter/project_reference_filter.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces project references with links. - class ProjectReferenceFilter < ReferenceFilter - self.reference_type = :project - - # Public: Find `namespace/project>` project references in text - # - # ProjectReferenceFilter.references_in(text) do |match, project| - # "<a href=...>#{project}></a>" - # end - # - # text - String text to search. - # - # Yields the String match, and the String project name. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(Project.markdown_reference_pattern) do |match| - yield match, "#{$~[:namespace]}/#{$~[:project]}" - end - end - - def call - ref_pattern = Project.markdown_reference_pattern - ref_pattern_start = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - project_link_filter(content) - end - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_pattern_start - replace_link_node_with_href(node, index, link) do - project_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc - end - - # Replace `namespace/project>` project references in text with links to the referenced - # project page. - # - # text - String text to replace references in. - # link_content - Original content of the link being replaced. - # - # Returns a String with `namespace/project>` references replaced with links. All links - # have `gfm` and `gfm-project` class names attached for styling. - def project_link_filter(text, link_content: nil) - self.class.references_in(text) do |match, project_path| - cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do - if project = projects_hash[project_path.downcase] - link_to_project(project, link_content: link_content) || match - else - match - end - end - end - end - - # Returns a Hash containing all Project objects for the project - # references in the current document. - # - # The keys of this Hash are the project paths, the values the - # corresponding Project objects. - def projects_hash - @projects ||= Project.eager_load(:route, namespace: [:route]) - .where_full_path_in(projects) - .index_by(&:full_path) - .transform_keys(&:downcase) - end - - # Returns all projects referenced in the current document. - def projects - refs = Set.new - - nodes.each do |node| - node.to_html.scan(Project.markdown_reference_pattern) do - refs << "#{$~[:namespace]}/#{$~[:project]}" - end - end - - refs.to_a - end - - private - - def urls - Gitlab::Routing.url_helpers - end - - def link_class - reference_class(:project) - end - - def link_to_project(project, link_content: nil) - url = urls.project_url(project, only_path: context[:only_path]) - data = data_attribute(project: project.id) - content = link_content || project.to_reference - - link_tag(url, data, content, project.name) - end - - def link_tag(url, data, link_content, title) - %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) - end - end - end -end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb deleted file mode 100644 index d22a0e0b504..00000000000 --- a/lib/banzai/filter/reference_filter.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: true - -# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js -module Banzai - module Filter - # Base class for GitLab Flavored Markdown reference filters. - # - # References within <pre>, <code>, <a>, and <style> elements are ignored. - # - # Context options: - # :project (required) - Current project, ignored if reference is cross-project. - # :only_path - Generate path-only links. - class ReferenceFilter < HTML::Pipeline::Filter - include RequestStoreReferenceCache - include OutputSafety - - class << self - attr_accessor :reference_type - - def call(doc, context = nil, result = nil) - new(doc, context, result).call_and_update_nodes - end - end - - def initialize(doc, context = nil, result = nil) - super - - @new_nodes = {} - @nodes = self.result[:reference_filter_nodes] - end - - def call_and_update_nodes - with_update_nodes { call } - end - - # Returns a data attribute String to attach to a reference link - # - # attributes - Hash, where the key becomes the data attribute name and the - # value is the data attribute value - # - # Examples: - # - # data_attribute(project: 1, issue: 2) - # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" - # - # data_attribute(project: 3, merge_request: 4) - # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" - # - # Returns a String - def data_attribute(attributes = {}) - attributes = attributes.reject { |_, v| v.nil? } - - attributes[:reference_type] ||= self.class.reference_type - attributes[:container] ||= 'body' - attributes[:placement] ||= 'top' - attributes.delete(:original) if context[:no_original_data] - attributes.map do |key, value| - %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") - end.join(' ') - end - - def ignore_ancestor_query - @ignore_ancestor_query ||= begin - parents = %w(pre code a style) - parents << 'blockquote' if context[:ignore_blockquotes] - - parents.map { |n| "ancestor::#{n}" }.join(' or ') - end - end - - def project - context[:project] - end - - def group - context[:group] - end - - def user - context[:user] - end - - def skip_project_check? - context[:skip_project_check] - end - - def reference_class(type, tooltip: true) - gfm_klass = "gfm gfm-#{type}" - - return gfm_klass unless tooltip - - "#{gfm_klass} has-tooltip" - end - - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project unless skip_project_check? - end - - # Iterates over all <a> and text() nodes in a document. - # - # Nodes are skipped whenever their ancestor is one of the nodes returned - # by `ignore_ancestor_query`. Link tags are not processed if they have a - # "gfm" class or the "href" attribute is empty. - def each_node - return to_enum(__method__) unless block_given? - - doc.xpath(query).each do |node| - yield node - end - end - - # Returns an Array containing all HTML nodes. - def nodes - @nodes ||= each_node.to_a - end - - # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. - def yield_valid_link(node) - link = unescape_link(node.attr('href').to_s) - inner_html = node.inner_html - - return unless link.force_encoding('UTF-8').valid_encoding? - - yield link, inner_html - end - - def unescape_link(href) - CGI.unescape(href) - end - - def replace_text_when_pattern_matches(node, index, pattern) - return unless node.text =~ pattern - - content = node.to_html - html = yield content - - replace_text_with_html(node, index, html) unless html == content - end - - def replace_link_node_with_text(node, index) - html = yield - - replace_text_with_html(node, index, html) unless html == node.text - end - - def replace_link_node_with_href(node, index, link) - html = yield - - replace_text_with_html(node, index, html) unless html == link - end - - def text_node?(node) - node.is_a?(Nokogiri::XML::Text) - end - - def element_node?(node) - node.is_a?(Nokogiri::XML::Element) - end - - private - - def query - @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] - | descendant-or-self::a[ - not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") - ]} - end - - def replace_text_with_html(node, index, html) - replace_and_update_new_nodes(node, index, html) - end - - def replace_and_update_new_nodes(node, index, html) - previous_node = node.previous - next_node = node.next - parent_node = node.parent - # Unfortunately node.replace(html) returns re-parented nodes, not the actual replaced nodes in the doc - # We need to find the actual nodes in the doc that were replaced - node.replace(html) - @new_nodes[index] = [] - - # We replaced node with new nodes, so we find first new node. If previous_node is nil, we take first parent child - new_node = previous_node ? previous_node.next : parent_node&.children&.first - - # We iterate from first to last replaced node and store replaced nodes in @new_nodes - while new_node && new_node != next_node - @new_nodes[index] << new_node.xpath(query) - new_node = new_node.next - end - - @new_nodes[index].flatten! - end - - def only_path? - context[:only_path] - end - - def with_update_nodes - @new_nodes = {} - yield.tap { update_nodes! } - end - - # Once Filter completes replacing nodes, we update nodes with @new_nodes - def update_nodes! - @new_nodes.sort_by { |index, _new_nodes| -index }.each do |index, new_nodes| - nodes[index, 1] = new_nodes - end - result[:reference_filter_nodes] = nodes - end - end - end -end diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb new file mode 100644 index 00000000000..7109373dbce --- /dev/null +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -0,0 +1,448 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # Issues, merge requests, Snippets, Commits and Commit Ranges share + # similar functionality in reference filtering. + class AbstractReferenceFilter < ReferenceFilter + include CrossProjectReference + + # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found + # reference (which we replace with placeholder during re-scaping). The + # random number helps ensure it's pretty close to unique. Since it's a + # transitory value (it never gets saved) we can initialize once, and it + # doesn't matter if it changes on a restart. + REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_" + REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze + + def self.object_class + # Implement in child class + # Example: MergeRequest + end + + def self.object_name + @object_name ||= object_class.name.underscore + end + + def self.object_sym + @object_sym ||= object_name.to_sym + end + + # Public: Find references in text (like `!123` for merge requests) + # + # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| + # object = find_object(project_ref, id) + # "<a href=...>#{object.to_reference}</a>" + # end + # + # text - String text to search. + # + # Yields the String match, the Integer referenced object ID, an optional String + # of the external project reference, and all of the matchdata. + # + # Returns a String replaced with the return of the block. + def self.references_in(text, pattern = object_class.reference_pattern) + text.gsub(pattern) do |match| + if ident = identifier($~) + yield match, ident, $~[:project], $~[:namespace], $~ + else + match + end + end + end + + def self.identifier(match_data) + symbol = symbol_from_match(match_data) + + parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol) + end + + def identifier(match_data) + self.class.identifier(match_data) + end + + def self.symbol_from_match(match) + key = object_sym + match[key] if match.names.include?(key.to_s) + end + + # Transform a symbol extracted from the text to a meaningful value + # In most cases these will be integers, so we call #to_i by default + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `parse_symbol(ref) == record_identifier(record)`. + def self.parse_symbol(symbol, match_data) + symbol.to_i + end + + # We assume that most classes are identifying records by ID. + # + # This method has the contract that if a string `ref` refers to a + # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`. + def record_identifier(record) + record.id + end + + def object_class + self.class.object_class + end + + def object_sym + self.class.object_sym + end + + def references_in(*args, &block) + self.class.references_in(*args, &block) + end + + # Implement in child class + # Example: project.merge_requests.find + def find_object(parent_object, id) + end + + # Override if the link reference pattern produces a different ID (global + # ID vs internal ID, for instance) to the regular reference pattern. + def find_object_from_link(parent_object, id) + find_object(parent_object, id) + end + + # Implement in child class + # Example: project_merge_request_url + def url_for_object(object, parent_object) + end + + def find_object_cached(parent_object, id) + cached_call(:banzai_find_object, id, path: [object_class, parent_object.id]) do + find_object(parent_object, id) + end + end + + def find_object_from_link_cached(parent_object, id) + cached_call(:banzai_find_object_from_link, id, path: [object_class, parent_object.id]) do + find_object_from_link(parent_object, id) + end + end + + def from_ref_cached(ref) + cached_call("banzai_#{parent_type}_refs".to_sym, ref) do + parent_from_ref(ref) + end + end + + def url_for_object_cached(object, parent_object) + cached_call(:banzai_url_for_object, object, path: [object_class, parent_object.id]) do + url_for_object(object, parent_object) + end + end + + def call + return doc unless project || group || user + + ref_pattern = object_class.reference_pattern + link_pattern = object_class.link_reference_pattern + + # Compile often used regexps only once outside of the loop + ref_pattern_anchor = /\A#{ref_pattern}\z/ + link_pattern_start = /\A#{link_pattern}/ + link_pattern_anchor = /\A#{link_pattern}\z/ + + nodes.each_with_index do |node, index| + if text_node?(node) && ref_pattern + replace_text_when_pattern_matches(node, index, ref_pattern) do |content| + object_link_filter(content, ref_pattern) + end + + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if ref_pattern && link =~ ref_pattern_anchor + replace_link_node_with_href(node, index, link) do + object_link_filter(link, ref_pattern, link_content: inner_html) + end + + next + end + + next unless link_pattern + + if link == inner_html && inner_html =~ link_pattern_start + replace_link_node_with_text(node, index) do + object_link_filter(inner_html, link_pattern, link_reference: true) + end + + next + end + + if link =~ link_pattern_anchor + replace_link_node_with_href(node, index, link) do + object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true) + end + + next + end + end + end + end + + doc + end + + # Replace references (like `!123` for merge requests) in text with links + # to the referenced object's details page. + # + # text - String text to replace references in. + # pattern - Reference pattern to match against. + # link_content - Original content of the link being replaced. + # link_reference - True if this was using the link reference pattern, + # false otherwise. + # + # Returns a String with references replaced with links. All links + # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. + def object_link_filter(text, pattern, link_content: nil, link_reference: false) + references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| + parent_path = if parent_type == :group + full_group_path(namespace_ref) + else + full_project_path(namespace_ref, project_ref) + end + + parent = from_ref_cached(parent_path) + + if parent + object = + if link_reference + find_object_from_link_cached(parent, id) + else + find_object_cached(parent, id) + end + end + + if object + title = object_link_title(object, matches) + klass = reference_class(object_sym) + + data_attributes = data_attributes_for(link_content || match, parent, object, + link_content: !!link_content, + link_reference: link_reference) + data = data_attribute(data_attributes) + + url = + if matches.names.include?("url") && matches[:url] + matches[:url] + else + url_for_object_cached(object, parent) + end + + content = link_content || object_link_text(object, matches) + + link = %(<a href="#{url}" #{data} + title="#{escape_once(title)}" + class="#{klass}">#{content}</a>) + + wrap_link(link, object) + else + match + end + end + end + + def wrap_link(link, object) + link + end + + def data_attributes_for(text, parent, object, link_content: false, link_reference: false) + object_parent_type = parent.is_a?(Group) ? :group : :project + + { + original: escape_html_entities(text), + link: link_content, + link_reference: link_reference, + object_parent_type => parent.id, + object_sym => object.id + } + end + + def object_link_text_extras(object, matches) + extras = [] + + if matches.names.include?("anchor") && matches[:anchor] && matches[:anchor] =~ /\A\#note_(\d+)\z/ + extras << "comment #{Regexp.last_match(1)}" + end + + extension = matches[:extension] if matches.names.include?("extension") + + extras << extension if extension + + extras + end + + def object_link_title(object, matches) + object.title + end + + def object_link_text(object, matches) + parent = project || group || user + text = object.reference_link_text(parent) + + extras = object_link_text_extras(object, matches) + text += " (#{extras.join(", ")})" if extras.any? + + text + end + + # Returns a Hash containing all object references (e.g. issue IDs) per the + # project they belong to. + def references_per_parent + @references_per ||= {} + + @references_per[parent_type] ||= begin + refs = Hash.new { |hash, key| hash[key] = Set.new } + regex = [ + object_class.link_reference_pattern, + object_class.reference_pattern + ].compact.reduce { |a, b| Regexp.union(a, b) } + + nodes.each do |node| + node.to_html.scan(regex) do + path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + + if ident = identifier($~) + refs[path] << ident + end + end + end + + refs + end + end + + # Returns a Hash containing referenced projects grouped per their full + # path. + def parent_per_reference + @per_reference ||= {} + + @per_reference[parent_type] ||= begin + refs = Set.new + + references_per_parent.each do |ref, _| + refs << ref + end + + find_for_paths(refs.to_a).index_by(&:full_path) + end + end + + def relation_for_paths(paths) + klass = parent_type.to_s.camelize.constantize + result = klass.where_full_path_in(paths) + return result if parent_type == :group + + result.includes(:namespace) if parent_type == :project + end + + # Returns projects for the given paths. + def find_for_paths(paths) + if Gitlab::SafeRequestStore.active? + cache = refs_cache + to_query = paths - cache.keys + + unless to_query.empty? + records = relation_for_paths(to_query) + + found = [] + records.each do |record| + ref = record.full_path + get_or_set_cache(cache, ref) { record } + found << ref + end + + not_found = to_query - found + not_found.each do |ref| + get_or_set_cache(cache, ref) { nil } + end + end + + cache.slice(*paths).values.compact + else + relation_for_paths(paths) + end + end + + def current_parent_path + @current_parent_path ||= parent&.full_path + end + + def current_project_namespace_path + @current_project_namespace_path ||= project&.namespace&.full_path + end + + def records_per_parent + @_records_per_project ||= {} + + @_records_per_project[object_class.to_s.underscore] ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + parent_per_reference.each do |path, parent| + record_ids = references_per_parent[path] + + parent_records(parent, record_ids).each do |record| + hash[parent][record_identifier(record)] = record + end + end + + hash + end + end + + private + + def full_project_path(namespace, project_ref) + return current_parent_path unless project_ref + + namespace_ref = namespace || current_project_namespace_path + "#{namespace_ref}/#{project_ref}" + end + + def refs_cache + Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} + end + + def parent_type + :project + end + + def parent + parent_type == :project ? project : group + end + + def full_group_path(group_ref) + return current_parent_path unless group_ref + + group_ref + end + + def unescape_html_entities(text) + CGI.unescapeHTML(text.to_s) + end + + def escape_html_entities(text) + CGI.escapeHTML(text.to_s) + end + + def escape_with_placeholders(text, placeholder_data) + escaped = escape_html_entities(text) + + escaped.gsub(REFERENCE_PLACEHOLDER_PATTERN) do |match| + placeholder_data[Regexp.last_match(1).to_i] + end + end + end + end + end +end + +Banzai::Filter::References::AbstractReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::AbstractReferenceFilter') diff --git a/lib/banzai/filter/references/alert_reference_filter.rb b/lib/banzai/filter/references/alert_reference_filter.rb new file mode 100644 index 00000000000..90fef536605 --- /dev/null +++ b/lib/banzai/filter/references/alert_reference_filter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + class AlertReferenceFilter < IssuableReferenceFilter + self.reference_type = :alert + + def self.object_class + AlertManagement::Alert + end + + def self.object_sym + :alert + end + + def parent_records(parent, ids) + parent.alert_management_alerts.where(iid: ids.to_a) + end + + def url_for_object(alert, project) + ::Gitlab::Routing.url_helpers.details_project_alert_management_url( + project, + alert.iid, + only_path: context[:only_path] + ) + end + end + end + end +end diff --git a/lib/banzai/filter/references/commit_range_reference_filter.rb b/lib/banzai/filter/references/commit_range_reference_filter.rb new file mode 100644 index 00000000000..ad79f8a173c --- /dev/null +++ b/lib/banzai/filter/references/commit_range_reference_filter.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces commit range references with links. + # + # This filter supports cross-project references. + class CommitRangeReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit_range + + def self.object_class + CommitRange + end + + def self.references_in(text, pattern = CommitRange.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ + end + end + + def initialize(*args) + super + + @commit_map = {} + end + + def find_object(project, id) + return unless project.is_a?(Project) + + range = CommitRange.new(id, project) + + range.valid_commits? ? range : nil + end + + def url_for_object(range, project) + h = Gitlab::Routing.url_helpers + h.project_compare_url(project, + range.to_param.merge(only_path: context[:only_path])) + end + + def object_link_title(range, matches) + nil + end + end + end + end +end diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb new file mode 100644 index 00000000000..457921bd07d --- /dev/null +++ b/lib/banzai/filter/references/commit_reference_filter.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces commit references with links. + # + # This filter supports cross-project references. + class CommitReferenceFilter < AbstractReferenceFilter + self.reference_type = :commit + + def self.object_class + Commit + end + + def self.references_in(text, pattern = Commit.reference_pattern) + text.gsub(pattern) do |match| + yield match, $~[:commit], $~[:project], $~[:namespace], $~ + end + end + + def find_object(project, id) + return unless project.is_a?(Project) && project.valid_repo? + + _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + + record + end + + def referenced_merge_request_commit_shas + return [] unless noteable.is_a?(MergeRequest) + + @referenced_merge_request_commit_shas ||= begin + referenced_shas = references_per_parent.values.reduce(:|).to_a + noteable.all_commit_shas.select do |sha| + referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) } + end + end + end + + # The default behaviour is `#to_i` - we just pass the hash through. + def self.parse_symbol(sha_hash, _match) + sha_hash + end + + def url_for_object(commit, project) + h = Gitlab::Routing.url_helpers + + if referenced_merge_request_commit_shas.include?(commit.id) + h.diffs_project_merge_request_url(project, + noteable, + commit_id: commit.id, + only_path: only_path?) + else + h.project_commit_url(project, + commit, + only_path: only_path?) + end + end + + def object_link_text_extras(object, matches) + extras = super + + path = matches[:path] if matches.names.include?("path") + if path == '/builds' + extras.unshift "builds" + end + + extras + end + + private + + def parent_records(parent, ids) + parent.commits_by(oids: ids.to_a) + end + + def noteable + context[:noteable] + end + + def only_path? + context[:only_path] + end + end + end + end +end diff --git a/lib/banzai/filter/references/design_reference_filter.rb b/lib/banzai/filter/references/design_reference_filter.rb new file mode 100644 index 00000000000..61234e61c15 --- /dev/null +++ b/lib/banzai/filter/references/design_reference_filter.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + class DesignReferenceFilter < AbstractReferenceFilter + class Identifier + include Comparable + attr_reader :issue_iid, :filename + + def initialize(issue_iid:, filename:) + @issue_iid = issue_iid + @filename = filename + end + + def as_composite_id(id_for_iid) + id = id_for_iid[issue_iid] + return unless id + + { issue_id: id, filename: filename } + end + + def <=>(other) + return unless other.is_a?(Identifier) + + [issue_iid, filename] <=> [other.issue_iid, other.filename] + end + alias_method :eql?, :== + + def hash + [issue_iid, filename].hash + end + end + + self.reference_type = :design + + def find_object(project, identifier) + records_per_parent[project][identifier] + end + + def parent_records(project, identifiers) + return [] unless project.design_management_enabled? + + iids = identifiers.map(&:issue_iid).to_set + issues = project.issues.where(iid: iids) + id_for_iid = issues.index_by(&:iid).transform_values(&:id) + issue_by_id = issues.index_by(&:id) + + designs(identifiers, id_for_iid).each do |d| + issue = issue_by_id[d.issue_id] + # optimisation: assign values we have already fetched + d.project = project + d.issue = issue + end + end + + def relation_for_paths(paths) + super.includes(:route, :namespace, :group) + end + + def parent_type + :project + end + + # optimisation to reuse the parent_per_reference query information + def parent_from_ref(ref) + parent_per_reference[ref || current_parent_path] + end + + def url_for_object(design, project) + path_options = { vueroute: design.filename } + Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options) + end + + def data_attributes_for(_text, _project, design, **_kwargs) + super.merge(issue: design.issue_id) + end + + def self.object_class + ::DesignManagement::Design + end + + def self.object_sym + :design + end + + def self.parse_symbol(raw, match_data) + filename = match_data[:url_filename] + iid = match_data[:issue].to_i + Identifier.new(filename: CGI.unescape(filename), issue_iid: iid) + end + + def record_identifier(design) + Identifier.new(filename: design.filename, issue_iid: design.issue.iid) + end + + private + + def designs(identifiers, id_for_iid) + identifiers + .map { |identifier| identifier.as_composite_id(id_for_iid) } + .compact + .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch + .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) } + end + end + end + end +end diff --git a/lib/banzai/filter/references/epic_reference_filter.rb b/lib/banzai/filter/references/epic_reference_filter.rb new file mode 100644 index 00000000000..4ee446e5317 --- /dev/null +++ b/lib/banzai/filter/references/epic_reference_filter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # The actual filter is implemented in the EE mixin + class EpicReferenceFilter < IssuableReferenceFilter + self.reference_type = :epic + + def self.object_class + Epic + end + + private + + def group + context[:group] || context[:project]&.group + end + end + end + end +end + +Banzai::Filter::References::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::EpicReferenceFilter') diff --git a/lib/banzai/filter/references/external_issue_reference_filter.rb b/lib/banzai/filter/references/external_issue_reference_filter.rb new file mode 100644 index 00000000000..247e20967df --- /dev/null +++ b/lib/banzai/filter/references/external_issue_reference_filter.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces external issue tracker references with links. + # References are ignored if the project doesn't use an external issue + # tracker. + # + # This filter does not support cross-project references. + class ExternalIssueReferenceFilter < ReferenceFilter + self.reference_type = :external_issue + + # Public: Find `JIRA-123` issue references in text + # + # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| + # "<a href=...>##{issue}</a>" + # end + # + # text - String text to search. + # + # Yields the String match and the String issue reference. + # + # Returns a String replaced with the return of the block. + def self.references_in(text, pattern) + text.gsub(pattern) do |match| + yield match, $~[:issue] + end + end + + def call + # Early return if the project isn't using an external tracker + return doc if project.nil? || default_issues_tracker? + + ref_pattern = issue_reference_pattern + ref_start_pattern = /\A#{ref_pattern}\z/ + + nodes.each_with_index do |node, index| + if text_node?(node) + replace_text_when_pattern_matches(node, index, ref_pattern) do |content| + issue_link_filter(content) + end + + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if link =~ ref_start_pattern + replace_link_node_with_href(node, index, link) do + issue_link_filter(link, link_content: inner_html) + end + end + end + end + end + + doc + end + + private + + # Replace `JIRA-123` issue references in text with links to the referenced + # issue's details page. + # + # text - String text to replace references in. + # link_content - Original content of the link being replaced. + # + # Returns a String with `JIRA-123` references replaced with links. All + # links have `gfm` and `gfm-issue` class names attached for styling. + def issue_link_filter(text, link_content: nil) + self.class.references_in(text, issue_reference_pattern) do |match, id| + url = url_for_issue(id) + klass = reference_class(:issue) + data = data_attribute(project: project.id, external_issue: id) + content = link_content || match + + %(<a href="#{url}" #{data} + title="#{escape_once(issue_title)}" + class="#{klass}">#{content}</a>) + end + end + + def url_for_issue(issue_id) + return '' if project.nil? + + url = if only_path? + project.external_issue_tracker.issue_path(issue_id) + else + project.external_issue_tracker.issue_url(issue_id) + end + + # Ensure we return a valid URL to prevent possible XSS. + URI.parse(url).to_s + rescue URI::InvalidURIError + '' + end + + def default_issues_tracker? + external_issues_cached(:default_issues_tracker?) + end + + def issue_reference_pattern + external_issues_cached(:external_issue_reference_pattern) + end + + def project + context[:project] + end + + def issue_title + "Issue in #{project.external_issue_tracker.title}" + end + + def external_issues_cached(attribute) + cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend + cached_attributes[project.id][attribute] + end + end + end + end +end diff --git a/lib/banzai/filter/references/feature_flag_reference_filter.rb b/lib/banzai/filter/references/feature_flag_reference_filter.rb new file mode 100644 index 00000000000..be9ded1ff43 --- /dev/null +++ b/lib/banzai/filter/references/feature_flag_reference_filter.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + class FeatureFlagReferenceFilter < IssuableReferenceFilter + self.reference_type = :feature_flag + + def self.object_class + Operations::FeatureFlag + end + + def self.object_sym + :feature_flag + end + + def parent_records(parent, ids) + parent.operations_feature_flags.where(iid: ids.to_a) + end + + def url_for_object(feature_flag, project) + ::Gitlab::Routing.url_helpers.edit_project_feature_flag_url( + project, + feature_flag.iid, + only_path: context[:only_path] + ) + end + + def object_link_title(object, matches) + object.name + end + end + end + end +end diff --git a/lib/banzai/filter/references/issuable_reference_filter.rb b/lib/banzai/filter/references/issuable_reference_filter.rb new file mode 100644 index 00000000000..b8ccb926ae9 --- /dev/null +++ b/lib/banzai/filter/references/issuable_reference_filter.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + class IssuableReferenceFilter < AbstractReferenceFilter + def record_identifier(record) + record.iid.to_i + end + + def find_object(parent, iid) + records_per_parent[parent][iid] + end + + def parent_from_ref(ref) + parent_per_reference[ref || current_parent_path] + end + end + end + end +end diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb new file mode 100644 index 00000000000..eacf261b15f --- /dev/null +++ b/lib/banzai/filter/references/issue_reference_filter.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces issue references with links. References to + # issues that do not exist are ignored. + # + # This filter supports cross-project references. + # + # When external issues tracker like Jira is activated we should not + # use issue reference pattern, but we should still be able + # to reference issues from other GitLab projects. + class IssueReferenceFilter < IssuableReferenceFilter + self.reference_type = :issue + + def self.object_class + Issue + end + + def url_for_object(issue, project) + return issue_path(issue, project) if only_path? + + issue_url(issue, project) + end + + def parent_records(parent, ids) + parent.issues.where(iid: ids.to_a) + end + + def object_link_text_extras(issue, matches) + super + design_link_extras(issue, matches.named_captures['path']) + end + + private + + def issue_path(issue, project) + Gitlab::Routing.url_helpers.namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: issue.iid) + end + + def issue_url(issue, project) + Gitlab::Routing.url_helpers.namespace_project_issue_url(namespace_id: project.namespace, project_id: project, id: issue.iid) + end + + def design_link_extras(issue, path) + if path == '/designs' && read_designs?(issue) + ['designs'] + else + [] + end + end + + def read_designs?(issue) + issue.project.design_management_enabled? + end + end + end + end +end diff --git a/lib/banzai/filter/references/iteration_reference_filter.rb b/lib/banzai/filter/references/iteration_reference_filter.rb new file mode 100644 index 00000000000..cf3d446147f --- /dev/null +++ b/lib/banzai/filter/references/iteration_reference_filter.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # The actual filter is implemented in the EE mixin + class IterationReferenceFilter < AbstractReferenceFilter + self.reference_type = :iteration + + def self.object_class + Iteration + end + end + end + end +end + +Banzai::Filter::References::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::IterationReferenceFilter') diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb new file mode 100644 index 00000000000..a6a5eec5d9a --- /dev/null +++ b/lib/banzai/filter/references/label_reference_filter.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces label references with links. + class LabelReferenceFilter < AbstractReferenceFilter + self.reference_type = :label + + def self.object_class + Label + end + + def find_object(parent_object, id) + find_labels(parent_object).find(id) + end + + def references_in(text, pattern = Label.reference_pattern) + labels = {} + unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| + namespace = $~[:namespace] + project = $~[:project] + project_path = full_project_path(namespace, project) + label = find_label_cached(project_path, $~[:label_id], $~[:label_name]) + + if label + labels[label.id] = yield match, label.id, project, namespace, $~ + "#{REFERENCE_PLACEHOLDER}#{label.id}" + else + match + end + end + + return text if labels.empty? + + escape_with_placeholders(unescaped_html, labels) + end + + def find_label_cached(parent_ref, label_id, label_name) + cached_call(:banzai_find_label_cached, label_name&.tr('"', '') || label_id, path: [object_class, parent_ref]) do + find_label(parent_ref, label_id, label_name) + end + end + + def find_label(parent_ref, label_id, label_name) + parent = parent_from_ref(parent_ref) + return unless parent + + label_params = label_params(label_id, label_name) + find_labels(parent).find_by(label_params) + end + + def find_labels(parent) + params = if parent.is_a?(Group) + { group_id: parent.id, + include_ancestor_groups: true, + only_group_labels: true } + else + { project: parent, + include_ancestor_groups: true } + end + + LabelsFinder.new(nil, params).execute(skip_authorization: true) + end + + # Parameters to pass to `Label.find_by` based on the given arguments + # + # id - Integer ID to pass. If present, returns {id: id} + # name - String name to pass. If `id` is absent, finds by name without + # surrounding quotes. + # + # Returns a Hash. + def label_params(id, name) + if name + { name: name.tr('"', '') } + else + { id: id.to_i } + end + end + + def url_for_object(label, parent) + label_url_method = + if context[:label_url_method] + context[:label_url_method] + elsif parent.is_a?(Project) + :project_issues_url + end + + return unless label_url_method + + Gitlab::Routing.url_helpers.public_send(label_url_method, parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend + end + + def object_link_text(object, matches) + label_suffix = '' + parent = project || group + + if project || full_path_ref?(matches) + project_path = full_project_path(matches[:namespace], matches[:project]) + parent_from_ref = from_ref_cached(project_path) + reference = parent_from_ref.to_human_reference(parent) + + label_suffix = " <i>in #{ERB::Util.html_escape(reference)}</i>" if reference.present? + end + + presenter = object.present(issuable_subject: parent) + LabelsHelper.render_colored_label(presenter, suffix: label_suffix) + end + + def wrap_link(link, label) + presenter = label.present(issuable_subject: project || group) + LabelsHelper.wrap_label_html(link, small: true, label: presenter) + end + + def full_path_ref?(matches) + matches[:namespace] && matches[:project] + end + + def reference_class(type, tooltip: true) + super + ' gl-link gl-label-link' + end + + def object_link_title(object, matches) + presenter = object.present(issuable_subject: project || group) + LabelsHelper.label_tooltip_title(presenter) + end + end + end + end +end + +Banzai::Filter::References::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::LabelReferenceFilter') diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb new file mode 100644 index 00000000000..872c33f6873 --- /dev/null +++ b/lib/banzai/filter/references/merge_request_reference_filter.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces merge request references with links. References + # to merge requests that do not exist are ignored. + # + # This filter supports cross-project references. + class MergeRequestReferenceFilter < IssuableReferenceFilter + self.reference_type = :merge_request + + def self.object_class + MergeRequest + end + + def url_for_object(mr, project) + h = Gitlab::Routing.url_helpers + h.project_merge_request_url(project, mr, + only_path: context[:only_path]) + end + + def object_link_title(object, matches) + # The method will return `nil` if object is not a commit + # allowing for properly handling the extended MR Tooltip + object_link_commit_title(object, matches) + end + + def object_link_text_extras(object, matches) + extras = super + + if commit_ref = object_link_commit_ref(object, matches) + klass = reference_class(:commit, tooltip: false) + commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>) + + return extras.unshift(commit_ref_tag) + end + + path = matches[:path] if matches.names.include?("path") + + case path + when '/diffs' + extras.unshift "diffs" + when '/commits' + extras.unshift "commits" + when '/builds' + extras.unshift "builds" + end + + extras + end + + def parent_records(parent, ids) + parent.merge_requests + .where(iid: ids.to_a) + .includes(target_project: :namespace) + end + + def reference_class(object_sym, options = {}) + super(object_sym, tooltip: false) + end + + def data_attributes_for(text, parent, object, **data) + super.merge(project_path: parent.full_path, iid: object.iid, mr_title: object.title) + end + + private + + def object_link_commit_title(object, matches) + object_link_commit(object, matches)&.title + end + + def object_link_commit_ref(object, matches) + object_link_commit(object, matches)&.short_id + end + + def object_link_commit(object, matches) + return unless matches.names.include?('query') && query = matches[:query] + + # Removes leading "?". CGI.parse expects "arg1&arg2&arg3" + params = CGI.parse(query.sub(/^\?/, '')) + + return unless commit_sha = params['commit_id']&.first + + if commit = find_commit_by_sha(object, commit_sha) + Commit.from_hash(commit.to_hash, object.project) + end + end + + def find_commit_by_sha(object, commit_sha) + @all_commits ||= {} + @all_commits[object.id] ||= object.all_commits + + @all_commits[object.id].find { |commit| commit.sha == commit_sha } + end + end + end + end +end diff --git a/lib/banzai/filter/references/milestone_reference_filter.rb b/lib/banzai/filter/references/milestone_reference_filter.rb new file mode 100644 index 00000000000..49110194ddc --- /dev/null +++ b/lib/banzai/filter/references/milestone_reference_filter.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces milestone references with links. + class MilestoneReferenceFilter < AbstractReferenceFilter + include Gitlab::Utils::StrongMemoize + + self.reference_type = :milestone + + def self.object_class + Milestone + end + + # Links to project milestones contain the IID, but when we're handling + # 'regular' references, we need to use the global ID to disambiguate + # between group and project milestones. + def find_object(parent, id) + return unless valid_context?(parent) + + find_milestone_with_finder(parent, id: id) + end + + def find_object_from_link(parent, iid) + return unless valid_context?(parent) + + find_milestone_with_finder(parent, iid: iid) + end + + def valid_context?(parent) + strong_memoize(:valid_context) do + group_context?(parent) || project_context?(parent) + end + end + + def group_context?(parent) + strong_memoize(:group_context) do + parent.is_a?(Group) + end + end + + def project_context?(parent) + strong_memoize(:project_context) do + parent.is_a?(Project) + end + end + + def references_in(text, pattern = Milestone.reference_pattern) + # We'll handle here the references that follow the `reference_pattern`. + # Other patterns (for example, the link pattern) are handled by the + # default implementation. + return super(text, pattern) if pattern != Milestone.reference_pattern + + milestones = {} + unescaped_html = unescape_html_entities(text).gsub(pattern) do |match| + milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) + + if milestone + milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~ + "#{REFERENCE_PLACEHOLDER}#{milestone.id}" + else + match + end + end + + return text if milestones.empty? + + escape_with_placeholders(unescaped_html, milestones) + end + + def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) + project_path = full_project_path(namespace_ref, project_ref) + + # Returns group if project is not found by path + parent = parent_from_ref(project_path) + + return unless parent + + milestone_params = milestone_params(milestone_id, milestone_name) + + find_milestone_with_finder(parent, milestone_params) + end + + def milestone_params(iid, name) + if name + { name: name.tr('"', '') } + else + { iid: iid.to_i } + end + end + + def find_milestone_with_finder(parent, params) + finder_params = milestone_finder_params(parent, params[:iid].present?) + + MilestonesFinder.new(finder_params).find_by(params) + end + + def milestone_finder_params(parent, find_by_iid) + { order: nil, state: 'all' }.tap do |params| + params[:project_ids] = parent.id if project_context?(parent) + + # We don't support IID lookups because IIDs can clash between + # group/project milestones and group/subgroup milestones. + params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid + end + end + + def self_and_ancestors_ids(parent) + if group_context?(parent) + parent.self_and_ancestors.select(:id) + elsif project_context?(parent) + parent.group&.self_and_ancestors&.select(:id) + end + end + + def url_for_object(milestone, project) + Gitlab::Routing + .url_helpers + .milestone_url(milestone, only_path: context[:only_path]) + end + + def object_link_text(object, matches) + milestone_link = escape_once(super) + reference = object.project&.to_reference_base(project) + + if reference.present? + "#{milestone_link} <i>in #{reference}</i>".html_safe + else + milestone_link + end + end + + def object_link_title(object, matches) + nil + end + end + end + end +end diff --git a/lib/banzai/filter/references/project_reference_filter.rb b/lib/banzai/filter/references/project_reference_filter.rb new file mode 100644 index 00000000000..522c6e0f5f3 --- /dev/null +++ b/lib/banzai/filter/references/project_reference_filter.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces project references with links. + class ProjectReferenceFilter < ReferenceFilter + self.reference_type = :project + + # Public: Find `namespace/project>` project references in text + # + # ProjectReferenceFilter.references_in(text) do |match, project| + # "<a href=...>#{project}></a>" + # end + # + # text - String text to search. + # + # Yields the String match, and the String project name. + # + # Returns a String replaced with the return of the block. + def self.references_in(text) + text.gsub(Project.markdown_reference_pattern) do |match| + yield match, "#{$~[:namespace]}/#{$~[:project]}" + end + end + + def call + ref_pattern = Project.markdown_reference_pattern + ref_pattern_start = /\A#{ref_pattern}\z/ + + nodes.each_with_index do |node, index| + if text_node?(node) + replace_text_when_pattern_matches(node, index, ref_pattern) do |content| + project_link_filter(content) + end + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if link =~ ref_pattern_start + replace_link_node_with_href(node, index, link) do + project_link_filter(link, link_content: inner_html) + end + end + end + end + end + + doc + end + + # Replace `namespace/project>` project references in text with links to the referenced + # project page. + # + # text - String text to replace references in. + # link_content - Original content of the link being replaced. + # + # Returns a String with `namespace/project>` references replaced with links. All links + # have `gfm` and `gfm-project` class names attached for styling. + def project_link_filter(text, link_content: nil) + self.class.references_in(text) do |match, project_path| + cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do + if project = projects_hash[project_path.downcase] + link_to_project(project, link_content: link_content) || match + else + match + end + end + end + end + + # Returns a Hash containing all Project objects for the project + # references in the current document. + # + # The keys of this Hash are the project paths, the values the + # corresponding Project objects. + def projects_hash + @projects ||= Project.eager_load(:route, namespace: [:route]) + .where_full_path_in(projects) + .index_by(&:full_path) + .transform_keys(&:downcase) + end + + # Returns all projects referenced in the current document. + def projects + refs = Set.new + + nodes.each do |node| + node.to_html.scan(Project.markdown_reference_pattern) do + refs << "#{$~[:namespace]}/#{$~[:project]}" + end + end + + refs.to_a + end + + private + + def urls + Gitlab::Routing.url_helpers + end + + def link_class + reference_class(:project) + end + + def link_to_project(project, link_content: nil) + url = urls.project_url(project, only_path: context[:only_path]) + data = data_attribute(project: project.id) + content = link_content || project.to_reference + + link_tag(url, data, content, project.name) + end + + def link_tag(url, data, link_content, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) + end + end + end + end +end diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb new file mode 100644 index 00000000000..dd15c43f5d8 --- /dev/null +++ b/lib/banzai/filter/references/reference_filter.rb @@ -0,0 +1,217 @@ +# frozen_string_literal: true + +# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js +module Banzai + module Filter + module References + # Base class for GitLab Flavored Markdown reference filters. + # + # References within <pre>, <code>, <a>, and <style> elements are ignored. + # + # Context options: + # :project (required) - Current project, ignored if reference is cross-project. + # :only_path - Generate path-only links. + class ReferenceFilter < HTML::Pipeline::Filter + include RequestStoreReferenceCache + include OutputSafety + + class << self + attr_accessor :reference_type + + def call(doc, context = nil, result = nil) + new(doc, context, result).call_and_update_nodes + end + end + + def initialize(doc, context = nil, result = nil) + super + + @new_nodes = {} + @nodes = self.result[:reference_filter_nodes] + end + + def call_and_update_nodes + with_update_nodes { call } + end + + # Returns a data attribute String to attach to a reference link + # + # attributes - Hash, where the key becomes the data attribute name and the + # value is the data attribute value + # + # Examples: + # + # data_attribute(project: 1, issue: 2) + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # + # data_attribute(project: 3, merge_request: 4) + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" + # + # Returns a String + def data_attribute(attributes = {}) + attributes = attributes.reject { |_, v| v.nil? } + + attributes[:reference_type] ||= self.class.reference_type + attributes[:container] ||= 'body' + attributes[:placement] ||= 'top' + attributes.delete(:original) if context[:no_original_data] + attributes.map do |key, value| + %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") + end.join(' ') + end + + def ignore_ancestor_query + @ignore_ancestor_query ||= begin + parents = %w(pre code a style) + parents << 'blockquote' if context[:ignore_blockquotes] + + parents.map { |n| "ancestor::#{n}" }.join(' or ') + end + end + + def project + context[:project] + end + + def group + context[:group] + end + + def user + context[:user] + end + + def skip_project_check? + context[:skip_project_check] + end + + def reference_class(type, tooltip: true) + gfm_klass = "gfm gfm-#{type}" + + return gfm_klass unless tooltip + + "#{gfm_klass} has-tooltip" + end + + # Ensure that a :project key exists in context + # + # Note that while the key might exist, its value could be nil! + def validate + needs :project unless skip_project_check? + end + + # Iterates over all <a> and text() nodes in a document. + # + # Nodes are skipped whenever their ancestor is one of the nodes returned + # by `ignore_ancestor_query`. Link tags are not processed if they have a + # "gfm" class or the "href" attribute is empty. + def each_node + return to_enum(__method__) unless block_given? + + doc.xpath(query).each do |node| + yield node + end + end + + # Returns an Array containing all HTML nodes. + def nodes + @nodes ||= each_node.to_a + end + + # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. + def yield_valid_link(node) + link = unescape_link(node.attr('href').to_s) + inner_html = node.inner_html + + return unless link.force_encoding('UTF-8').valid_encoding? + + yield link, inner_html + end + + def unescape_link(href) + CGI.unescape(href) + end + + def replace_text_when_pattern_matches(node, index, pattern) + return unless node.text =~ pattern + + content = node.to_html + html = yield content + + replace_text_with_html(node, index, html) unless html == content + end + + def replace_link_node_with_text(node, index) + html = yield + + replace_text_with_html(node, index, html) unless html == node.text + end + + def replace_link_node_with_href(node, index, link) + html = yield + + replace_text_with_html(node, index, html) unless html == link + end + + def text_node?(node) + node.is_a?(Nokogiri::XML::Text) + end + + def element_node?(node) + node.is_a?(Nokogiri::XML::Element) + end + + private + + def query + @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] + | descendant-or-self::a[ + not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") + ]} + end + + def replace_text_with_html(node, index, html) + replace_and_update_new_nodes(node, index, html) + end + + def replace_and_update_new_nodes(node, index, html) + previous_node = node.previous + next_node = node.next + parent_node = node.parent + # Unfortunately node.replace(html) returns re-parented nodes, not the actual replaced nodes in the doc + # We need to find the actual nodes in the doc that were replaced + node.replace(html) + @new_nodes[index] = [] + + # We replaced node with new nodes, so we find first new node. If previous_node is nil, we take first parent child + new_node = previous_node ? previous_node.next : parent_node&.children&.first + + # We iterate from first to last replaced node and store replaced nodes in @new_nodes + while new_node && new_node != next_node + @new_nodes[index] << new_node.xpath(query) + new_node = new_node.next + end + + @new_nodes[index].flatten! + end + + def only_path? + context[:only_path] + end + + def with_update_nodes + @new_nodes = {} + yield.tap { update_nodes! } + end + + # Once Filter completes replacing nodes, we update nodes with @new_nodes + def update_nodes! + @new_nodes.sort_by { |index, _new_nodes| -index }.each do |index, new_nodes| + nodes[index, 1] = new_nodes + end + result[:reference_filter_nodes] = nodes + end + end + end + end +end diff --git a/lib/banzai/filter/references/snippet_reference_filter.rb b/lib/banzai/filter/references/snippet_reference_filter.rb new file mode 100644 index 00000000000..bf7e0f78609 --- /dev/null +++ b/lib/banzai/filter/references/snippet_reference_filter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces snippet references with links. References to + # snippets that do not exist are ignored. + # + # This filter supports cross-project references. + class SnippetReferenceFilter < AbstractReferenceFilter + self.reference_type = :snippet + + def self.object_class + Snippet + end + + def find_object(project, id) + return unless project.is_a?(Project) + + project.snippets.find_by(id: id) + end + + def url_for_object(snippet, project) + h = Gitlab::Routing.url_helpers + h.project_snippet_url(project, snippet, + only_path: context[:only_path]) + end + end + end + end +end diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb new file mode 100644 index 00000000000..04665973f51 --- /dev/null +++ b/lib/banzai/filter/references/user_reference_filter.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # HTML filter that replaces user or group references with links. + # + # A special `@all` reference is also supported. + class UserReferenceFilter < ReferenceFilter + self.reference_type = :user + + # Public: Find `@user` user references in text + # + # UserReferenceFilter.references_in(text) do |match, username| + # "<a href=...>@#{user}</a>" + # end + # + # text - String text to search. + # + # Yields the String match, and the String user name. + # + # Returns a String replaced with the return of the block. + def self.references_in(text) + text.gsub(User.reference_pattern) do |match| + yield match, $~[:user] + end + end + + def call + return doc if project.nil? && group.nil? && !skip_project_check? + + ref_pattern = User.reference_pattern + ref_pattern_start = /\A#{ref_pattern}\z/ + + nodes.each_with_index do |node, index| + if text_node?(node) + replace_text_when_pattern_matches(node, index, ref_pattern) do |content| + user_link_filter(content) + end + elsif element_node?(node) + yield_valid_link(node) do |link, inner_html| + if link =~ ref_pattern_start + replace_link_node_with_href(node, index, link) do + user_link_filter(link, link_content: inner_html) + end + end + end + end + end + + doc + end + + # Replace `@user` user references in text with links to the referenced + # user's profile page. + # + # text - String text to replace references in. + # link_content - Original content of the link being replaced. + # + # Returns a String with `@user` references replaced with links. All links + # have `gfm` and `gfm-project_member` class names attached for styling. + def user_link_filter(text, link_content: nil) + self.class.references_in(text) do |match, username| + if username == 'all' && !skip_project_check? + link_to_all(link_content: link_content) + else + cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do + if namespace = namespaces[username.downcase] + link_to_namespace(namespace, link_content: link_content) || match + else + match + end + end + end + end + end + + # Returns a Hash containing all Namespace objects for the username + # references in the current document. + # + # The keys of this Hash are the namespace paths, the values the + # corresponding Namespace objects. + def namespaces + @namespaces ||= Namespace.eager_load(:owner, :route) + .where_full_path_in(usernames) + .index_by(&:full_path) + .transform_keys(&:downcase) + end + + # Returns all usernames referenced in the current document. + def usernames + refs = Set.new + + nodes.each do |node| + node.to_html.scan(User.reference_pattern) do + refs << $~[:user] + end + end + + refs.to_a + end + + private + + def urls + Gitlab::Routing.url_helpers + end + + def link_class + [reference_class(:project_member, tooltip: false), "js-user-link"].join(" ") + end + + def link_to_all(link_content: nil) + author = context[:author] + + if author && !team_member?(author) + link_content + else + parent_url(link_content, author) + end + end + + def link_to_namespace(namespace, link_content: nil) + if namespace.is_a?(Group) + link_to_group(namespace.full_path, namespace, link_content: link_content) + else + link_to_user(namespace.path, namespace, link_content: link_content) + end + end + + def link_to_group(group, namespace, link_content: nil) + url = urls.group_url(group, only_path: context[:only_path]) + data = data_attribute(group: namespace.id) + content = link_content || Group.reference_prefix + group + + link_tag(url, data, content, namespace.full_name) + end + + def link_to_user(user, namespace, link_content: nil) + url = urls.user_url(user, only_path: context[:only_path]) + data = data_attribute(user: namespace.owner_id) + content = link_content || User.reference_prefix + user + + link_tag(url, data, content, namespace.owner_name) + end + + def link_tag(url, data, link_content, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) + end + + def parent + context[:project] || context[:group] + end + + def parent_group? + parent.is_a?(Group) + end + + def team_member?(user) + if parent_group? + parent.member?(user) + else + parent.team.member?(user) + end + end + + def parent_url(link_content, author) + if parent_group? + url = urls.group_url(parent, only_path: context[:only_path]) + data = data_attribute(group: group.id, author: author.try(:id)) + else + url = urls.project_url(parent, only_path: context[:only_path]) + data = data_attribute(project: project.id, author: author.try(:id)) + end + + content = link_content || User.reference_prefix + 'all' + link_tag(url, data, content, 'All Project and Group Members') + end + end + end + end +end diff --git a/lib/banzai/filter/references/vulnerability_reference_filter.rb b/lib/banzai/filter/references/vulnerability_reference_filter.rb new file mode 100644 index 00000000000..e5f2408eda4 --- /dev/null +++ b/lib/banzai/filter/references/vulnerability_reference_filter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Banzai + module Filter + module References + # The actual filter is implemented in the EE mixin + class VulnerabilityReferenceFilter < IssuableReferenceFilter + self.reference_type = :vulnerability + + def self.object_class + Vulnerability + end + + private + + def project + context[:project] + end + end + end + end +end + +Banzai::Filter::References::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::VulnerabilityReferenceFilter') diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb index 66b9aac3e7e..04bbcabd93f 100644 --- a/lib/banzai/filter/repository_link_filter.rb +++ b/lib/banzai/filter/repository_link_filter.rb @@ -60,7 +60,7 @@ module Banzai def get_uri_types(paths) return {} if paths.empty? - uri_types = Hash[paths.collect { |name| [name, nil] }] + uri_types = paths.to_h { |name| [name, nil] } get_blob_types(paths).each do |name, type| if type == :blob diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb deleted file mode 100644 index f4b6edb6174..00000000000 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces snippet references with links. References to - # snippets that do not exist are ignored. - # - # This filter supports cross-project references. - class SnippetReferenceFilter < AbstractReferenceFilter - self.reference_type = :snippet - - def self.object_class - Snippet - end - - def find_object(project, id) - return unless project.is_a?(Project) - - project.snippets.find_by(id: id) - end - - def url_for_object(snippet, project) - h = Gitlab::Routing.url_helpers - h.project_snippet_url(project, snippet, - only_path: context[:only_path]) - end - end - end -end diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index f52ffe117d9..ca26e6d1581 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -42,7 +42,7 @@ module Banzai TEXT_QUERY = %Q(descendant-or-self::text()[ not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')}) and contains(., ']\(') - ]).freeze + ]) def call doc.xpath(TEXT_QUERY).each do |node| diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb index ae093580001..56a14ec0737 100644 --- a/lib/banzai/filter/suggestion_filter.rb +++ b/lib/banzai/filter/suggestion_filter.rb @@ -10,7 +10,7 @@ module Banzai def call return doc unless suggestions_filter_enabled? - doc.search('pre.suggestion > code').each do |node| + doc.search('pre.language-suggestion > code').each do |node| node.add_class(TAG_CLASS) end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 1d3bbe43344..731a2bb4c77 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -37,7 +37,7 @@ module Banzai begin code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) - css_classes << " #{language}" if language + css_classes << " language-#{language}" if language rescue # Gracefully handle syntax highlighter bugs/errors to ensure users can # still access an issue/comment/etc. First, retry with the plain text diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb deleted file mode 100644 index 262385524f4..00000000000 --- a/lib/banzai/filter/user_reference_filter.rb +++ /dev/null @@ -1,180 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # HTML filter that replaces user or group references with links. - # - # A special `@all` reference is also supported. - class UserReferenceFilter < ReferenceFilter - self.reference_type = :user - - # Public: Find `@user` user references in text - # - # UserReferenceFilter.references_in(text) do |match, username| - # "<a href=...>@#{user}</a>" - # end - # - # text - String text to search. - # - # Yields the String match, and the String user name. - # - # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(User.reference_pattern) do |match| - yield match, $~[:user] - end - end - - def call - return doc if project.nil? && group.nil? && !skip_project_check? - - ref_pattern = User.reference_pattern - ref_pattern_start = /\A#{ref_pattern}\z/ - - nodes.each_with_index do |node, index| - if text_node?(node) - replace_text_when_pattern_matches(node, index, ref_pattern) do |content| - user_link_filter(content) - end - elsif element_node?(node) - yield_valid_link(node) do |link, inner_html| - if link =~ ref_pattern_start - replace_link_node_with_href(node, index, link) do - user_link_filter(link, link_content: inner_html) - end - end - end - end - end - - doc - end - - # Replace `@user` user references in text with links to the referenced - # user's profile page. - # - # text - String text to replace references in. - # link_content - Original content of the link being replaced. - # - # Returns a String with `@user` references replaced with links. All links - # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text, link_content: nil) - self.class.references_in(text) do |match, username| - if username == 'all' && !skip_project_check? - link_to_all(link_content: link_content) - else - cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do - if namespace = namespaces[username.downcase] - link_to_namespace(namespace, link_content: link_content) || match - else - match - end - end - end - end - end - - # Returns a Hash containing all Namespace objects for the username - # references in the current document. - # - # The keys of this Hash are the namespace paths, the values the - # corresponding Namespace objects. - def namespaces - @namespaces ||= Namespace.eager_load(:owner, :route) - .where_full_path_in(usernames) - .index_by(&:full_path) - .transform_keys(&:downcase) - end - - # Returns all usernames referenced in the current document. - def usernames - refs = Set.new - - nodes.each do |node| - node.to_html.scan(User.reference_pattern) do - refs << $~[:user] - end - end - - refs.to_a - end - - private - - def urls - Gitlab::Routing.url_helpers - end - - def link_class - [reference_class(:project_member, tooltip: false), "js-user-link"].join(" ") - end - - def link_to_all(link_content: nil) - author = context[:author] - - if author && !team_member?(author) - link_content - else - parent_url(link_content, author) - end - end - - def link_to_namespace(namespace, link_content: nil) - if namespace.is_a?(Group) - link_to_group(namespace.full_path, namespace, link_content: link_content) - else - link_to_user(namespace.path, namespace, link_content: link_content) - end - end - - def link_to_group(group, namespace, link_content: nil) - url = urls.group_url(group, only_path: context[:only_path]) - data = data_attribute(group: namespace.id) - content = link_content || Group.reference_prefix + group - - link_tag(url, data, content, namespace.full_name) - end - - def link_to_user(user, namespace, link_content: nil) - url = urls.user_url(user, only_path: context[:only_path]) - data = data_attribute(user: namespace.owner_id) - content = link_content || User.reference_prefix + user - - link_tag(url, data, content, namespace.owner_name) - end - - def link_tag(url, data, link_content, title) - %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) - end - - def parent - context[:project] || context[:group] - end - - def parent_group? - parent.is_a?(Group) - end - - def team_member?(user) - if parent_group? - parent.member?(user) - else - parent.team.member?(user) - end - end - - def parent_url(link_content, author) - if parent_group? - url = urls.group_url(parent, only_path: context[:only_path]) - data = data_attribute(group: group.id, author: author.try(:id)) - else - url = urls.project_url(parent, only_path: context[:only_path]) - data = data_attribute(project: project.id, author: author.try(:id)) - end - - content = link_content || User.reference_prefix + 'all' - link_tag(url, data, content, 'All Project and Group Members') - end - end - end -end diff --git a/lib/banzai/filter/vulnerability_reference_filter.rb b/lib/banzai/filter/vulnerability_reference_filter.rb deleted file mode 100644 index a59e9836d69..00000000000 --- a/lib/banzai/filter/vulnerability_reference_filter.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - # The actual filter is implemented in the EE mixin - class VulnerabilityReferenceFilter < IssuableReferenceFilter - self.reference_type = :vulnerability - - def self.object_class - Vulnerability - end - - private - - def project - context[:project] - end - end - end -end - -Banzai::Filter::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::VulnerabilityReferenceFilter') diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index f4cc8beeb52..b4c2e7efae3 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -6,7 +6,7 @@ module Banzai class Rewriter def initialize(link_string, wiki:, slug:) @uri = Addressable::URI.parse(link_string) - @wiki_base_path = wiki && wiki.wiki_base_path + @wiki_base_path = wiki&.wiki_base_path @slug = slug end @@ -41,7 +41,8 @@ module Banzai # Any link _not_ of the form `http://example.com/` def apply_relative_link_rules! if @uri.relative? && @uri.path.present? - link = ::File.join(@wiki_base_path, @uri.path) + link = @uri.path + link = ::File.join(@wiki_base_path, link) unless link.starts_with?(@wiki_base_path) link = "#{link}##{@uri.fragment}" if @uri.fragment @uri = Addressable::URI.parse(link) end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index e5ec0a0a006..028e3c44dc3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -51,19 +51,19 @@ module Banzai def self.reference_filters [ - Filter::UserReferenceFilter, - Filter::ProjectReferenceFilter, - Filter::DesignReferenceFilter, - Filter::IssueReferenceFilter, - Filter::ExternalIssueReferenceFilter, - Filter::MergeRequestReferenceFilter, - Filter::SnippetReferenceFilter, - Filter::CommitRangeReferenceFilter, - Filter::CommitReferenceFilter, - Filter::LabelReferenceFilter, - Filter::MilestoneReferenceFilter, - Filter::AlertReferenceFilter, - Filter::FeatureFlagReferenceFilter + Filter::References::UserReferenceFilter, + Filter::References::ProjectReferenceFilter, + Filter::References::DesignReferenceFilter, + Filter::References::IssueReferenceFilter, + Filter::References::ExternalIssueReferenceFilter, + Filter::References::MergeRequestReferenceFilter, + Filter::References::SnippetReferenceFilter, + Filter::References::CommitRangeReferenceFilter, + Filter::References::CommitReferenceFilter, + Filter::References::LabelReferenceFilter, + Filter::References::MilestoneReferenceFilter, + Filter::References::AlertReferenceFilter, + Filter::References::FeatureFlagReferenceFilter ] end diff --git a/lib/banzai/pipeline/label_pipeline.rb b/lib/banzai/pipeline/label_pipeline.rb index 725cccc4b2b..ccfda2052e6 100644 --- a/lib/banzai/pipeline/label_pipeline.rb +++ b/lib/banzai/pipeline/label_pipeline.rb @@ -6,7 +6,7 @@ module Banzai def self.filters @filters ||= FilterArray[ Filter::SanitizationFilter, - Filter::LabelReferenceFilter + Filter::References::LabelReferenceFilter ] end end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index 4bf98099662..65a5e28b704 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -17,15 +17,15 @@ module Banzai def self.reference_filters [ - Filter::UserReferenceFilter, - Filter::IssueReferenceFilter, - Filter::ExternalIssueReferenceFilter, - Filter::MergeRequestReferenceFilter, - Filter::SnippetReferenceFilter, - Filter::CommitRangeReferenceFilter, - Filter::CommitReferenceFilter, - Filter::AlertReferenceFilter, - Filter::FeatureFlagReferenceFilter + Filter::References::UserReferenceFilter, + Filter::References::IssueReferenceFilter, + Filter::References::ExternalIssueReferenceFilter, + Filter::References::MergeRequestReferenceFilter, + Filter::References::SnippetReferenceFilter, + Filter::References::CommitRangeReferenceFilter, + Filter::References::CommitReferenceFilter, + Filter::References::AlertReferenceFilter, + Filter::References::FeatureFlagReferenceFilter ] end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 97a03895ff3..caba9570ab9 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -5,7 +5,7 @@ module Banzai class WikiPipeline < FullPipeline def self.filters @filters ||= begin - super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) + super.insert_before(Filter::ImageLazyLoadFilter, Filter::GollumTagsFilter) .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) end end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 2e81863e53a..ef99122cdfd 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -3,9 +3,9 @@ module BulkImports module Clients class Http - API_VERSION = 'v4'.freeze - DEFAULT_PAGE = 1.freeze - DEFAULT_PER_PAGE = 30.freeze + API_VERSION = 'v4' + DEFAULT_PAGE = 1 + DEFAULT_PER_PAGE = 30 ConnectionError = Class.new(StandardError) @@ -23,7 +23,7 @@ module BulkImports resource_url(resource), headers: request_headers, follow_redirects: false, - query: query.merge(request_query) + query: query.reverse_merge(request_query) ) end end diff --git a/lib/bulk_imports/common/extractors/rest_extractor.rb b/lib/bulk_imports/common/extractors/rest_extractor.rb new file mode 100644 index 00000000000..b18e27fd475 --- /dev/null +++ b/lib/bulk_imports/common/extractors/rest_extractor.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Extractors + class RestExtractor + def initialize(options = {}) + @query = options[:query] + end + + def extract(context) + client = http_client(context.configuration) + params = query.to_h(context) + response = client.get(params[:resource], params[:query]) + + BulkImports::Pipeline::ExtractedData.new( + data: response.parsed_response, + page_info: page_info(response.headers) + ) + end + + private + + attr_reader :query + + def http_client(configuration) + @http_client ||= BulkImports::Clients::Http.new( + uri: configuration.url, + token: configuration.access_token, + per_page: 100 + ) + end + + def page_info(headers) + next_page = headers['x-next-page'] + + { + 'has_next_page' => next_page.present?, + 'next_page' => next_page + } + end + end + end + end +end diff --git a/lib/bulk_imports/common/transformers/user_reference_transformer.rb b/lib/bulk_imports/common/transformers/user_reference_transformer.rb index ca077b4ef43..c330ea59113 100644 --- a/lib/bulk_imports/common/transformers/user_reference_transformer.rb +++ b/lib/bulk_imports/common/transformers/user_reference_transformer.rb @@ -12,7 +12,7 @@ module BulkImports DEFAULT_REFERENCE = 'user' def initialize(options = {}) - @reference = options[:reference] || DEFAULT_REFERENCE + @reference = options[:reference].to_s.presence || DEFAULT_REFERENCE @suffixed_reference = "#{@reference}_id" end diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb index 23efbc33581..f957cf0be52 100644 --- a/lib/bulk_imports/groups/graphql/get_labels_query.rb +++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb @@ -8,11 +8,11 @@ module BulkImports def to_s <<-'GRAPHQL' - query ($full_path: ID!, $cursor: String) { + query ($full_path: ID!, $cursor: String, $per_page: Int) { group(fullPath: $full_path) { - labels(first: 100, after: $cursor, onlyGroupLabels: true) { + labels(first: $per_page, after: $cursor, onlyGroupLabels: true) { page_info: pageInfo { - end_cursor: endCursor + next_page: endCursor has_next_page: hasNextPage } nodes { @@ -31,7 +31,8 @@ module BulkImports def variables(context) { full_path: context.entity.source_full_path, - cursor: context.entity.next_page_for(:labels) + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE } end diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/groups/graphql/get_members_query.rb index e3a78124a47..e44d3c5aa9b 100644 --- a/lib/bulk_imports/groups/graphql/get_members_query.rb +++ b/lib/bulk_imports/groups/graphql/get_members_query.rb @@ -7,11 +7,11 @@ module BulkImports extend self def to_s <<-'GRAPHQL' - query($full_path: ID!, $cursor: String) { + query($full_path: ID!, $cursor: String, $per_page: Int) { group(fullPath: $full_path) { - group_members: groupMembers(relations: DIRECT, first: 100, after: $cursor) { + group_members: groupMembers(relations: DIRECT, first: $per_page, after: $cursor) { page_info: pageInfo { - end_cursor: endCursor + next_page: endCursor has_next_page: hasNextPage } nodes { @@ -34,7 +34,8 @@ module BulkImports def variables(context) { full_path: context.entity.source_full_path, - cursor: context.entity.next_page_for(:group_members) + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE } end diff --git a/lib/bulk_imports/groups/graphql/get_milestones_query.rb b/lib/bulk_imports/groups/graphql/get_milestones_query.rb index 2ade87e6fa0..5dd5b31cf0e 100644 --- a/lib/bulk_imports/groups/graphql/get_milestones_query.rb +++ b/lib/bulk_imports/groups/graphql/get_milestones_query.rb @@ -8,14 +8,15 @@ module BulkImports def to_s <<-'GRAPHQL' - query ($full_path: ID!, $cursor: String) { + query ($full_path: ID!, $cursor: String, $per_page: Int) { group(fullPath: $full_path) { - milestones(first: 100, after: $cursor, includeDescendants: false) { + milestones(first: $per_page, after: $cursor, includeDescendants: false) { page_info: pageInfo { - end_cursor: endCursor + next_page: endCursor has_next_page: hasNextPage } nodes { + iid title description state @@ -33,7 +34,8 @@ module BulkImports def variables(context) { full_path: context.entity.source_full_path, - cursor: context.entity.next_page_for(:milestones) + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE } end diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb new file mode 100644 index 00000000000..8569ff3f77a --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class BadgesPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::RestExtractor, + query: BulkImports::Groups::Rest::GetBadgesQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(_, data) + return if data.blank? + + { + name: data['name'], + link_url: data['link_url'], + image_url: data['image_url'] + } + end + + def load(context, data) + return if data.blank? + + context.group.badges.create!(data) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/groups/pipelines/entity_finisher.rb new file mode 100644 index 00000000000..1d237bc0f7f --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/entity_finisher.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class EntityFinisher + def initialize(context) + @context = context + end + + def run + return if context.entity.finished? + + context.entity.finish! + + logger.info( + bulk_import_id: context.bulk_import.id, + bulk_import_entity_id: context.entity.id, + bulk_import_entity_type: context.entity.source_type, + pipeline_class: self.class.name, + message: 'Entity finished' + ) + end + + private + + attr_reader :context + + def logger + @logger ||= Gitlab::Import::Logger.build + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb index 9f8b8682751..0dc4a968b84 100644 --- a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb @@ -14,18 +14,6 @@ module BulkImports def load(context, data) Labels::CreateService.new(data).execute(group: context.group) end - - def after_run(extracted_data) - context.entity.update_tracker_for( - relation: :labels, - has_next_page: extracted_data.has_next_page?, - next_page: extracted_data.next_page - ) - - if extracted_data.has_next_page? - run - end - end end end end diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb index 32fc931e8c3..5e4293d2c06 100644 --- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb @@ -17,18 +17,6 @@ module BulkImports context.group.members.create!(data) end - - def after_run(extracted_data) - context.entity.update_tracker_for( - relation: :group_members, - has_next_page: extracted_data.has_next_page?, - next_page: extracted_data.next_page - ) - - if extracted_data.has_next_page? - run - end - end end end end diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb index 8497162e0e7..9b2be30735c 100644 --- a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb @@ -19,18 +19,6 @@ module BulkImports context.group.milestones.create!(data) end - def after_run(extracted_data) - context.entity.update_tracker_for( - relation: :milestones, - has_next_page: extracted_data.has_next_page?, - next_page: extracted_data.next_page - ) - - if extracted_data.has_next_page? - run - end - end - private def authorized? diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/groups/rest/get_badges_query.rb new file mode 100644 index 00000000000..79ffdd9a1f6 --- /dev/null +++ b/lib/bulk_imports/groups/rest/get_badges_query.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Rest + module GetBadgesQuery + extend self + + def to_h(context) + encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) + + { + resource: ['groups', encoded_full_path, 'badges'].join('/'), + query: { + page: context.tracker.next_page + } + } + end + end + end + end +end diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb deleted file mode 100644 index f016b552fd4..00000000000 --- a/lib/bulk_imports/importers/group_importer.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Importers - class GroupImporter - def initialize(entity) - @entity = entity - end - - def execute - context = BulkImports::Pipeline::Context.new(entity) - - pipelines.each { |pipeline| pipeline.new(context).run } - - entity.finish! - end - - private - - attr_reader :entity - - def pipelines - [ - BulkImports::Groups::Pipelines::GroupPipeline, - BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, - BulkImports::Groups::Pipelines::MembersPipeline, - BulkImports::Groups::Pipelines::LabelsPipeline, - BulkImports::Groups::Pipelines::MilestonesPipeline - ] - end - end - end -end - -BulkImports::Importers::GroupImporter.prepend_if_ee('EE::BulkImports::Importers::GroupImporter') diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 14445162737..df4f020d6b2 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -15,6 +15,10 @@ module BulkImports @context = context end + def tracker + @tracker ||= context.tracker + end + included do private diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index dd121b2dbed..3c69c729f36 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -3,25 +3,33 @@ module BulkImports module Pipeline class Context - attr_reader :entity, :bulk_import attr_accessor :extra - def initialize(entity, extra = {}) - @entity = entity - @bulk_import = entity.bulk_import + attr_reader :tracker + + def initialize(tracker, extra = {}) + @tracker = tracker @extra = extra end + def entity + @entity ||= tracker.entity + end + def group - entity.group + @group ||= entity.group + end + + def bulk_import + @bulk_import ||= entity.bulk_import end def current_user - bulk_import.user + @current_user ||= bulk_import.user end def configuration - bulk_import.configuration + @configuration ||= bulk_import.configuration end end end diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb index 685a91a4afe..c9e54b61dd3 100644 --- a/lib/bulk_imports/pipeline/extracted_data.rb +++ b/lib/bulk_imports/pipeline/extracted_data.rb @@ -11,11 +11,14 @@ module BulkImports end def has_next_page? - @page_info['has_next_page'] + Gitlab::Utils.to_boolean( + @page_info&.dig('has_next_page'), + default: false + ) end def next_page - @page_info['end_cursor'] + @page_info&.dig('next_page') end def each(&block) diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index e3535e585cc..b756fba3bee 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -14,19 +14,24 @@ module BulkImports extracted_data = extracted_data_from - extracted_data&.each do |entry| - transformers.each do |transformer| - entry = run_pipeline_step(:transformer, transformer.class.name) do - transformer.transform(context, entry) + if extracted_data + extracted_data.each do |entry| + transformers.each do |transformer| + entry = run_pipeline_step(:transformer, transformer.class.name) do + transformer.transform(context, entry) + end end - end - run_pipeline_step(:loader, loader.class.name) do - loader.load(context, entry) + run_pipeline_step(:loader, loader.class.name) do + loader.load(context, entry) + end end - end - if respond_to?(:after_run) + tracker.update!( + has_next_page: extracted_data.has_next_page?, + next_page: extracted_data.next_page + ) + run_pipeline_step(:after_run) do after_run(extracted_data) end @@ -34,7 +39,7 @@ module BulkImports info(message: 'Pipeline finished') rescue MarkedAsFailedError - log_skip + skip!('Skipping pipeline due to failed entity') end private # rubocop:disable Lint/UselessAccessModifier @@ -46,7 +51,11 @@ module BulkImports yield rescue MarkedAsFailedError - log_skip(step => class_name) + skip!( + 'Skipping pipeline due to failed entity', + pipeline_step: step, + step_class: class_name + ) rescue => e log_import_failure(e, step) @@ -61,14 +70,21 @@ module BulkImports end end + def after_run(extracted_data) + run if extracted_data.has_next_page? + end + def mark_as_failed warn(message: 'Pipeline failed') context.entity.fail_op! + tracker.fail_op! end - def log_skip(extra = {}) - info({ message: 'Skipping due to failed pipeline status' }.merge(extra)) + def skip!(message, extra = {}) + warn({ message: message }.merge(extra)) + + tracker.skip! end def log_import_failure(exception, step) diff --git a/lib/constraints/admin_constrainer.rb b/lib/constraints/admin_constrainer.rb index 59c855a1b73..2f32cc7ad91 100644 --- a/lib/constraints/admin_constrainer.rb +++ b/lib/constraints/admin_constrainer.rb @@ -3,7 +3,7 @@ module Constraints class AdminConstrainer def matches?(request) - if Feature.enabled?(:user_mode_in_session) + if Gitlab::CurrentSettings.admin_mode admin_mode_enabled?(request) else user_is_admin?(request) diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb index 40dd92befd2..aafa9b1c182 100644 --- a/lib/container_registry/config.rb +++ b/lib/container_registry/config.rb @@ -5,7 +5,8 @@ module ContainerRegistry attr_reader :tag, :blob, :data def initialize(tag, blob) - @tag, @blob = tag, blob + @tag = tag + @blob = blob @data = Gitlab::Json.parse(blob.data) end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 09c0aa66a0d..614b1b5e6c6 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -10,7 +10,8 @@ module ContainerRegistry delegate :revision, :short_revision, to: :config_blob, allow_nil: true def initialize(repository, name) - @repository, @name = repository, name + @repository = repository + @name = name end def valid? diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb index 6116009f171..43ceed9519b 100644 --- a/lib/csv_builder.rb +++ b/lib/csv_builder.rb @@ -14,7 +14,7 @@ # CsvBuilder.new(@posts, columns).render # class CsvBuilder - DEFAULT_ORDER_BY = 'id'.freeze + DEFAULT_ORDER_BY = 'id' DEFAULT_BATCH_SIZE = 1000 PREFIX_REGEX = /^[=\+\-@;]/.freeze diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb index d653a0ec1e1..9e512086593 100644 --- a/lib/declarative_policy/preferred_scope.rb +++ b/lib/declarative_policy/preferred_scope.rb @@ -5,7 +5,8 @@ module DeclarativePolicy PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope" def with_preferred_scope(scope) - Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY] + old_scope = Thread.current[PREFERRED_SCOPE_KEY] + Thread.current[PREFERRED_SCOPE_KEY] = scope yield ensure Thread.current[PREFERRED_SCOPE_KEY] = old_scope diff --git a/lib/error_tracking/sentry_client/issue.rb b/lib/error_tracking/sentry_client/issue.rb index 513fb3daabe..bdc567bd859 100644 --- a/lib/error_tracking/sentry_client/issue.rb +++ b/lib/error_tracking/sentry_client/issue.rb @@ -113,9 +113,7 @@ module ErrorTracking uri = URI(url) uri.path.squeeze!('/') # Remove trailing slash - uri = uri.to_s.delete_suffix('/') - - uri + uri.to_s.delete_suffix('/') end def map_to_errors(issues) diff --git a/lib/feature.rb b/lib/feature.rb index 7c926b25587..709610b91be 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -18,10 +18,6 @@ class Feature superclass.table_name = 'feature_gates' end - class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore - # overrides methods in EE - end - InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException class << self diff --git a/lib/feature/active_support_cache_store_adapter.rb b/lib/feature/active_support_cache_store_adapter.rb new file mode 100644 index 00000000000..ae2d623abe1 --- /dev/null +++ b/lib/feature/active_support_cache_store_adapter.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# rubocop:disable Gitlab/NamespacedClass +# This class was already nested this way before moving to a separate file +class Feature + class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore + def enable(feature, gate, thing) + result = @adapter.enable(feature, gate, thing) + @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) + result + end + + def disable(feature, gate, thing) + result = @adapter.disable(feature, gate, thing) + @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) + result + end + + def remove(feature) + result = @adapter.remove(feature) + @cache.delete(FeaturesKey) + @cache.write(key_for(feature.key), {}, @write_options) + result + end + end +end +# rubocop:disable Gitlab/NamespacedClass diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 71908d2130f..e9868732172 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -11,7 +11,8 @@ class FileSizeValidator < ActiveModel::EachValidator if range = (options.delete(:in) || options.delete(:within)) raise ArgumentError, ":in and :within must be a Range" unless range.is_a?(Range) - options[:minimum], options[:maximum] = range.begin, range.end + options[:minimum] = range.begin + options[:maximum] = range.end options[:maximum] -= 1 if range.exclude_end? end diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb new file mode 100644 index 00000000000..d826c51a73d --- /dev/null +++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'rails/generators' +require_relative '../usage_metric_definition_generator' + +module Gitlab + module UsageMetricDefinition + class RedisHllGenerator < Rails::Generators::Base + desc 'Generates a metric definition .yml file with defaults for Redis HLL.' + + argument :category, type: :string, desc: "Category name" + argument :event, type: :string, desc: "Event name" + + def create_metrics + Gitlab::UsageMetricDefinitionGenerator.start(["#{key_path}_weekly", '--dir', '7d']) + Gitlab::UsageMetricDefinitionGenerator.start(["#{key_path}_monthly", '--dir', '28d']) + end + + private + + def key_path + "redis_hll_counters.#{category}.#{event}" + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb index 7a01050ed0c..cadc319a212 100644 --- a/lib/generators/gitlab/usage_metric_definition_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -5,6 +5,11 @@ require 'rails/generators' module Gitlab class UsageMetricDefinitionGenerator < Rails::Generators::Base Directory = Struct.new(:name, :time_frame, :value_type) do + def initialize(...) + super + freeze + end + def match?(str) (name == str || time_frame == str) && str != 'none' end @@ -60,9 +65,7 @@ module Gitlab private def metric_name_suggestion - return unless Feature.enabled?(:product_intelligence_metrics_names_suggestions, default_enabled: :yaml) - - "\nname: #{Usage::Metrics::NamesSuggestions::Generator.generate(key_path)}" + "\nname: \"#{Usage::Metrics::NamesSuggestions::Generator.generate(key_path)}\"" end def file_path @@ -101,7 +104,7 @@ module Gitlab end def metric_definitions - @definitions ||= Gitlab::Usage::MetricDefinition.definitions + @definitions ||= Gitlab::Usage::MetricDefinition.definitions(skip_validation: true) end def metric_definition_exists? diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 11ecfb951aa..ddf08c8dc20 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -108,10 +108,21 @@ module Gitlab !%w[true 1].include?(ENV['FOSS_ONLY'].to_s) end + def self.jh? + @is_jh ||= + ee? && + root.join('jh').exist? && + !%w[true 1].include?(ENV['EE_ONLY'].to_s) + end + def self.ee yield if ee? end + def self.jh + yield if jh? + end + def self.http_proxy_env? HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] } end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index c8b8d6c259d..786c5bf675b 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -132,7 +132,7 @@ module Gitlab EnvironmentsFinder .new(project, nil, { name: environment_name }) - .find + .execute .first end end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 178ebe0d4d4..b4752ed9e5b 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -31,14 +31,34 @@ module Gitlab @params = params @sort = params[:sort] || :end_event @direction = params[:direction] || :desc + @page = params[:page] || 1 + @per_page = MAX_RECORDS end + # rubocop: disable CodeReuse/ActiveRecord def serialized_records strong_memoize(:serialized_records) do # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records if default_test_stage? || default_staging_stage? + ci_build_join = mr_metrics_table + .join(build_table) + .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + .join_sources + + records = ordered_and_limited_query + .joins(ci_build_join) + .select(build_table[:id], *time_columns) + + yield records if block_given? + ci_build_records = preload_ci_build_associations(records) + AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] }) else + records = ordered_and_limited_query.select(*columns, *time_columns) + + yield records if block_given? + records = preload_associations(records) + records.map do |record| project = record.project attributes = record.attributes.merge({ @@ -51,10 +71,11 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord private - attr_reader :stage, :query, :params, :sort, :direction + attr_reader :stage, :query, :params, :sort, :direction, :page, :per_page def columns MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| @@ -74,41 +95,32 @@ module Gitlab MAPPINGS.fetch(subject_class).fetch(:serializer_class).new end - # Loading Ci::Build records instead of MergeRequest records # rubocop: disable CodeReuse/ActiveRecord - def ci_build_records - ci_build_join = mr_metrics_table - .join(build_table) - .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - .join_sources - - q = ordered_and_limited_query - .joins(ci_build_join) - .select(build_table[:id], *time_columns) - - results = execute_query(q).to_a + def preload_ci_build_associations(records) + results = records.map(&:attributes) Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] })) end + # rubocop: enable CodeReuse/ActiveRecord def ordered_and_limited_query - order_by(query, sort, direction, columns).limit(MAX_RECORDS) + strong_memoize(:ordered_and_limited_query) do + order_by(query, sort, direction, columns).page(page).per(per_page).without_count + end end - def records - results = ordered_and_limited_query - .select(*columns, *time_columns) - + # rubocop: disable CodeReuse/ActiveRecord + def preload_associations(records) # using preloader instead of includes to avoid AR generating a large column list ActiveRecord::Associations::Preloader.new.preload( - results, + records, MAPPINGS.fetch(subject_class).fetch(:includes_for_query) ) - results + records end - # rubocop: enable CodeReuse/ActiveRecord + # rubocop: enable CodeReuse/ActiveRecord def time_columns [ stage.start_event.timestamp_projection.as('start_event_timestamp'), diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index e367d33d743..723486231b1 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -3,8 +3,8 @@ module Gitlab module Analytics class UniqueVisits - def track_visit(visitor_id, target_id, time = Time.zone.now) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(target_id, values: visitor_id, time: time) + def track_visit(*args, **kwargs) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(*args, **kwargs) end # Returns number of unique visitors for given targets in given time frame diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index a75da3a682b..ceda82cb6f6 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -8,6 +8,9 @@ module Gitlab Attribute = Struct.new(:name, :type) + LOG_KEY = Labkit::Context::LOG_KEY + KNOWN_KEYS = Labkit::Context::KNOWN_KEYS + APPLICATION_ATTRIBUTES = [ Attribute.new(:project, Project), Attribute.new(:namespace, Namespace), @@ -24,6 +27,10 @@ module Gitlab application_context.use(&block) end + def self.with_raw_context(attributes = {}, &block) + Labkit::Context.with_context(attributes, &block) + end + def self.push(args) application_context = new(**args) Labkit::Context.push(application_context.to_lazy_hash) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 4c6254c9e69..6f6ac79c16b 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -24,9 +24,9 @@ module Gitlab PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' JOB_TOKEN_PARAM = :job_token - DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'.freeze + DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN' RUNNER_TOKEN_PARAM = :token RUNNER_JOB_TOKEN_PARAM = :token diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index b7bb61f0677..7f85d3b1cd3 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -5,7 +5,7 @@ module Gitlab module Ldap class Adapter SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze - MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size attr_reader :provider, :ldap diff --git a/lib/gitlab/auth/saml/origin_validator.rb b/lib/gitlab/auth/saml/origin_validator.rb index 4ecc688888f..ff0d25314f7 100644 --- a/lib/gitlab/auth/saml/origin_validator.rb +++ b/lib/gitlab/auth/saml/origin_validator.rb @@ -4,7 +4,7 @@ module Gitlab module Auth module Saml class OriginValidator - AUTH_REQUEST_SESSION_KEY = "last_authn_request_id".freeze + AUTH_REQUEST_SESSION_KEY = "last_authn_request_id" def initialize(session) @session = session || {} diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb index 553571d5d00..6d1df95c66d 100644 --- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -97,13 +97,13 @@ module Gitlab ActiveRecord::Base.connection.execute <<~SQL WITH - starting_iids(project_id, iid) as ( + starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT project_id, MAX(COALESCE(iid, 0)) FROM #{table} WHERE project_id BETWEEN #{start_id} AND #{end_id} GROUP BY project_id ), - with_calculated_iid(id, iid) as ( + with_calculated_iid(id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT design.id, init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC) FROM #{table} as design, starting_iids as init diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb index 7484027a0fa..030dfd2d99b 100644 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -8,7 +8,7 @@ module Gitlab updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) Project.connection.execute <<-SQL - WITH repository_storage_cte as ( + WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{updated_repository_storages.to_sql} ) UPDATE projects diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 60682bd2ec1..b89ea7dc250 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -34,12 +34,18 @@ module Gitlab parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + batch_metrics.time_operation(:update_all) do + sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + end sleep(PAUSE_SECONDS) end end + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + private def connection diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb index 6014ccc12eb..691bdb457d7 100644 --- a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb +++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb @@ -8,7 +8,7 @@ module Gitlab def perform(start_id, stop_id) ActiveRecord::Base.connection.execute <<~SQL - WITH merge_requests_batch AS ( + WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT id, target_project_id FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)} ) diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index 68665db522e..83c01afa432 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab def sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO project_features ( project_id, merge_requests_access_level, diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index e750b8ca374..b8e4562b3bf 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -136,7 +136,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def create_sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at) #{select_insert_values_sql(from_id, to_id)} RETURNING * @@ -149,7 +149,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def update_sql(from_id, to_id) <<~SQL - WITH updated_records AS ( + WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( UPDATE services SET active = TRUE WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}' AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb index d767cbfd8f5..cd5b4ab103d 100644 --- a/lib/gitlab/background_migration/fix_user_namespace_names.rb +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -14,7 +14,7 @@ module Gitlab def fix_namespace_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES - WITH namespaces_to_update AS ( + WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT namespaces.id, users.name AS correct_name @@ -39,7 +39,7 @@ module Gitlab def fix_namespace_route_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name AS correct_name diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb index 6b99685fd68..e534f2449aa 100644 --- a/lib/gitlab/background_migration/fix_user_project_route_names.rb +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -8,7 +8,7 @@ module Gitlab class FixUserProjectRouteNames def perform(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name || ' / ' || projects.name AS correct_name diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb new file mode 100644 index 00000000000..b7a912da060 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # migrates pages from legacy storage to zip format + # we intentionally use application code here because + # it has a lot of dependencies including models, carrierwave uploaders and service objects + # and copying all or part of this code in the background migration doesn't add much value + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion + class MigratePagesToZipStorage + def perform(start_id, stop_id) + ::Pages::MigrateFromLegacyStorageService.new(Gitlab::AppLogger, + ignore_invalid_entries: false, + mark_projects_as_not_deployed: false) + .execute_for_batch(start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index 4eaef26c9c6..9ecf53317d0 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -6,7 +6,7 @@ module Gitlab # project_features.container_registry_access_level for the projects within # the given range of ids. class MoveContainerRegistryEnabledToProjectFeature - MAX_BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 300 module Migratable # Migration model namespace isolated from application code. diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb index 78140b768fc..28ff2070209 100644 --- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb @@ -8,21 +8,23 @@ module Gitlab class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation self.table_name = 'project_settings' - UPSERT_SQL = <<~SQL - WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS ( - SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) - ) - INSERT INTO project_settings - (project_id, has_vulnerabilities, created_at, updated_at) - (SELECT * FROM upsert_data) - ON CONFLICT (project_id) - DO UPDATE SET - has_vulnerabilities = true, - updated_at = EXCLUDED.updated_at - SQL - def self.upsert_for(project_ids) - connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') }) + connection.execute(upsert_sql % { project_ids: project_ids.join(', ') }) + end + + def self.upsert_sql + <<~SQL + WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) + ) + INSERT INTO project_settings + (project_id, has_vulnerabilities, created_at, updated_at) + (SELECT * FROM upsert_data) + ON CONFLICT (project_id) + DO UPDATE SET + has_vulnerabilities = true, + updated_at = EXCLUDED.updated_at + SQL end end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb index eb4bc0aaf28..28cc4a5e3fa 100644 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -11,7 +11,7 @@ module Gitlab MergeRequest .where(merge_request_assignees_not_exists_clause) .where(id: from_id..to_id) - .where('assignee_id IS NOT NULL') + .where.not(assignee_id: nil) .select(:id, :assignee_id) .to_sql diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 7b18e617c81..888a12f2330 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -32,7 +32,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid }.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN".freeze + PACK_PATTERN = "NnnnnN" def self.call(value) Digest::UUID.uuid_v5(namespace_id, value) diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb new file mode 100644 index 00000000000..5930d65bc2c --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Project + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb new file mode 100644 index 00000000000..f5f98517433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::User + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb index 1d3a3af81a1..8610129533d 100644 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -7,7 +7,7 @@ module Gitlab module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor - REFERABLES = %i(isolated_mentioned_group).freeze + REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze REFERABLES.each do |type| define_method("#{type}s") do diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb new file mode 100644 index 00000000000..0334ea1dd08 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Gitlab::IsolatedVisibilityLevel module + # + # Define allowed public modes that can be used for + # GitLab projects to determine project public mode + # + module IsolatedVisibilityLevel + extend ::ActiveSupport::Concern + + included do + scope :public_to_user, -> (user = nil) do + where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user)) + end + end + + PRIVATE = 0 unless const_defined?(:PRIVATE) + INTERNAL = 10 unless const_defined?(:INTERNAL) + PUBLIC = 20 unless const_defined?(:PUBLIC) + + class << self + def levels_for_user(user = nil) + return [PUBLIC] unless user + + if user.can_read_all_resources? + [PRIVATE, INTERNAL, PUBLIC] + elsif user.external? + [PUBLIC] + else + [INTERNAL, PUBLIC] + end + end + end + + def private? + visibility_level_value == PRIVATE + end + + def internal? + visibility_level_value == INTERNAL + end + + def public? + visibility_level_value == PUBLIC + end + + def visibility_level_value + self[visibility_level_field] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb index bdb4d6c7d48..f4cc96c8bc0 100644 --- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class CommitUserMention < ActiveRecord::Base self.table_name = 'commit_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :commit_id diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb new file mode 100644 index 00000000000..ba6b783f9f1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + # isolated FeatureGate module + module IsolatedFeatureGate + def flipper_id + return if new_record? + + "#{self.class.name}:#{id}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index be9c0ad2b3a..f684f789ea9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -70,8 +70,8 @@ module Gitlab def build_mention_values(resource_foreign_key) refs = all_references(author) - mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) - mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) + mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id)) + mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id)) mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb index 5cadfa45b5b..75759ed0111 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -6,7 +6,7 @@ module Gitlab module Models module Concerns module Namespace - # extracted methods for recursive traversing of namespace hierarchy + # isolate recursive traversal code for namespace hierarchy module RecursiveTraversal extend ActiveSupport::Concern diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb index bdb90b5d2b9..d010d68600d 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -10,6 +10,9 @@ module Gitlab include EachBatch include Concerns::MentionableMigrationMethods + self.table_name = 'design_management_designs' + self.inheritance_column = :_type_disabled + def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb index 68205ecd3c2..eb00f6cfa3f 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class DesignUserMention < ActiveRecord::Base self.table_name = 'design_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :design_id diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index 61d9244a4c9..cfd9a4faa9b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -17,10 +17,10 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'epics' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :project - belongs_to :group + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb index 4e3ce9bf3a7..579e4d99612 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class EpicUserMention < ActiveRecord::Base self.table_name = 'epic_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :epic_id diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb index bc04172b9a2..a8b4b59b06c 100644 --- a/lib/gitlab/background_migration/user_mentions/models/group.rb +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -7,6 +7,8 @@ module Gitlab # isolated Group model class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace self.store_full_sti_class = false + self.inheritance_column = :_type_disabled + has_one :saml_provider def self.declarative_policy_class diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb index 6b52afea17c..13addcc3c55 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -17,10 +17,11 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'merge_requests' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :target_project, class_name: "Project" - belongs_to :source_project, class_name: "Project" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" + belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" alias_attribute :project, :target_project diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb index e9b85e9cb8c..4a85892d7b8 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class MergeRequestUserMention < ActiveRecord::Base self.table_name = 'merge_request_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :merge_request_id diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 8fa0db5fd4b..a2b50c41f4a 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -5,9 +5,11 @@ module Gitlab module UserMentions module Models # isolated Namespace model - class Namespace < ApplicationRecord - include FeatureGate - include ::Gitlab::VisibilityLevel + class Namespace < ActiveRecord::Base + self.inheritance_column = :_type_disabled + + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal @@ -21,8 +23,13 @@ module Gitlab parent_id.present? || parent.present? end + # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. + def feature_available?(feature) + licensed_feature_available?(feature) + end + # Overridden in EE::Namespace - def feature_available?(_feature) + def licensed_feature_available?(_feature) false end end diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index a3224c8c456..7da933c7b11 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -16,9 +16,9 @@ module Gitlab attr_mentionable :note, pipeline: :note cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true - belongs_to :author, class_name: "User" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" belongs_to :noteable, polymorphic: true - belongs_to :project + belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" def for_personal_snippet? noteable && noteable.class.name == 'PersonalSnippet' diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb new file mode 100644 index 00000000000..4e02bf97d12 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/project.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Project < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" + belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + alias_method :parent, :namespace + + # Returns a collection of projects that is either public or visible to the + # logged in user. + def self.public_or_visible_to_user(user = nil, min_access_level = nil) + min_access_level = nil if user&.can_read_all_resources? + + return public_to_user unless user + + if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User) + where('EXISTS (?) OR projects.visibility_level IN (?)', + user.authorizations_for_projects(min_access_level: min_access_level), + levels_for_user(user)) + end + end + + def grafana_integration + nil + end + + def default_issues_tracker? + true # we do not care of the issue tracker type(internal or external) when parsing mentions + end + + def visibility_level_field + :visibility_level + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb new file mode 100644 index 00000000000..a30220b6934 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/user.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class User < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + + self.table_name = 'users' + self.inheritance_column = :_type_disabled + + has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id') + authorizations = project_authorizations + .select(1) + .where("project_authorizations.project_id = #{related_project_column}") + + return authorizations unless min_access_level.present? + + authorizations.where('project_authorizations.access_level >= ?', min_access_level) + end + + def can_read_all_resources? + can?(:read_all_resources) + end + + def can?(action, subject = :global) + Ability.allowed?(self, action, subject) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb index baacc912df3..665ad7abcbb 100644 --- a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb +++ b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb @@ -27,7 +27,7 @@ module Gitlab joins(:user) .merge(UserModel.active) .where(id: (start_id..stop_id)) - .where('emails.confirmed_at IS NOT NULL') + .where.not('emails.confirmed_at' => nil) .where('emails.confirmed_at = users.confirmed_at') .where('emails.email <> users.email') .where('NOT EXISTS (SELECT 1 FROM user_synced_attributes_metadata WHERE user_id=users.id AND email_synced IS true)') @@ -57,7 +57,7 @@ module Gitlab def update_email_records(start_id, stop_id) EmailModel.connection.execute <<-SQL - WITH md5_strings as ( + WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{email_query_for_update(start_id, stop_id).to_sql} ) UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)} diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb index e18f1320ea4..62fc8cd048e 100644 --- a/lib/gitlab/batch_pop_queueing.rb +++ b/lib/gitlab/batch_pop_queueing.rb @@ -46,7 +46,8 @@ module Gitlab def initialize(namespace, queue_id) raise ArgumentError if namespace.empty? || queue_id.empty? - @namespace, @queue_id = namespace, queue_id + @namespace = namespace + @queue_id = queue_id end ## diff --git a/lib/gitlab/bullet.rb b/lib/gitlab/bullet.rb new file mode 100644 index 00000000000..f5f8a316855 --- /dev/null +++ b/lib/gitlab/bullet.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Bullet + extend self + + def enabled? + Gitlab::Utils.to_boolean(ENV['ENABLE_BULLET'], default: false) + end + alias_method :extra_logging_enabled?, :enabled? + + def configure_bullet? + defined?(::Bullet) && (enabled? || Rails.env.development?) + end + end +end diff --git a/lib/gitlab/bullet/exclusions.rb b/lib/gitlab/bullet/exclusions.rb new file mode 100644 index 00000000000..f897ff492d9 --- /dev/null +++ b/lib/gitlab/bullet/exclusions.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Bullet + class Exclusions + def initialize(config_file = Gitlab.root.join('config/bullet.yml')) + @config_file = config_file + end + + def execute + exclusions.map { |v| v['exclude'] } + end + + def validate_paths! + exclusions.each do |properties| + next unless properties['path_with_method'] + + file = properties['exclude'].first + + raise "Bullet: File used by #{config_file} doesn't exist, validate the #{file} exclusion!" unless File.exist?(file) + end + end + + private + + attr_reader :config_file + + def exclusions + @exclusions ||= if File.exist?(config_file) + YAML.load_file(config_file)['exclusions']&.values || [] + else + [] + end + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index d981f263c5e..9e958eb52fb 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -69,7 +69,9 @@ module Gitlab def load_from_project return unless commit - self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch + self.sha = commit.sha + self.status = commit.status + self.ref = project.default_branch end # We only cache the status for the HEAD commit of a project diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index 105050936ce..be8009750da 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -17,7 +17,24 @@ module Gitlab # The default template to use for generating release sections. DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) - attr_accessor :date_format, :categories, :template + # The regex to use for extracting the version from a Git tag. + # + # This regex is based on the official semantic versioning regex (as found + # on https://semver.org/), with the addition of allowing a "v" at the + # start of a tag name. + # + # We default to a strict regex as we simply don't know what kind of data + # users put in their tags. As such, using simpler patterns (e.g. just + # `\d+` for the major version) could lead to unexpected results. + # + # We use a String here as `Gitlab::UntrustedRegexp` is a mutable object. + DEFAULT_TAG_REGEX = '^v?(?P<major>0|[1-9]\d*)' \ + '\.(?P<minor>0|[1-9]\d*)' \ + '\.(?P<patch>0|[1-9]\d*)' \ + '(?:-(?P<pre>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))' \ + '?(?:\+(?P<meta>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' + + attr_accessor :date_format, :categories, :template, :tag_regex def self.from_git(project) if (yaml = project.repository.changelog_config) @@ -46,6 +63,10 @@ module Gitlab end end + if (regex = hash['tag_regex']) + config.tag_regex = regex + end + config end @@ -54,6 +75,7 @@ module Gitlab @date_format = DEFAULT_DATE_FORMAT @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) @categories = {} + @tag_regex = DEFAULT_TAG_REGEX end def contributor?(user) diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 029a9210dc9..495f12882e5 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -43,9 +43,9 @@ module Gitlab Kernel.sleep(duration_s) end - # Kill will send a SIGKILL signal to the current process - def self.kill - Process.kill("KILL", Process.pid) + # Kill will send the given signal to the current process. + def self.kill(signal) + Process.kill(signal, Process.pid) end def self.run_gc diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index c5afb16ab1a..88d624503df 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -17,7 +17,9 @@ module Gitlab attr_reader :stream, :path, :full_version def initialize(stream, path, **opts) - @stream, @path, @opts = stream, path, opts + @stream = stream + @path = path + @opts = opts @full_version = read_version end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index d3f030c3b36..23b0c93a3ee 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,12 +17,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root + attr_reader :root, :context, :ref - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil) @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) + @ref = ref + @config = expand_config(config) @root = Entry::Root.new(@config) @@ -94,9 +96,7 @@ module Gitlab initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash - initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash - - initial_config + Config::EdgeStagesInjector.new(initial_config).to_hash end def find_sha(project) diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index cf599ce5294..f9688c500d2 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -8,8 +8,8 @@ module Gitlab # Entry that represents a cache configuration # class Cache < ::Gitlab::Config::Entry::Simplifiable - strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) } - strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) } + strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) } + strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) } class Caches < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable @@ -17,8 +17,6 @@ module Gitlab MULTIPLE_CACHE_LIMIT = 4 validations do - validates :config, presence: true - validate do unless config.is_a?(Hash) || config.is_a?(Array) errors.add(:config, 'can only be a Hash or an Array') diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 9584d19bdec..947b6787aa0 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -124,7 +124,9 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - variables: root_and_job_variables_value, + variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: job_variables, + root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, resource_group: resource_group }.compact @@ -139,6 +141,14 @@ module Gitlab root_variables.merge(variables_value.to_h) end + def job_variables + variables_value.to_h + end + + def root_variables_inheritance + inherit_entry&.variables_entry&.value + end + def manual_action? self.when == 'manual' end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index aa34cfb3acc..e869e0bbb31 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -25,8 +25,7 @@ module Gitlab def value @config - .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } - .to_h + .to_h { |key, value| [key.to_s, Array(value).map(&:to_s)] } end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index dc164d752be..efb469ee32a 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -18,7 +18,7 @@ module Gitlab end def value - Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }] + @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } end def self.default(**) @@ -26,7 +26,7 @@ module Gitlab end def value_with_data - Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }] + @config.to_h { |key, value| [key.to_s, expand_value(value)] } end def use_value_data? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index b85b7a9edeb..3216d4eaac4 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) @@ -63,6 +64,17 @@ module Gitlab end end + def expand_wildcard_paths(location) + return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml) + + # We only support local files for wildcard paths + return location unless location[:local] && location[:local].include?('*') + + context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| + { local: path } + end + end + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) { remote: location } diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index 5a23836d8a0..5cabbc86d3e 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -43,9 +43,10 @@ module Gitlab { name: name, instance: instance, - variables: variables, + variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: variables, parallel: { total: total } - } + }.compact end def name diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c811ef211d6..12e182b38fc 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.instance_variables_ui_enabled? - ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) - end - def self.pipeline_latest? ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) end @@ -60,16 +56,12 @@ module Gitlab ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) end - def self.display_codequality_backend_comparison?(project) - ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) - end - def self.multiple_cache_per_job? ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) end - def self.ci_commit_pipeline_mini_graph_vue_enabled?(project) - ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml) + def self.gldropdown_tags_enabled? + ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index af06e124736..a6ae249fa58 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -72,16 +72,16 @@ module Gitlab def key @key ||= begin - key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) - Gitlab::CurrentSettings.ci_jwt_signing_key - else - Rails.application.secrets.openid_connect_signing_key - end + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + Gitlab::CurrentSettings.ci_jwt_signing_key + else + Rails.application.secrets.openid_connect_signing_key + end - raise NoSigningKeyError unless key_data + raise NoSigningKeyError unless key_data - OpenSSL::PKey::RSA.new(key_data) - end + OpenSSL::PKey::RSA.new(key_data) + end end def public_key diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 815fe6bac6d..c3c1728602c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -12,7 +12,7 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :pipeline_seed + :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do include Gitlab::Utils::StrongMemoize @@ -84,7 +84,7 @@ module Gitlab end def metrics - @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new + @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end def observe_creation_duration(duration) @@ -97,6 +97,11 @@ module Gitlab .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + def increment_pipeline_failure_reason_counter(reason) + metrics.pipeline_failure_reason_counter + .increment(reason: (reason || :unknown_failure).to_s) + end + def dangling_build? %i[ondemand_dast_scan webide].include?(source) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index c3fbd0c9e24..8f1c49563f2 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -14,6 +14,7 @@ module Gitlab result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, + ref: @pipeline.ref, sha: @pipeline.sha, user: current_user, parent_pipeline: parent_pipeline diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index 3c910963a2a..cceaa52de16 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -9,6 +9,8 @@ module Gitlab include Chain::Helpers def perform! + @command.workflow_rules_result = workflow_rules_result + error('Pipeline filtered out by workflow rules.') unless workflow_passed? end @@ -19,27 +21,33 @@ module Gitlab private def workflow_passed? - strong_memoize(:workflow_passed) do - workflow_rules.evaluate(@pipeline, global_context).pass? + workflow_rules_result.pass? + end + + def workflow_rules_result + strong_memoize(:workflow_rules_result) do + workflow_rules.evaluate(@pipeline, global_context) end end def workflow_rules Gitlab::Ci::Build::Rules.new( - workflow_config[:rules], default_when: 'always') + workflow_rules_config, default_when: 'always') end def global_context Gitlab::Ci::Build::Context::Global.new( - @pipeline, yaml_variables: workflow_config[:yaml_variables]) + @pipeline, yaml_variables: @command.yaml_processor_result.root_variables) end def has_workflow_rules? - workflow_config[:rules].present? + workflow_rules_config.present? end - def workflow_config - @command.yaml_processor_result.workflow_attributes || {} + def workflow_rules_config + strong_memoize(:workflow_rules_config) do + @command.yaml_processor_result.workflow_rules + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index d7271df1694..9988b6f18ed 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -12,7 +12,8 @@ module Gitlab end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? + + drop_pipeline!(drop_reason) # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -24,8 +25,21 @@ module Gitlab pipeline.add_warning_message(message) end - def persist_pipeline? - command.save_incompleted && !pipeline.readonly? + private + + def drop_pipeline!(drop_reason) + return if pipeline.readonly? + + if drop_reason && command.save_incompleted + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the state transition transaction of the drop! call + # which it will lock the InternalId row for the whole transaction + pipeline.ensure_project_iid! + + pipeline.drop!(drop_reason) + else + command.increment_pipeline_failure_reason_counter(drop_reason) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb index 0d7449813b4..b17ae77d445 100644 --- a/lib/gitlab/ci/pipeline/chain/metrics.rb +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -14,7 +14,7 @@ module Gitlab end def counter - ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter end end end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb index 1eb7474e915..c1b6dfb7e36 100644 --- a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -8,9 +8,7 @@ module Gitlab # After pipeline has been successfully created we can start processing it. class Process < Chain::Base def perform! - ::Ci::ProcessPipelineService - .new(@pipeline) - .execute + ::Ci::InitialPipelineProcessWorker.perform_async(pipeline.id) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 7b537125b9b..66fc6741252 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -11,6 +11,10 @@ module Gitlab def perform! raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result + if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) + raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result + end + # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! pipeline.ensure_ci_ref! @@ -38,7 +42,21 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(pipeline, stages_attributes) + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end + end + + def context + Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) + end + + def root_variables + if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + else + @command.yaml_processor_result.root_variables end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index d056501a6d3..6149d2f04d7 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -10,77 +10,116 @@ module Gitlab InvalidResponseCode = Class.new(StandardError) - VALIDATION_REQUEST_TIMEOUT = 5 + DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5 + ACCEPTED_STATUS = 200 + DOT_COM_REJECTED_STATUS = 406 + GENERAL_REJECTED_STATUS = (400..499).freeze def perform! + return unless enabled? + pipeline_authorized = validate_external log_message = pipeline_authorized ? 'authorized' : 'not authorized' - Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: @pipeline.project.id, user_id: @pipeline.user.id) + Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: project.id, user_id: current_user.id) error('External validation failed', drop_reason: :external_validation_failure) unless pipeline_authorized end def break? - @pipeline.errors.any? + pipeline.errors.any? end private + def enabled? + return true unless Gitlab.com? + + ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml) + end + def validate_external return true unless validation_service_url # 200 - accepted - # 4xx - not accepted + # 406 - not accepted on GitLab.com + # 4XX - not accepted for other installations # everything else - accepted and logged response_code = validate_service_request.code case response_code - when 200 + when ACCEPTED_STATUS true - when 400..499 + when rejected_status false else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" end rescue => ex - Gitlab::ErrorTracking.track_exception(ex) + Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) true end + def rejected_status + if Gitlab.com? + DOT_COM_REJECTED_STATUS + else + GENERAL_REJECTED_STATUS + end + end + def validate_service_request + headers = { + 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id, + 'X-Gitlab-Token' => validation_service_token + }.compact + Gitlab::HTTP.post( - validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, - body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes) + validation_service_url, timeout: validation_service_timeout, + headers: headers, + body: validation_service_payload.to_json ) end + def validation_service_timeout + timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i + return timeout if timeout > 0 + + DEFAULT_VALIDATION_REQUEST_TIMEOUT + end + def validation_service_url - ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + end + + def validation_service_token + Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN'] end - def validation_service_payload(pipeline, stages_attributes) + def validation_service_payload { project: { - id: pipeline.project.id, - path: pipeline.project.full_path + id: project.id, + path: project.full_path, + created_at: project.created_at&.iso8601 }, user: { - id: pipeline.user.id, - username: pipeline.user.username, - email: pipeline.user.email + id: current_user.id, + username: current_user.username, + email: current_user.email, + created_at: current_user.created_at&.iso8601 }, pipeline: { sha: pipeline.sha, ref: pipeline.ref, type: pipeline.source }, - builds: builds_validation_payload(stages_attributes) - }.to_json + builds: builds_validation_payload + } end - def builds_validation_payload(stages_attributes) - stages_attributes.map { |stage| stage[:builds] }.flatten + def builds_validation_payload + stages_attributes.flat_map { |stage| stage[:builds] } .map(&method(:build_validation_payload)) end @@ -97,9 +136,15 @@ module Gitlab ].flatten.compact } end + + def stages_attributes + command.yaml_processor_result.stages_attributes + end end end end end end end + +Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index c77f4dcca5a..6cb6fd3920d 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -4,55 +4,57 @@ module Gitlab module Ci module Pipeline class Metrics - include Gitlab::Utils::StrongMemoize + def self.pipeline_creation_duration_histogram + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] - def pipeline_creation_duration_histogram - strong_memoize(:pipeline_creation_duration_histogram) do - name = :gitlab_ci_pipeline_creation_duration_seconds - comment = 'Pipeline creation duration' - labels = {} - buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.pipeline_size_histogram + name = :gitlab_ci_pipeline_size_builds + comment = 'Pipeline size' + labels = { source: nil } + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.pipeline_processing_events_counter + name = :gitlab_ci_pipeline_processing_events_total + comment = 'Total amount of pipeline processing events' - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + Gitlab::Metrics.counter(name, comment) end - def pipeline_size_histogram - strong_memoize(:pipeline_size_histogram) do - name = :gitlab_ci_pipeline_size_builds - comment = 'Pipeline size' - labels = { source: nil } - buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + def self.pipelines_created_counter + name = :pipelines_created_total + comment = 'Counter of pipelines created' - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + Gitlab::Metrics.counter(name, comment) end - def pipeline_processing_events_counter - strong_memoize(:pipeline_processing_events_counter) do - name = :gitlab_ci_pipeline_processing_events_total - comment = 'Total amount of pipeline processing events' + def self.legacy_update_jobs_counter + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end - def pipelines_created_counter - strong_memoize(:pipelines_created_count) do - name = :pipelines_created_total - comment = 'Counter of pipelines created' + def self.pipeline_failure_reason_counter + name = :gitlab_ci_pipeline_failure_reasons + comment = 'Counter of pipeline failure reasons' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end - def legacy_update_jobs_counter - strong_memoize(:legacy_update_jobs_counter) do - name = :ci_legacy_update_jobs_as_retried_total - comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + def self.job_failure_reason_counter + name = :gitlab_ci_job_failure_reasons + comment = 'Counter of job failure reasons' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 11b01822e4b..39dee7750d6 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,12 +11,15 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(pipeline, attributes, previous_stages) - @pipeline = pipeline + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline @seed_attributes = attributes @previous_stages = previous_stages @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) + @job_variables = @seed_attributes.delete(:job_variables) + @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @using_rules = attributes.key?(:rules) @using_only = attributes.key?(:only) @@ -29,7 +32,9 @@ module Gitlab @rules = Gitlab::Ci::Build::Rules .new(attributes.delete(:rules), default_when: 'on_success') @cache = Gitlab::Ci::Build::Cache - .new(attributes.delete(:cache), pipeline) + .new(attributes.delete(:cache), @pipeline) + + recalculate_yaml_variables! end def name @@ -206,6 +211,14 @@ module Gitlab { options: { allow_failure_criteria: nil } } end + + def recalculate_yaml_variables! + return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml) + + @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb new file mode 100644 index 00000000000..6194a78f682 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/context.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Context + attr_reader :pipeline, :root_variables + + def initialize(pipeline, root_variables: []) + @pipeline = pipeline + @root_variables = root_variables + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index da9d853cf68..e1a15fb8d5b 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -7,8 +7,8 @@ module Gitlab class Pipeline include Gitlab::Utils::StrongMemoize - def initialize(pipeline, stages_attributes) - @pipeline = pipeline + def initialize(context, stages_attributes) + @context = context @stages_attributes = stages_attributes end @@ -37,7 +37,7 @@ module Gitlab def stage_seeds strong_memoize(:stage_seeds) do seeds = @stages_attributes.inject([]) do |previous_stages, attributes| - seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@pipeline, attributes, previous_stages) + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages) previous_stages + [seed] end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index b600df2f656..c988ea10e41 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,13 +10,14 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(pipeline, attributes, previous_stages) - @pipeline = pipeline + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline @attributes = attributes @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(@pipeline, attributes, previous_stages) + Seed::Build.new(context, attributes, previous_stages) end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 5398c19e536..7ecb9a1db16 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -9,12 +9,12 @@ module Gitlab QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze - QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze - QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze + QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' - JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze + JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5 OPERATION_COUNTERS = [ :build_can_pick, @@ -94,13 +94,13 @@ module Gitlab self.class.queue_depth_total.observe({ queue: queue }, size.to_f) end - def observe_queue_size(size_proc) + def observe_queue_size(size_proc, runner_type) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) - self.class.queue_size_total.observe({}, size_proc.call.to_f) + self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f) end - def observe_queue_time + def observe_queue_time(metric, runner_type) start_time = ::Gitlab::Metrics::System.monotonic_time result = yield @@ -108,7 +108,15 @@ module Gitlab return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time - self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f) + + case metric + when :process + self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) + when :retrieve + self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) + else + raise ArgumentError unless Rails.env.production? + end result end @@ -187,7 +195,18 @@ module Gitlab strong_memoize(:queue_iteration_duration_seconds) do name = :gitlab_ci_queue_iteration_duration_seconds comment = 'Time it takes to find a build in CI/CD queue' - buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS + buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_retrieval_duration_seconds + strong_memoize(:queue_retrieval_duration_seconds) do + name = :gitlab_ci_queue_retrieval_duration_seconds + comment = 'Time it takes to execute a SQL query to retrieve builds queue' + buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 060a1e2399b..27c41c384b8 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -6,6 +6,7 @@ module Gitlab class CodequalityReports attr_reader :degradations, :error_message + SEVERITY_PRIORITIES = %w(blocker critical major minor info).map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... } CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s def initialize @@ -29,12 +30,17 @@ module Gitlab @degradations.values end + def sort_degradations! + @degradations = @degradations.sort_by do |_fingerprint, degradation| + SEVERITY_PRIORITIES[degradation.dig(:severity)] + end.to_h + end + private def valid_degradation?(degradation) - JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation) - rescue JSON::Schema::ValidationError => e - set_error_message("Invalid degradation format: #{e.message}") + JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation) + rescue StandardError => _ false end end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb index 10748b8ca02..e34d9675c10 100644 --- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -7,6 +7,11 @@ module Gitlab def initialize(base_report, head_report) @base_report = base_report @head_report = head_report + + unless not_found? + @base_report.sort_degradations! + @head_report.sort_degradations! + end end def success? diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index c024e794ad5..37d0da38065 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -6,32 +6,32 @@ module Gitlab class TestFailureHistory include Gitlab::Utils::StrongMemoize - def initialize(failed_test_cases, project) - @failed_test_cases = build_map(failed_test_cases) + def initialize(failed_junit_tests, project) + @failed_junit_tests = build_map(failed_junit_tests) @project = project end def load! recent_failures_count.each do |key_hash, count| - failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) + failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master) end end private - attr_reader :report, :project, :failed_test_cases + attr_reader :report, :project, :failed_junit_tests def recent_failures_count - ::Ci::TestCaseFailure.recent_failures_count( + ::Ci::UnitTestFailure.recent_failures_count( project: project, - test_case_keys: failed_test_cases.keys + unit_test_keys: failed_junit_tests.keys ) end - def build_map(test_cases) + def build_map(junit_tests) {}.tap do |hash| - test_cases.each do |test_case| - hash[test_case.key] = test_case + junit_tests.each do |test| + hash[test.key] = test end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index dd0bfa768a8..365864d3317 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -51,10 +51,7 @@ module Gitlab attr_reader :errors - def initialize(current_user:, group: nil, project: nil, os:, arch:) - @current_user = current_user - @group = group - @project = project + def initialize(os:, arch:) @os = os @arch = arch @errors = [] @@ -77,7 +74,7 @@ module Gitlab server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) runner_executable = environment[:runner_executable] - "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}" + "#{runner_executable} register --url #{server_url} --registration-token $REGISTRATION_TOKEN" end end @@ -108,30 +105,6 @@ module Gitlab def get_file(path) File.read(Rails.root.join(path).to_s) end - - def registration_token - project_token || group_token || instance_token - end - - def project_token - return unless @project - raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project) - - @project.runners_token - end - - def group_token - return unless @group - raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group) - - @group.runners_token - end - - def instance_token - raise Gitlab::Access::AccessDeniedError unless @current_user&.admin? - - Gitlab::CurrentSettings.runners_registration_token - end end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index f6562737838..787dee3b267 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -26,7 +26,9 @@ module Gitlab bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', - reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines' + reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', + project_deleted: 'pipeline project was deleted', + user_blocked: 'pipeline user was blocked' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 5ebbbf15682..2ff36bcc657 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -113,9 +113,10 @@ promoteBeta: promoteProduction: extends: .promote_job stage: production - # We only allow production promotion on `master` because - # it has its own production scoped secret variables + # We only allow production promotion on the default branch because + # it has its own production scoped secret variables. only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH script: - bundle exec fastlane promote_beta_to_production diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index 15cdbf63cb1..d0c63ab6edf 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,27 +1,31 @@ -docker-build-master: - # Official docker image. - image: docker:latest - stage: build - services: - - docker:dind - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker build --pull -t "$CI_REGISTRY_IMAGE" . - - docker push "$CI_REGISTRY_IMAGE" - only: - - master - +# Build a Docker image with CI/CD and push to the GitLab registry. +# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html +# +# This template uses one generic job with conditional builds +# for the default branch and all other (MR) branches. docker-build: - # Official docker image. + # Use the official docker image. image: docker:latest stage: build services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) script: - - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . - - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" - except: - - master + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml new file mode 100644 index 00000000000..90812083917 --- /dev/null +++ b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml @@ -0,0 +1,9 @@ +# This file is a template demonstrating the `script` keyword. +# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script + +# After committing this template, visit CI/CD > Jobs to see the script output. + +job: + script: + # provide a shell script as argument for this keyword. + - echo "Hello World" diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml new file mode 100644 index 00000000000..c7fb1321055 --- /dev/null +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml @@ -0,0 +1,91 @@ +# This template is provided and maintained by Indeni, an official Technology Partner with GitLab. +# See https://about.gitlab.com/partners/technology-partners/#security for more information. + +# For more information about Indeni Cloudrail: https://indeni.com/cloudrail/ +# +# This file shows an example of using Indeni Cloudrail with GitLab CI/CD. +# It is not designed to be included in an existing CI/CD configuration with the "include:" keyword. +# Documentation about this integration: https://indeni.com/doc-indeni-cloudrail/integrate-with-ci-cd/gitlab-instructions +# +# For an example of this used in a GitLab repository, see: https://gitlab.com/indeni/cloudrail-demo/-/blob/master/.gitlab-ci.yml + +# The sast-report output complies with GitLab's format. This report displays Cloudrail's +# results in the Security tab in the pipeline view, if you have that feature enabled +# (GitLab Ultimate only). Otherwise, Cloudrail generates a JUnit report, which displays +# in the "Test summary" in merge requests. + +# Note that Cloudrail's input is the Terraform plan. That is why we've included in this +# template an example of doing that. You are welcome to replace it with your own way +# of generating a Terraform plan. + +# Before you can use this template, get a Cloudrail API key from the Cloudrail web +# user interface. Save it as a CI/CD variable named CLOUDRAIL_API_KEY in your project +# settings. + +variables: + TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +default: + before_script: + - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +stages: + - init_and_plan + - cloudrail + +init_and_plan: + stage: init_and_plan + image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - terraform init + - terraform plan -out=plan.out + artifacts: + name: "$CI_COMMIT_BRANCH-terraform_plan" + paths: + - ./**/plan.out + - ./**/.terraform + +cloudrail_scan: + stage: cloudrail + image: indeni/cloudrail-cli:1.2.44 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - | + if [[ "${GITLAB_FEATURES}" == *"security_dashboard"* ]]; then + echo "You are licensed for GitLab Security Dashboards. Your scan results will display in the Security Dashboard." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format json-gitlab-sast \ + --output-file ${CI_PROJECT_DIR}/cloudrail-sast-report.json \ + --auto-approve + else + echo "Your scan results will display in the GitLab Test results visualization panel." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format junit \ + --output-file ${CI_PROJECT_DIR}/cloudrail-junit-report.xml \ + --auto-approve + fi + artifacts: + reports: + sast: cloudrail-sast-report.json + junit: cloudrail-junit-report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 5edb26a0b56..01907ef9e2e 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -20,15 +20,48 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS fi - mv sitespeed-results/data/performance.json browser-performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..196d42f3e3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - docker:20.10.6-dind script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index fd6c51ea350..b29342216fc 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -36,6 +36,7 @@ code_quality: REPORT_STDOUT \ REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ + CODECLIMATE_PREFIX \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 654a03ced5f..bf42cd52605 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -12,7 +12,7 @@ stages: variables: FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest + FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 FUZZAPI_REPORT: gl-api-fuzzing-report.json @@ -45,7 +45,7 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 + FUZZAPI_API: http://localhost:5000 FUZZAPI_NEW_REPORT: 1 FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles @@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://apifuzzer:5000 allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null @@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd: -e TZ=America/Los_Angeles \ -e GITLAB_FEATURES \ -p 80:80 \ + -p 5000:5000 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ @@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd: docker run \ --name worker \ --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd: --name worker \ --network $FUZZAPI_D_NETWORK \ -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd: -v $CI_PROJECT_DIR:/app \ -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ -p 81:80 \ + -p 5001:5000 \ -p 8001:8000 \ -p 515:514 \ --restart=no \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..215029dc952 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -0,0 +1,270 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + FUZZAPI_PROFILE: Quick + FUZZAPI_VERSION: latest + FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml + FUZZAPI_TIMEOUT: 30 + FUZZAPI_REPORT: gl-api-fuzzing-report.json + FUZZAPI_REPORT_ASSET_PATH: assets + # + FUZZAPI_D_NETWORK: testing-net + # + # Wait up to 5 minutes for API Fuzzer and target url to become + # available (non 500 response to HTTP(s)) + FUZZAPI_SERVICE_START_TIMEOUT: "300" + # + FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + # + +apifuzzer_fuzz_unlicensed: + stage: fuzz + allow_failure: true + rules: + - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' + - when: never + script: + - | + echo "Error: Your GitLab project is not licensed for API Fuzzing." + - exit 1 + +apifuzzer_fuzz: + stage: fuzz + image: + name: $FUZZAPI_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://localhost:80 + FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE + when: never + - if: $FUZZAPI_D_WORKER_IMAGE + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + script: + # + # Validate options + - | + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER + reports: + api_fuzzing: $FUZZAPI_REPORT + +apifuzzer_fuzz_dnd: + stage: fuzz + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + services: + - docker:19.03.12-dind + script: + # + # + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # + - docker network create --driver bridge $FUZZAPI_D_NETWORK + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start peach testing engine container + - | + docker run -d \ + --name apifuzzer \ + --network $FUZZAPI_D_NETWORK \ + -e Proxy:Port=8000 \ + -e TZ=America/Los_Angeles \ + -e GITLAB_FEATURES \ + -p 80:80 \ + -p 8000:8000 \ + -p 514:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + dotnet /peach/Peach.Web.dll + # + # Start target container + - | + if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ + docker run -d \ + --name target \ + --network $FUZZAPI_D_NETWORK \ + $FUZZAPI_D_TARGET_ENV \ + $FUZZAPI_D_TARGET_PORTS \ + $FUZZAPI_D_TARGET_VOLUME \ + --restart=no \ + $FUZZAPI_D_TARGET_IMAGE \ + ; fi + # + # Start worker container if provided + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ + $FUZZAPI_D_WORKER_ENV \ + $FUZZAPI_D_WORKER_PORTS \ + $FUZZAPI_D_WORKER_VOLUME \ + --restart=no \ + $FUZZAPI_D_WORKER_IMAGE \ + ; fi + # + # Start API Fuzzing provided worker if no other worker present + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -v $CI_PROJECT_DIR:/app \ + -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ + -p 81:80 \ + -p 8001:8000 \ + -p 515:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + worker-entry \ + ; fi + # + # Propagate exit code from api fuzzing scanner (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + after_script: + # + # Shutdown all containers + - echo "Stopping all containers" + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi + - docker stop worker + - docker stop apifuzzer + # + # Save docker logs + - docker logs apifuzzer &> gl-api_fuzzing-logs.log + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi + - docker logs worker &> gl-api_fuzzing-worker-logs.log + # + artifacts: + when: always + paths: + - ./gl-api_fuzzing*.log + - ./gl-api_fuzzing*.zip + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + reports: + api_fuzzing: $FUZZAPI_REPORT + +# end diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 64001c2828a..c628e30b2c7 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -6,14 +6,10 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" CS_MAJOR_VERSION: 3 -container_scanning: +.cs_common: stage: test image: "$CS_ANALYZER_IMAGE" variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details @@ -21,19 +17,44 @@ container_scanning: # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependencies: [] + +container_scanning: + extends: .cs_common + variables: + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image + # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes + CLAIR_DB_IMAGE_TAG: "latest" + CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" + CS_PROJECT: 'klar' services: - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: - /analyzer run + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION =~ /^[0-3]$/ + +container_scanning_new: + extends: .cs_common + variables: + CS_PROJECT: 'container-scanning' + script: + - gtcs scan artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependencies: [] + paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION !~ /^[0-3]$/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index fc1acd09714..533f8bb25f8 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,3 +1,16 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST: +# +# stages: +# - build +# - test +# - deploy +# - dast + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ # Configure the scanning tool through the environment variables. @@ -9,6 +22,19 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_PROFILE: Full + DAST_API_VERSION: latest + DAST_API_CONFIG: .gitlab-dast-api.yml + DAST_API_TIMEOUT: 30 + DAST_API_REPORT: gl-dast-api-report.json + DAST_API_REPORT_ASSET_PATH: assets + # + # Wait up to 5 minutes for API Security and target url to become + # available (non 500 response to HTTP(s)) + DAST_API_SERVICE_START_TIMEOUT: "300" + # + DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -25,6 +51,11 @@ dast: reports: dast: gl-dast-report.json rules: + - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || + $DAST_API_OPENAPI || + $DAST_API_POSTMAN_COLLECTION || + $DAST_API_HAR ) + when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -40,4 +71,72 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && + $DAST_API_BETA == null && $DAST_API_SPECIFICATION + +dast_api: + stage: dast + image: + name: $DAST_API_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + API_SECURITY_MODE: DAST + DAST_API_NEW_REPORT: 1 + DAST_API_PROJECT: $CI_PROJECT_PATH + DAST_API_API: http://127.0.0.1:5000 + DAST_API_LOG_SCANNER: gl-dast-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $DAST_API_BETA == null + when: never + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && + $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdast\b/ + script: + # + # Run user provided pre-script + - sh -c "$DAST_API_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $DAST_API_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$DAST_API_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $DAST_API_REPORT_ASSET_PATH + - $DAST_API_REPORT + - $DAST_API_LOG_SCANNER + - gl-*.log + reports: + dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 9693a4fbca2..3ebccfbba4a 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -160,7 +160,7 @@ mobsf-android-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -186,7 +186,7 @@ mobsf-ios-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -303,6 +303,10 @@ semgrep-sast: $SAST_EXPERIMENTAL_FEATURES == 'true' exists: - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' sobelow-sast: extends: .sast-analyzer @@ -348,3 +352,4 @@ spotbugs-sast: - '**/*.groovy' - '**/*.java' - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e591e3cc1e2..404d4a4c6db 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -18,9 +18,32 @@ performance: - docker:stable-dind script: - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 3258d965c93..c25c4339c35 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -11,7 +11,7 @@ module Gitlab LOCK_SLEEP = 0.001.seconds WATCH_FLAG_TTL = 10.seconds - UPDATE_FREQUENCY_DEFAULT = 30.seconds + UPDATE_FREQUENCY_DEFAULT = 60.seconds UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds ArchiveError = Class.new(StandardError) @@ -93,6 +93,10 @@ module Gitlab end end + def erase_trace_chunks! + job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + end + def erase! ## # Erase the archived trace @@ -100,7 +104,7 @@ module Gitlab ## # Erase the live trace - job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + erase_trace_chunks! FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace ensure @@ -114,7 +118,11 @@ module Gitlab end def update_interval - being_watched? ? UPDATE_FREQUENCY_WHEN_BEING_WATCHED : UPDATE_FREQUENCY_DEFAULT + if being_watched? + UPDATE_FREQUENCY_WHEN_BEING_WATCHED + else + UPDATE_FREQUENCY_DEFAULT + end end def being_watched! @@ -176,9 +184,14 @@ module Gitlab end def unsafe_archive! - raise AlreadyArchivedError, 'Could not archive again' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? + if trace_artifact + unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml) + + raise AlreadyArchivedError, 'Could not archive again' + end + if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) @@ -197,6 +210,18 @@ module Gitlab end end + def unsafe_trace_cleanup! + return unless trace_artifact + + if trace_artifact.archived_trace_exists? + # An archive already exists, so make sure to remove the trace chunks + erase_trace_chunks! + else + # An archive already exists, but its associated file does not, so remove it + trace_artifact.destroy! + end + end + def in_write_lock(&blk) lock_key = "trace:write:lock:#{job.id}" in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk) diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index e2a54f90ecb..3a62f01e2e3 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -23,7 +23,21 @@ module Gitlab def transform_from_yaml_variables(vars) return vars.stringify_keys if vars.is_a?(Hash) - vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + end + + def inherit_yaml_variables(from:, to:, inheritance:) + merge_variables(apply_inheritance(from, inheritance), to) + end + + private + + def apply_inheritance(variables, inheritance) + case inheritance + when true then variables + when false then {} + when Array then variables.select { |var| inheritance.include?(var[:key]) } + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 3459b69bebc..f96a6629849 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -38,11 +38,12 @@ module Gitlab .map { |job| build_attributes(job[:name]) } end - def workflow_attributes - { - rules: hash_config.dig(:workflow, :rules), - yaml_variables: transform_to_yaml_variables(variables) - } + def workflow_rules + @workflow_rules ||= hash_config.dig(:workflow, :rules) + end + + def root_variables + @root_variables ||= transform_to_yaml_variables(variables) end def jobs @@ -68,7 +69,9 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), + yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: transform_to_yaml_variables(job[:job_variables]), + root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], only: job[:only], @@ -101,7 +104,7 @@ module Gitlab end def merged_yaml - @ci_config&.to_hash&.to_yaml + @ci_config&.to_hash&.deep_stringify_keys&.to_yaml end def variables_with_data diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index ac0071cdc53..fdff8fb32d3 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -28,20 +28,34 @@ module Gitlab def package_metadata(package) json = package.composer_metadatum.composer_json - json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version) + json.merge( + 'dist' => package_dist(package), + 'source' => package_source(package), + 'uid' => package.id, + 'version' => package.version + ) end def package_dist(package) - sha = package.composer_metadatum.target_sha archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true) { 'type' => 'zip', - 'url' => expose_url(archive_api_path) + "?sha=#{sha}", - 'reference' => sha, + 'url' => expose_url(archive_api_path) + "?sha=#{package.composer_target_sha}", + 'reference' => package.composer_target_sha, 'shasum' => '' } end + + def package_source(package) + git_url = package.project.http_url_to_repo + + { + 'type' => 'git', + 'url' => git_url, + 'reference' => package.composer_target_sha + } + end end end end diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index d03997b4158..c3d90aa78fb 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -7,7 +7,7 @@ module Gitlab class ConanToken - HMAC_KEY = 'gitlab-conan-packages'.freeze + HMAC_KEY = 'gitlab-conan-packages' attr_reader :access_token_id, :user_id diff --git a/lib/gitlab/contributor.rb b/lib/gitlab/contributor.rb index d74d5a86aa0..c1c270bc9e6 100644 --- a/lib/gitlab/contributor.rb +++ b/lib/gitlab/contributor.rb @@ -5,7 +5,9 @@ module Gitlab attr_accessor :email, :name, :commits, :additions, :deletions def initialize - @commits, @additions, @deletions = 0, 0, 0 + @commits = 0 + @additions = 0 + @deletions = 0 end end end diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 4428354642d..c113cebd72f 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -16,34 +16,16 @@ module Gitlab ::Digest::SHA256.base64digest("#{value}#{salt}") end - def aes256_gcm_encrypt(value, nonce: nil) - aes256_gcm_encrypt_using_static_nonce(value) + def aes256_gcm_encrypt(value, nonce: AES256_GCM_IV_STATIC) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: nonce)) + Base64.strict_encode64(encrypted_token) end - def aes256_gcm_decrypt(value) + def aes256_gcm_decrypt(value, nonce: AES256_GCM_IV_STATIC) return unless value - nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC encrypted_token = Base64.decode64(value) - decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) - decrypted_token - end - - def dynamic_nonce(value) - TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC - end - - def aes256_gcm_encrypt_using_static_nonce(value) - create_encrypted_token(value, AES256_GCM_IV_STATIC) - end - - def read_only? - Gitlab::Database.read_only? - end - - def create_encrypted_token(value, iv) - encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv)) - Base64.strict_encode64(encrypted_token) + Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index c4af5e6608e..0e4fc8efa95 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,7 +12,7 @@ module Gitlab author_url = build_author_url(build.commit, commit) - data = { + { object_kind: 'build', ref: build.ref, @@ -26,6 +26,7 @@ module Gitlab build_name: build.name, build_stage: build.stage, build_status: build.status, + build_created_at: build.created_at, build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, @@ -66,8 +67,6 @@ module Gitlab environment: build_environment(build) } - - data end private @@ -84,7 +83,6 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 7fd1b9cd228..a56029c0d1d 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -77,7 +77,6 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb new file mode 100644 index 00000000000..7c45f416638 --- /dev/null +++ b/lib/gitlab/database/as_with_materialized.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries. + class AsWithMaterialized < Arel::Nodes::Binary + extend Gitlab::Utils::StrongMemoize + + MATERIALIZED = Arel.sql(' MATERIALIZED') + EMPTY_STRING = Arel.sql('') + attr_reader :expr + + def initialize(left, right, materialized: true) + @expr = if materialized && self.class.materialized_supported? + MATERIALIZED + else + EMPTY_STRING + end + + super(left, right) + end + + # Note: to be deleted after the minimum PG version is set to 12.0 + def self.materialized_supported? + strong_memoize(:materialized_supported) do + Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above + end + end + + # Note: to be deleted after the minimum PG version is set to 12.0 + def self.materialized_if_supported + materialized_supported? ? 'MATERIALIZED' : '' + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb new file mode 100644 index 00000000000..3e6d7ac3c9f --- /dev/null +++ b/lib/gitlab/database/background_migration/batch_metrics.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchMetrics + attr_reader :timings + + def initialize + @timings = {} + end + + def time_operation(label) + start_time = monotonic_time + + yield + + timings_for_label(label) << monotonic_time - start_time + end + + private + + def timings_for_label(label) + timings[label] ||= [] + end + + def monotonic_time + Gitlab::Metrics::System.monotonic_time + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 0c9add9b355..4aa33ed7946 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -5,7 +5,7 @@ module Gitlab module BackgroundMigration class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' - BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze + BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" self.table_name = :batched_background_migrations @@ -23,8 +23,15 @@ module Gitlab finished: 3 } - def interval_elapsed? - last_job.nil? || last_job.created_at <= Time.current - interval + def self.active_migration + active.queue_order.first + end + + def interval_elapsed?(variance: 0) + return true unless last_job + + interval_with_variance = interval - variance + last_job.created_at <= Time.current - interval_with_variance end def create_batched_job!(min, max) @@ -50,6 +57,13 @@ module Gitlab def batch_class_name=(class_name) write_attribute(:batch_class_name, class_name.demodulize) end + + def prometheus_labels + @prometheus_labels ||= { + migration_id: id, + migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name] + } + end end end end diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 5f8a5ec06a5..cf8b61f5feb 100644 --- a/lib/gitlab/database/background_migration/scheduler.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -3,12 +3,22 @@ module Gitlab module Database module BackgroundMigration - class Scheduler - def perform(migration_wrapper: BatchedMigrationWrapper.new) - active_migration = BatchedMigration.active.queue_order.first - - return unless active_migration&.interval_elapsed? + class BatchedMigrationRunner + def initialize(migration_wrapper = BatchedMigrationWrapper.new) + @migration_wrapper = migration_wrapper + end + # Runs the next batched_job for a batched_background_migration. + # + # The batch bounds of the next job are calculated at runtime, based on the migration + # configuration and the bounds of the most recently created batched_job. Updating the + # migration configuration will cause future jobs to use the updated batch sizes. + # + # The job instance will automatically receive a set of arguments based on the migration + # configuration. For more details, see the BatchedMigrationWrapper class. + # + # Note that this method is primarily intended to called by a scheduled worker. + def run_migration_job(active_migration) if next_batched_job = create_next_batched_job!(active_migration) migration_wrapper.perform(next_batched_job) else @@ -16,8 +26,26 @@ module Gitlab end end + # Runs all remaining batched_jobs for a batched_background_migration. + # + # This method is intended to be used in a test/dev environment to execute the background + # migration inline. It should NOT be used in a real environment for any non-trivial migrations. + def run_entire_migration(migration) + unless Rails.env.development? || Rails.env.test? + raise 'this method is not intended for use in real environments' + end + + while migration.active? + run_migration_job(migration) + + migration.reload_last_job + end + end + private + attr_reader :migration_wrapper + def create_next_batched_job!(active_migration) next_batch_range = find_next_batch_range(active_migration) diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index 299bd992197..c276f8ce75b 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -4,6 +4,15 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationWrapper + extend Gitlab::Utils::StrongMemoize + + # Wraps the execution of a batched_background_migration. + # + # Updates the job's tracking records with the status of the migration + # when starting and finishing execution, and optionally saves batch_metrics + # the migration provides, if any are given. + # + # The job's batch_metrics are serialized to JSON for storage. def perform(batch_tracking_record) start_tracking_execution(batch_tracking_record) @@ -16,6 +25,7 @@ module Gitlab raise e ensure finish_tracking_execution(batch_tracking_record) + track_prometheus_metrics(batch_tracking_record) end private @@ -34,12 +44,75 @@ module Gitlab tracking_record.migration_column_name, tracking_record.sub_batch_size, *tracking_record.migration_job_arguments) + + if job_instance.respond_to?(:batch_metrics) + tracking_record.metrics = job_instance.batch_metrics + end end def finish_tracking_execution(tracking_record) tracking_record.finished_at = Time.current tracking_record.save! end + + def track_prometheus_metrics(tracking_record) + migration = tracking_record.batched_migration + base_labels = migration.prometheus_labels + + metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size) + metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size) + metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size) + + # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1) + efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f + metric_for(:histogram_time_efficiency).observe(base_labels, efficiency) + + if metrics = tracking_record.metrics + metrics['timings']&.each do |key, timings| + summary = metric_for(:histogram_timings) + labels = base_labels.merge(operation: key) + + timings.each do |timing| + summary.observe(labels, timing) + end + end + end + end + + def metric_for(name) + self.class.metrics[name] + end + + def self.metrics + strong_memoize(:metrics) do + { + gauge_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_batch_size, + 'Batch size for a batched migration job' + ), + gauge_sub_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_sub_batch_size, + 'Sub-batch size for a batched migration job' + ), + counter_updated_tuples: Gitlab::Metrics.counter( + :batched_migration_job_updated_tuples_total, + 'Number of tuples updated by batched migration job' + ), + histogram_timings: Gitlab::Metrics.histogram( + :batched_migration_job_duration_seconds, + 'Timings for a batched migration job', + {}, + [0.1, 0.25, 0.5, 1, 5].freeze + ), + histogram_time_efficiency: Gitlab::Metrics.histogram( + :batched_migration_job_time_efficiency, + 'Ratio of job duration to interval', + {}, + [0.5, 0.9, 1, 1.5, 2].freeze + ) + } + end + end end end end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 5a506da0d05..9002d39e1ee 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -88,11 +88,16 @@ module Gitlab batch_start = start while batch_start < finish - batch_end = [batch_start + batch_size, finish].min - batch_relation = build_relation_batch(batch_start, batch_end, mode) - begin - results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end, mode) + + op_args = @operation_args + if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) + op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] + end + + results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache @@ -102,6 +107,18 @@ module Gitlab log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) return FALLBACK end + rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + mode: mode, + message: "LooseIndexScanDistinctCount column error: #{error.message}" + ) + + return FALLBACK end sleep(SLEEP_TIME_IN_SECONDS) @@ -123,7 +140,11 @@ module Gitlab private def build_relation_batch(start, finish, mode) - @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + if use_loose_index_scan_for_distinct_values?(mode) + Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) + else + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + end end def batch_size_for_mode_and_operation(mode, operation) @@ -165,6 +186,14 @@ module Gitlab message: "Query has been canceled with message: #{error.message}" ) end + + def use_loose_index_scan_for_distinct_values?(mode) + Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct + end + + def not_group_by_query? + !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? + end end end end diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index 1403d561890..b1f9da30585 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -130,7 +130,7 @@ module Gitlab def sql <<~SQL - WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)}) + WITH cte(#{list_of(cte_columns)}) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (VALUES #{list_of(values)}) UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id SQL end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 89190320cf9..a7bfafe2815 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -3,10 +3,6 @@ module Gitlab module Database module Count - class PgClass < ActiveRecord::Base - self.table_name = 'pg_class' - end - # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables. # # Specifically, it relies on the column reltuples in said table. An additional @@ -74,7 +70,7 @@ module Gitlab def get_statistics(table_names, check_statistics: true) time = 6.hours.ago - query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") + query = ::Gitlab::Database::PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") .where(relname: table_names) .where('schemaname = current_schema()') .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb new file mode 100644 index 00000000000..884f4d47ff8 --- /dev/null +++ b/lib/gitlab/database/loose_index_scan_distinct_count.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This class builds efficient batched distinct query by using loose index scan. + # Consider the following example: + # > Issue.distinct(:project_id).where(project_id: (1...100)).count + # + # Note: there is an index on project_id + # + # This query will read each element in the index matching the project_id filter. + # If for a project_id has 100_000 issues, all 100_000 elements will be read. + # + # A loose index scan will read only one entry from the index for each project_id to reduce the number of disk reads. + # + # Usage: + # + # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100) + # + # The query will return the number of distinct projects_ids between 1 and 100 + # + # Getting the Arel query: + # + # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100) + class LooseIndexScanDistinctCount + COLUMN_ALIAS = 'distinct_count_column' + + ColumnConfigurationError = Class.new(StandardError) + + def initialize(scope, column) + if scope.is_a?(ActiveRecord::Relation) + @scope = scope + @model = scope.model + else + @scope = scope.where({}) + @model = scope + end + + @column = transform_column(column) + end + + def count(from:, to:) + build_query(from: from, to: to).count(COLUMN_ALIAS) + end + + def build_query(from:, to:) # rubocop:disable Metrics/AbcSize + cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false }) + table = model.arel_table + + cte << @scope + .dup + .select(column.as(COLUMN_ALIAS)) + .where(column.gteq(from)) + .where(column.lt(to)) + .order(column) + .limit(1) + + inner_query = @scope + .dup + .where(column.gt(cte.table[COLUMN_ALIAS])) + .where(column.lt(to)) + .select(column.as(COLUMN_ALIAS)) + .order(column) + .limit(1) + + cte << cte.table + .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS)) + .where(cte.table[COLUMN_ALIAS].lt(to)) + + model + .with + .recursive(cte.to_arel) + .from(cte.alias_to(table)) + .unscope(where: :source_type) + .unscope(where: model.inheritance_column) # Remove STI query, not needed here + end + + private + + attr_reader :column, :model + + # Transforms the column so it can be used in Arel expressions + # + # 'table.column' => 'table.column' + # 'column' => 'table_name.column' + # :column => 'table_name.column' + # Arel::Attributes::Attribute => name of the column + def transform_column(column) + if column.is_a?(String) || column.is_a?(Symbol) + column_as_string = column.to_s + column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.') + + Arel.sql(column_as_string) + elsif column.is_a?(Arel::Attributes::Attribute) + column + else + raise ColumnConfigurationError.new("Cannot transform the column: #{column.inspect}, please provide the column name as string") + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 31e733050e1..d06a73da8ac 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -4,6 +4,7 @@ module Gitlab module Database module MigrationHelpers include Migrations::BackgroundMigrationHelpers + include DynamicModelHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -576,17 +577,7 @@ module Gitlab # old_column - The name of the old column. # new_column - The name of the new column. def install_rename_triggers(table, old_column, new_column) - trigger_name = rename_trigger_name(table, old_column, new_column) - quoted_table = quote_table_name(table) - quoted_old = quote_column_name(old_column) - quoted_new = quote_column_name(new_column) - - install_rename_triggers_for_postgresql( - trigger_name, - quoted_table, - quoted_old, - quoted_new - ) + install_rename_triggers_for_postgresql(table, old_column, new_column) end # Changes the type of a column concurrently. @@ -927,19 +918,67 @@ module Gitlab # This is crucial for Primary Key conversions, because setting a column # as the PK converts even check constraints to NOT NULL constraints # and forces an inline re-verification of the whole table. - # - It backfills the new column with the values of the existing primary key - # by scheduling background jobs. - # - It tracks the scheduled background jobs through the use of - # Gitlab::Database::BackgroundMigrationJob + # - It sets up a trigger to keep the two columns in sync. + # + # Note: this helper is intended to be used in a regular (pre-deployment) migration. + # + # This helper is part 1 of a multi-step migration process: + # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations + # 3. remaining steps TBD, see #288005 + # + # table - The name of the database table containing the column + # column - The name of the column that we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id) + unless table_exists?(table) + raise "Table #{table} does not exist" + end + + unless column_exists?(table, primary_key) + raise "Column #{primary_key} does not exist on #{table}" + end + + unless column_exists?(table, column) + raise "Column #{column} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + old_column = column_for(table, column) + tmp_column = "#{column}_convert_to_bigint" + + with_lock_retries do + if (column.to_s == primary_key.to_s) || !old_column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) + else + add_column(table, tmp_column, :bigint, default: old_column.default) + end + + install_rename_triggers(table, column, tmp_column) + end + end + + # Backfills the new column used in the conversion of an integer column to bigint using background migrations. + # + # - This helper should be called from a post-deployment migration. + # - In order for this helper to work properly, the new column must be first initialized with + # the `initialize_conversion_of_integer_to_bigint` helper. + # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration, # which allows a more thorough check that all jobs succeeded in the # cleanup migration and is way faster for very large tables. - # - It sets up a trigger to keep the two columns in sync - # - It does not schedule a cleanup job: we have to do that with followup - # post deployment migrations in the next release. # - # This needs to be done manually by using the - # `cleanup_initialize_conversion_of_integer_to_bigint` - # (not yet implemented - check #288005) + # Note: this helper is intended to be used in a post-deployment migration, to ensure any new code is + # deployed (including background job changes) before we begin processing the background migration. + # + # This helper is part 2 of a multi-step migration process: + # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations + # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column # column - The name of the column that we want to convert to bigint. @@ -960,7 +999,7 @@ module Gitlab # and set the batch_size to 50_000 which will require # ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space # between the scheduled jobs - def initialize_conversion_of_integer_to_bigint( + def backfill_conversion_of_integer_to_bigint( table, column, primary_key: :id, @@ -969,10 +1008,6 @@ module Gitlab interval: 2.minutes ) - if transaction_open? - raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction' - end - unless table_exists?(table) raise "Table #{table} does not exist" end @@ -985,87 +1020,42 @@ module Gitlab raise "Column #{column} does not exist on #{table}" end - check_trigger_permissions!(table) - - old_column = column_for(table, column) tmp_column = "#{column}_convert_to_bigint" - with_lock_retries do - if (column.to_s == primary_key.to_s) || !old_column.null - # If the column to be converted is either a PK or is defined as NOT NULL, - # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow - # That way, we skip the expensive validation step required to add - # a NOT NULL constraint at the end of the process - add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) - else - add_column(table, tmp_column, :bigint, default: old_column.default) - end - - install_rename_triggers(table, column, tmp_column) - end - - source_model = Class.new(ActiveRecord::Base) do - include EachBatch - - self.table_name = table - self.inheritance_column = :_type_disabled + unless column_exists?(table, tmp_column) + raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`' end - queue_background_migration_jobs_by_range_at_intervals( - source_model, + batched_migration = queue_batched_background_migration( 'CopyColumnUsingBackgroundMigrationJob', - interval, + table, + primary_key, + column, + tmp_column, + job_interval: interval, batch_size: batch_size, - other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column], - track_jobs: true, - primary_column_name: primary_key - ) + sub_batch_size: sub_batch_size) if perform_background_migration_inline? # To ensure the schema is up to date immediately we perform the # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob') + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration) end end # Performs a concurrent column rename when using PostgreSQL. - def install_rename_triggers_for_postgresql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE OR REPLACE FUNCTION #{trigger}() - RETURNS trigger AS - $BODY$ - BEGIN - NEW.#{new} := NEW.#{old}; - RETURN NEW; - END; - $BODY$ - LANGUAGE 'plpgsql' - VOLATILE - EOF - - execute <<-EOF.strip_heredoc - DROP TRIGGER IF EXISTS #{trigger} - ON #{table} - EOF - - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger} - BEFORE INSERT OR UPDATE - ON #{table} - FOR EACH ROW - EXECUTE FUNCTION #{trigger}() - EOF + def install_rename_triggers_for_postgresql(table, old, new, trigger_name: nil) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) end # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) - execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") - execute("DROP FUNCTION IF EXISTS #{trigger}()") + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) end # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) - 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) end # Returns an Array containing the indexes for the given column diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index e8cbea72887..8d5ea652bfc 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -190,7 +190,7 @@ module Gitlab migration_status = batch_max_value.nil? ? :finished : :active batch_max_value ||= batch_min_value - Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( job_class_name: job_class_name, table_name: batch_table_name, column_name: batch_column_name, @@ -202,6 +202,17 @@ module Gitlab sub_batch_size: sub_batch_size, job_arguments: job_arguments, status: migration_status) + + # This guard is necessary since #total_tuple_count was only introduced schema-wise, + # after this migration helper had been used for the first time. + return migration unless migration.respond_to?(:total_tuple_count) + + # We keep track of the estimated number of tuples to reason later + # about the overall progress of a migration. + migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + migration.save! + + migration end def perform_background_migration_inline? @@ -236,6 +247,14 @@ module Gitlab Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) end + def delete_queued_jobs(class_name) + Gitlab::BackgroundMigration.steal(class_name) do |job| + job.delete + + false + end + end + private def track_in_database(class_name, arguments) diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 2def3a4d3a9..4402c42b136 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -6,6 +6,80 @@ module Gitlab module ForeignKeyHelpers include ::Gitlab::Database::SchemaHelpers + # Adds a foreign key with only minimal locking on the tables involved. + # + # In concept it works similarly to add_concurrent_foreign_key, but we have + # to add a special helper for partitioned tables for the following reasons: + # - add_concurrent_foreign_key sets the constraint to `NOT VALID` + # before validating it + # - Setting an FK to NOT VALID is not supported currently in Postgres (up to PG13) + # - Also, PostgreSQL will currently ignore NOT VALID constraints on partitions + # when adding a valid FK to the partitioned table, so they have to + # also be validated before we can add the final FK. + # Solution: + # - Add the foreign key first to each partition by using + # add_concurrent_foreign_key and validating it + # - Once all partitions have a foreign key, add it also to the partitioned + # table (there will be no need for a validation at that level) + # For those reasons, this method does not include an option to delay the + # validation, we have to force validate: true. + # + # source - The source (partitioned) table containing the foreign key. + # target - The target table the key points to. + # column - The name of the column to create the foreign key on. + # on_delete - The action to perform when associated data is removed, + # defaults to "CASCADE". + # name - The name of the foreign key. + # + def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil) + partition_options = { + column: column, + on_delete: on_delete, + + # We'll use the same FK name for all partitions and match it to + # the name used for the partitioned table to follow the convention + # used by PostgreSQL when adding FKs to new partitions + name: name.presence || concurrent_partitioned_foreign_key_name(source, column), + + # Force the FK validation to true for partitions (and the partitioned table) + validate: true + } + + if foreign_key_exists?(source, target, **partition_options) + warning_message = "Foreign key not created because it exists already " \ + "(this may be due to an aborted migration or similar): " \ + "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\ + "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}" + + Gitlab::AppLogger.warn warning_message + + return + end + + partitioned_table = find_partitioned_table(source) + + partitioned_table.postgres_partitions.order(:name).each do |partition| + add_concurrent_foreign_key(partition.identifier, target, **partition_options) + end + + with_lock_retries do + add_foreign_key(source, target, **partition_options) + end + end + + # Returns the name for a concurrent partitioned foreign key. + # + # Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers) + # we just keep a separate method in case we want a different behavior + # for partitioned tables + # + def concurrent_partitioned_foreign_key_name(table, column, prefix: 'fk_rails_') + identifier = "#{table}_#{column}_fk" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + "#{prefix}#{hashed_identifier}" + end + # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the # same functionality at the database level by using triggers. diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 1c289391e21..9ccbdc9930e 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -223,6 +223,28 @@ module Gitlab replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name) end + def drop_nonpartitioned_archive_table(table_name) + assert_table_is_allowed(table_name) + + archived_table_name = make_archived_table_name(table_name) + + with_lock_retries do + drop_sync_trigger(table_name) + end + + drop_table(archived_table_name) + end + + def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key) + function_name = make_sync_function_name(source_table_name) + trigger_name = make_sync_trigger_name(source_table_name) + + create_sync_function(function_name, partitioned_table_name, unique_key) + create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") + + create_sync_trigger(source_table_name, trigger_name, function_name) + end + private def assert_table_is_allowed(table_name) @@ -316,16 +338,6 @@ module Gitlab create_range_partition(partition_name, table_name, lower_bound, upper_bound) end - def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key) - function_name = make_sync_function_name(source_table_name) - trigger_name = make_sync_trigger_name(source_table_name) - - create_sync_function(function_name, partitioned_table_name, unique_key) - create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") - - create_sync_trigger(source_table_name, trigger_name, function_name) - end - def drop_sync_trigger(source_table_name) trigger_name = make_sync_trigger_name(source_table_name) drop_trigger(source_table_name, trigger_name) diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb new file mode 100644 index 00000000000..0ce9eebc14c --- /dev/null +++ b/lib/gitlab/database/pg_class.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PgClass < ActiveRecord::Base + self.table_name = 'pg_class' + + def self.for_table(relname) + joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") + .where('schemaname = current_schema()') + .find_by(relname: relname) + end + + def cardinality_estimate + tuples = reltuples.to_i + + return if tuples < 1 + + tuples + end + end + end +end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 62dfaeeaae3..e8b49c7f62c 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -41,19 +41,6 @@ module Gitlab BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2) BIT_31_MASK = "B'0#{'1' * 31}'" BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'" - # @example source_query - # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits - # FROM %{relation} - # WHERE %{pkey} >= %{batch_start} - # AND %{pkey} < %{batch_end} - # AND %{column} IS NOT NULL - BUCKETED_DATA_SQL = <<~SQL - WITH hashed_attributes AS (%{source_query}) - SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, - (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash - FROM hashed_attributes - GROUP BY 1 - SQL WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid) @@ -103,7 +90,7 @@ module Gitlab def hll_buckets_for_batch(start, finish) @relation .connection - .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) + .execute(bucketed_data_sql % { source_query: source_query(start, finish) }) .map(&:values) .to_h end @@ -139,6 +126,22 @@ module Gitlab def actual_finish(finish) finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0 end + + # @example source_query + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} + # AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + def bucketed_data_sql + <<~SQL + WITH hashed_attributes AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (%{source_query}) + SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, + (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash + FROM hashed_attributes + GROUP BY 1 + SQL + end end end end diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index 40845c0d5e0..20bf6fa4d30 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -10,7 +10,7 @@ module Gitlab # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity. # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE... - SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze + SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY" # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity. # diff --git a/lib/gitlab/database/unidirectional_copy_trigger.rb b/lib/gitlab/database/unidirectional_copy_trigger.rb new file mode 100644 index 00000000000..029c894a5ff --- /dev/null +++ b/lib/gitlab/database/unidirectional_copy_trigger.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class UnidirectionalCopyTrigger + def self.on_table(table_name, connection: ActiveRecord::Base.connection) + new(table_name, connection) + end + + def name(from_column_names, to_column_names) + from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names) + + unchecked_name(from_column_names, to_column_names) + end + + def create(from_column_names, to_column_names, trigger_name: nil) + from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names) + trigger_name ||= unchecked_name(from_column_names, to_column_names) + + assignment_clauses = assignment_clauses_for_columns(from_column_names, to_column_names) + + connection.execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{trigger_name}() + RETURNS trigger AS + $BODY$ + BEGIN + #{assignment_clauses}; + RETURN NEW; + END; + $BODY$ + LANGUAGE 'plpgsql' + VOLATILE + SQL + + connection.execute(<<~SQL) + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table_name} + SQL + + connection.execute(<<~SQL) + CREATE TRIGGER #{trigger_name} + BEFORE INSERT OR UPDATE + ON #{quoted_table_name} + FOR EACH ROW + EXECUTE FUNCTION #{trigger_name}() + SQL + end + + def drop(trigger_name) + connection.execute("DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}") + connection.execute("DROP FUNCTION IF EXISTS #{trigger_name}()") + end + + private + + attr_reader :table_name, :connection + + def initialize(table_name, connection) + @table_name = table_name + @connection = connection + end + + def quoted_table_name + @quoted_table_name ||= connection.quote_table_name(table_name) + end + + def check_column_names!(from_column_names, to_column_names) + from_column_names = Array.wrap(from_column_names) + to_column_names = Array.wrap(to_column_names) + + unless from_column_names.size == to_column_names.size + raise ArgumentError, 'number of source and destination columns must match' + end + + [from_column_names, to_column_names] + end + + def unchecked_name(from_column_names, to_column_names) + joined_column_names = from_column_names.zip(to_column_names).flatten.join('_') + 'trigger_' + Digest::SHA256.hexdigest("#{table_name}_#{joined_column_names}").first(12) + end + + def assignment_clauses_for_columns(from_column_names, to_column_names) + combined_column_names = to_column_names.zip(from_column_names) + + assignment_clauses = combined_column_names.map do |(new_name, old_name)| + new_name = connection.quote_column_name(new_name) + old_name = connection.quote_column_name(old_name) + + "NEW.#{new_name} := NEW.#{old_name}" + end + + assignment_clauses.join(";\n ") + end + end + end +end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index baa46e7e306..8385bbbb3de 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,7 +3,7 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project + attr_reader :diff_file, :diff_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff @@ -22,29 +22,15 @@ module Gitlab end def highlight - @diff_lines.map.with_index do |diff_line, i| + populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + + @diff_lines.map.with_index do |diff_line, index| diff_line = diff_line.dup # ignore highlighting for "match" lines next diff_line if diff_line.meta? - rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) - - if line_inline_diffs = inline_diffs[i] - begin - # MarkerRange objects are converted to Ranges to keep the previous behavior - # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 - if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) - line_inline_diffs = line_inline_diffs.map { |marker_range| marker_range.to_range } - end - - rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) - # This should only happen when the encoding of the diff doesn't - # match the blob, which is a bug. But we shouldn't fail to render - # completely in that case, even though we want to report the error. - rescue RangeError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441') - end - end + rich_line = apply_syntax_highlight(diff_line) + rich_line = apply_marker_ranges_highlight(diff_line, rich_line, index) diff_line.rich_text = rich_line @@ -54,9 +40,87 @@ module Gitlab private + def populate_marker_ranges + pair_selector = Gitlab::Diff::PairSelector.new(@raw_lines) + + pair_selector.each do |old_index, new_index| + old_line = diff_lines[old_index] + new_line = diff_lines[new_index] + + old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new(old_line.text, new_line.text, offset: 1).inline_diffs + + old_line.set_marker_ranges(old_diffs) + new_line.set_marker_ranges(new_diffs) + end + end + + def apply_syntax_highlight(diff_line) + highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) + end + + def apply_marker_ranges_highlight(diff_line, rich_line, index) + marker_ranges = if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + diff_line.marker_ranges + else + inline_diffs[index] + end + + return rich_line if marker_ranges.blank? + + begin + # MarkerRange objects are converted to Ranges to keep the previous behavior + # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 + if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) + marker_ranges = marker_ranges.map { |marker_range| marker_range.to_range } + end + + InlineDiffMarker.new(diff_line.text, rich_line).mark(marker_ranges) + # This should only happen when the encoding of the diff doesn't + # match the blob, which is a bug. But we shouldn't fail to render + # completely in that case, even though we want to report the error. + rescue RangeError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441') + end + end + def highlight_line(diff_line) return unless diff_file && diff_file.diff_refs + if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml) + diff_line_highlighting(diff_line) + else + blob_highlighting(diff_line) + end + end + + def diff_line_highlighting(diff_line) + rich_line = syntax_highlighter(diff_line).highlight( + diff_line.text(prefix: false), + context: { line_number: diff_line.line } + )&.html_safe + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + if rich_line + line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' + rich_line.prepend(line_prefix).concat("\n") + end + end + + def syntax_highlighter(diff_line) + path = diff_line.removed? ? diff_file.old_path : diff_file.new_path + + @syntax_highlighter ||= {} + @syntax_highlighter[path] ||= Gitlab::Highlight.new( + path, + @raw_lines, + language: repository&.gitattribute(path, 'gitlab-language') + ) + end + + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324159 + # ------------------------------------------------------------------------ + def blob_highlighting(diff_line) rich_line = if diff_line.unchanged? || diff_line.added? new_lines[diff_line.new_pos - 1]&.html_safe @@ -72,6 +136,8 @@ module Gitlab end end + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638 + # ------------------------------------------------------------------------ def inline_diffs @inline_diffs ||= InlineDiff.for_lines(@raw_lines) end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index c5e9bfdc321..209462fd6e9 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -71,9 +71,12 @@ module Gitlab strong_memoize(:redis_key) do [ 'highlighted-diff-files', - diffable.cache_key, VERSION, + diffable.cache_key, + VERSION, diff_options, - Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml) + Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml), + Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), + Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) ].join(":") end end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index dd73e4d6c15..f70618195d0 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -18,6 +18,7 @@ module Gitlab CharDiff.new(old_line, new_line).changed_ranges(offset: offset) end + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638 class << self def for_lines(lines) pair_selector = Gitlab::Diff::PairSelector.new(lines) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 98ed2400d82..6cf414e29cc 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,19 +8,24 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code - attr_writer :rich_text - attr_accessor :text, :index, :type, :old_pos, :new_pos + attr_reader :line_code, :marker_ranges + attr_writer :text, :rich_text + attr_accessor :index, :type, :old_pos, :new_pos def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) - @text, @type, @index = text, type, index - @old_pos, @new_pos = old_pos, new_pos + @text = text + @type = type + @index = index + @old_pos = old_pos + @new_pos = new_pos @parent_file = parent_file @rich_text = rich_text # When line code is not provided from cache store we build it # using the parent_file(Diff::File or Conflict::File). @line_code = line_code || calculate_line_code + + @marker_ranges = [] end def self.init_from_hash(hash) @@ -48,6 +53,16 @@ module Gitlab hash end + def set_marker_ranges(marker_ranges) + @marker_ranges = marker_ranges + end + + def text(prefix: true) + return @text if prefix + + @text&.slice(1..).to_s + end + def old_line old_pos unless added? || meta? end diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb index 6e17ffaf6ff..f3e6fc455ac 100644 --- a/lib/gitlab/diff/suggestions_parser.rb +++ b/lib/gitlab/diff/suggestions_parser.rb @@ -17,7 +17,7 @@ module Gitlab no_original_data: true, suggestions_filter_enabled: supports_suggestion) doc = Nokogiri::HTML(html) - suggestion_nodes = doc.search('pre.suggestion') + suggestion_nodes = doc.search('pre.language-suggestion') return [] if suggestion_nodes.empty? @@ -29,9 +29,8 @@ module Gitlab lines_above, lines_below = nil if lang_param && suggestion_params = fetch_suggestion_params(lang_param) - lines_above, lines_below = - suggestion_params[:above], - suggestion_params[:below] + lines_above = suggestion_params[:above] + lines_below = suggestion_params[:below] end Gitlab::Diff::Suggestion.new(node.text, diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb deleted file mode 100644 index 457a3c12206..00000000000 --- a/lib/gitlab/downtime_check.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - # Checks if a set of migrations requires downtime or not. - class DowntimeCheck - # The constant containing the boolean that indicates if downtime is needed - # or not. - DOWNTIME_CONST = :DOWNTIME - - # The constant that specifies the reason for the migration requiring - # downtime. - DOWNTIME_REASON_CONST = :DOWNTIME_REASON - - # Checks the given migration paths and returns an Array of - # `Gitlab::DowntimeCheck::Message` instances. - # - # migrations - The migration file paths to check. - def check(migrations) - migrations.map do |path| - require(path) - - migration_class = class_for_migration_file(path) - - unless migration_class.const_defined?(DOWNTIME_CONST) - raise "The migration in #{path} does not specify if it requires " \ - "downtime or not" - end - - if online?(migration_class) - Message.new(path) - else - reason = downtime_reason(migration_class) - - unless reason - raise "The migration in #{path} requires downtime but no reason " \ - "was given" - end - - Message.new(path, true, reason) - end - end - end - - # Checks the given migrations and prints the results to STDOUT/STDERR. - # - # migrations - The migration file paths to check. - def check_and_print(migrations) - check(migrations).each do |message| - puts message.to_s # rubocop: disable Rails/Output - end - end - - # Returns the class for the given migration file path. - def class_for_migration_file(path) - File.basename(path, File.extname(path)).split('_', 2).last.camelize - .constantize - end - - # Returns true if the given migration can be performed without downtime. - def online?(migration) - migration.const_get(DOWNTIME_CONST, false) == false - end - - # Returns the downtime reason, or nil if none was defined. - def downtime_reason(migration) - if migration.const_defined?(DOWNTIME_REASON_CONST) - migration.const_get(DOWNTIME_REASON_CONST, false) - else - nil - end - end - end -end diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb deleted file mode 100644 index 5debb754943..00000000000 --- a/lib/gitlab/downtime_check/message.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class DowntimeCheck - class Message - attr_reader :path, :offline - - OFFLINE = "\e[31moffline\e[0m" - ONLINE = "\e[32monline\e[0m" - - # path - The file path of the migration. - # offline - When set to `true` the migration will require downtime. - # reason - The reason as to why the migration requires downtime. - def initialize(path, offline = false, reason = nil) - @path = path - @offline = offline - @reason = reason - end - - def to_s - label = offline ? OFFLINE : ONLINE - - message = ["[#{label}]: #{path}"] - - if reason? - message << ":\n\n#{reason}\n\n" - end - - message.join - end - - def reason? - @reason.present? - end - - def reason - @reason.strip.lines.map(&:strip).join("\n") - end - end - end -end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index dfed8db8df0..47d361fb95c 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -16,6 +16,12 @@ module Gitlab Rack::Timeout::RequestTimeoutException ].freeze + PROCESSORS = [ + ::Gitlab::ErrorTracking::Processor::SidekiqProcessor, + ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor, + ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor + ].freeze + class << self def configure Raven.configure do |config| @@ -97,7 +103,9 @@ module Gitlab inject_context_for_exception(event, hint[:exception]) custom_fingerprinting(event, hint[:exception]) - event + PROCESSORS.reduce(event) do |processed_event, processor| + processor.call(processed_event) + end end def process_exception(exception, sentry: false, logging: true, extra:) diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb index 5185205e94e..758f6aa11d7 100644 --- a/lib/gitlab/error_tracking/processor/context_payload_processor.rb +++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb @@ -9,9 +9,21 @@ module Gitlab # integrations are re-implemented and use Gitlab::ErrorTracking, this # processor should be removed. def process(payload) + return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}) payload.deep_merge!(context_payload) end + + def self.call(event) + return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + + Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}).each do |key, value| + event.public_send(key).deep_merge!(value) # rubocop:disable GitlabSecurity/PublicSend + end + + event + end end end end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index 871e9c4b7c8..419098dbd09 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -6,60 +6,126 @@ module Gitlab class GrpcErrorProcessor < ::Raven::Processor DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') - def process(value) - process_first_exception_value(value) - process_custom_fingerprint(value) + def process(payload) + return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - value - end - - # Sentry can report multiple exceptions in an event. Sanitize - # only the first one since that's what is used for grouping. - def process_first_exception_value(value) - exceptions = value.dig(:exception, :values) - - return unless exceptions.is_a?(Array) - - entry = exceptions.first - - return unless entry.is_a?(Hash) - - exception_type = entry[:type] - raw_message = entry[:value] - - return unless exception_type&.start_with?('GRPC::') - return unless raw_message.present? - - message, debug_str = split_debug_error_string(raw_message) - - entry[:value] = message if message - extra = value[:extra] || {} - extra[:grpc_debug_error_string] = debug_str if debug_str - end - - def process_custom_fingerprint(value) - fingerprint = value[:fingerprint] - - return value unless custom_grpc_fingerprint?(fingerprint) + self.class.process_first_exception_value(payload) + self.class.process_custom_fingerprint(payload) - message, _ = split_debug_error_string(fingerprint[1]) - fingerprint[1] = message if message + payload end - private - - def custom_grpc_fingerprint?(fingerprint) - fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') - end - - def split_debug_error_string(message) - return unless message - - match = DEBUG_ERROR_STRING_REGEX.match(message) - - return unless match - - [match[1], match[2]] + class << self + def call(event) + return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + + process_first_exception_value(event) + process_custom_fingerprint(event) + + event + end + + # Sentry can report multiple exceptions in an event. Sanitize + # only the first one since that's what is used for grouping. + def process_first_exception_value(event_or_payload) + exceptions = exceptions(event_or_payload) + + return unless exceptions.is_a?(Array) + + exception = exceptions.first + + return unless valid_exception?(exception) + + exception_type, raw_message = type_and_value(exception) + + return unless exception_type&.start_with?('GRPC::') + return unless raw_message.present? + + message, debug_str = split_debug_error_string(raw_message) + + set_new_values!(event_or_payload, exception, message, debug_str) + end + + def process_custom_fingerprint(event) + fingerprint = fingerprint(event) + + return event unless custom_grpc_fingerprint?(fingerprint) + + message, _ = split_debug_error_string(fingerprint[1]) + fingerprint[1] = message if message + end + + private + + def custom_grpc_fingerprint?(fingerprint) + fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') + end + + def split_debug_error_string(message) + return unless message + + match = DEBUG_ERROR_STRING_REGEX.match(message) + + return unless match + + [match[1], match[2]] + end + + # The below methods can be removed once we remove the + # sentry_processors_before_send feature flag, and we can + # assume we always have an Event object + def exceptions(event_or_payload) + case event_or_payload + when Raven::Event + # Better in new version, will be event_or_payload.exception.values + event_or_payload.instance_variable_get(:@interfaces)[:exception]&.values + when Hash + event_or_payload.dig(:exception, :values) + end + end + + def valid_exception?(exception) + case exception + when Raven::SingleExceptionInterface + exception&.value + when Hash + true + else + false + end + end + + def type_and_value(exception) + case exception + when Raven::SingleExceptionInterface + [exception.type, exception.value] + when Hash + exception.values_at(:type, :value) + end + end + + def set_new_values!(event_or_payload, exception, message, debug_str) + case event_or_payload + when Raven::Event + # Worse in new version, no setter! Have to poke at the + # instance variable + exception.value = message if message + event_or_payload.extra[:grpc_debug_error_string] = debug_str if debug_str + when Hash + exception[:value] = message if message + extra = event_or_payload[:extra] || {} + extra[:grpc_debug_error_string] = debug_str if debug_str + end + end + + def fingerprint(event_or_payload) + case event_or_payload + when Raven::Event + event_or_payload.fingerprint + when Hash + event_or_payload[:fingerprint] + end + end end end end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index 272cb689ad5..93310745ece 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -8,39 +8,66 @@ module Gitlab class SidekiqProcessor < ::Raven::Processor FILTERED_STRING = '[FILTERED]' - def self.filter_arguments(args, klass) - args.lazy.with_index.map do |arg, i| - case arg - when Numeric - arg - else - if permitted_arguments_for_worker(klass).include?(i) + class << self + def filter_arguments(args, klass) + args.lazy.with_index.map do |arg, i| + case arg + when Numeric arg else - FILTERED_STRING + if permitted_arguments_for_worker(klass).include?(i) + arg + else + FILTERED_STRING + end end end end - end - def self.permitted_arguments_for_worker(klass) - @permitted_arguments_for_worker ||= {} - @permitted_arguments_for_worker[klass] ||= - begin - klass.constantize&.loggable_arguments&.to_set - rescue - Set.new + def permitted_arguments_for_worker(klass) + @permitted_arguments_for_worker ||= {} + @permitted_arguments_for_worker[klass] ||= + begin + klass.constantize&.loggable_arguments&.to_set + rescue + Set.new + end + end + + def loggable_arguments(args, klass) + Gitlab::Utils::LogLimitedArray + .log_limited_array(filter_arguments(args, klass)) + .map(&:to_s) + .to_a + end + + def call(event) + return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + + sidekiq = event&.extra&.dig(:sidekiq) + + return event unless sidekiq + + sidekiq = sidekiq.deep_dup + sidekiq.delete(:jobstr) + + # 'args' in this hash => from Gitlab::ErrorTracking.track_* + # 'args' in :job => from default error handler + job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] + + if job_holder['args'] + job_holder['args'] = filter_arguments(job_holder['args'], job_holder['class']).to_a end - end - def self.loggable_arguments(args, klass) - Gitlab::Utils::LogLimitedArray - .log_limited_array(filter_arguments(args, klass)) - .map(&:to_s) - .to_a + event.extra[:sidekiq] = sidekiq + + event + end end def process(value, key = nil) + return value if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + sidekiq = value.dig(:extra, :sidekiq) return value unless sidekiq diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index b602393b59e..ef0236f8275 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -15,14 +15,14 @@ module Gitlab PREFIX = 'gitlab:exclusive_lease' NoKey = Class.new(ArgumentError) - LUA_CANCEL_SCRIPT = <<~EOS.freeze + LUA_CANCEL_SCRIPT = <<~EOS local key, uuid = KEYS[1], ARGV[1] if redis.call("get", key) == uuid then redis.call("del", key) end EOS - LUA_RENEW_SCRIPT = <<~EOS.freeze + LUA_RENEW_SCRIPT = <<~EOS local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2] if redis.call("get", key) == uuid then redis.call("expire", key, ttl) diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 1bb29ba3eac..145bb6d7b8f 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,10 +34,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - upgrade_link_in_user_menu_a: { - tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA', - use_backwards_compatible_subject_index: true - }, invite_members_version_b: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', use_backwards_compatible_subject_index: true diff --git a/lib/gitlab/external_authorization/access.rb b/lib/gitlab/external_authorization/access.rb index e111c41fcc2..21fa728fd3a 100644 --- a/lib/gitlab/external_authorization/access.rb +++ b/lib/gitlab/external_authorization/access.rb @@ -10,7 +10,8 @@ module Gitlab :load_type def initialize(user, label) - @user, @label = user, label + @user = user + @label = label end def loaded? diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb index acdc028b4dc..509daeb0248 100644 --- a/lib/gitlab/external_authorization/cache.rb +++ b/lib/gitlab/external_authorization/cache.rb @@ -6,7 +6,8 @@ module Gitlab VALIDITY_TIME = 6.hours def initialize(user, label) - @user, @label = user, label + @user = user + @label = label end def load diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb index fc859304eab..582051010d3 100644 --- a/lib/gitlab/external_authorization/client.rb +++ b/lib/gitlab/external_authorization/client.rb @@ -13,7 +13,8 @@ module Gitlab }.freeze def initialize(user, label) - @user, @label = user, label + @user = user + @label = label end def request_access @@ -51,18 +52,18 @@ module Gitlab def body @body ||= begin - body = { - user_identifier: @user.email, - project_classification_label: @label, - identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } } - } + body = { + user_identifier: @user.email, + project_classification_label: @label, + identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } } + } - if @user.ldap_identity - body[:user_ldap_dn] = @user.ldap_identity.extern_uid - end + if @user.ldap_identity + body[:user_ldap_dn] = @user.ldap_identity.extern_uid + end - body - end + body + end end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index bd5d2e53180..612865ed1be 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -199,8 +199,7 @@ module Gitlab def linkify_issues(str) str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') - str + str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') end def escape_for_markdown(str) @@ -208,8 +207,7 @@ module Gitlab str = str.gsub(/^-/, "\\-") str = str.gsub("`", "\\~") str = str.delete("\r") - str = str.gsub("\n", " \n") - str + str.gsub("\n", " \n") end def format_content(raw_content) diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 9e24306c05e..a5b1b7d914b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -30,8 +30,10 @@ module Gitlab end def process_raw_blame(output) - lines, final = [], [] - info, commits = {}, {} + lines = [] + final = [] + info = {} + commits = {} # process the output output.split("\n").each do |line| diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index ff99803d8de..51baed32935 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -390,7 +390,7 @@ module Gitlab @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) - @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }] + @trailers = commit.trailers.to_h { |t| [t.key, t.value] } end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 19462e6cb02..fb947c80b7e 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -82,6 +82,30 @@ module Gitlab !!@overflow end + def overflow_max_lines? + !!@overflow_max_lines + end + + def overflow_max_bytes? + !!@overflow_max_bytes + end + + def overflow_max_files? + !!@overflow_max_files + end + + def collapsed_safe_lines? + !!@collapsed_safe_lines + end + + def collapsed_safe_files? + !!@collapsed_safe_files + end + + def collapsed_safe_bytes? + !!@collapsed_safe_bytes + end + def size @size ||= count # forces a loop using each method end @@ -103,10 +127,9 @@ module Gitlab end def decorate! - collection = each_with_index do |element, i| + each_with_index do |element, i| @array[i] = yield(element) end - collection end alias_method :to_ary, :to_a @@ -121,7 +144,15 @@ module Gitlab end def over_safe_limits?(files) - files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes + if files >= safe_max_files + @collapsed_safe_files = true + elsif @line_count > safe_max_lines + @collapsed_safe_lines = true + elsif @byte_count >= safe_max_bytes + @collapsed_safe_bytes = true + end + + @collapsed_safe_files || @collapsed_safe_lines || @collapsed_safe_bytes end def expand_diff? @@ -154,6 +185,7 @@ module Gitlab if @enforce_limits && i >= max_files @overflow = true + @overflow_max_files = true break end @@ -166,10 +198,19 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes) + if @enforce_limits && @line_count >= max_lines + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + @overflow_max_lines = true + break + end + + if @enforce_limits && @byte_count >= max_bytes # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true + @overflow_max_bytes = true break end diff --git a/lib/gitlab/git/merge_base.rb b/lib/gitlab/git/merge_base.rb index b27f7038c26..905d72cadbf 100644 --- a/lib/gitlab/git/merge_base.rb +++ b/lib/gitlab/git/merge_base.rb @@ -6,7 +6,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize def initialize(repository, refs) - @repository, @refs = repository, refs + @repository = repository + @refs = refs end # Returns the SHA of the first common ancestor diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb index c62994432d3..1182db10c34 100644 --- a/lib/gitlab/git/patches/commit_patches.rb +++ b/lib/gitlab/git/patches/commit_patches.rb @@ -7,7 +7,10 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors def initialize(user, repository, branch, patch_collection) - @user, @repository, @branch, @patches = user, repository, branch, patch_collection + @user = user + @repository = repository + @branch = branch + @patches = patch_collection end def commit diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e316d52ac05..3361cee733b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -599,9 +599,9 @@ module Gitlab tags.find { |tag| tag.name == name } end - def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) + def merge_to_ref(user, **kwargs) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) + gitaly_operation_client.user_merge_to_ref(user, **kwargs) end end @@ -1017,6 +1017,10 @@ module Gitlab gitaly_repository_client.search_files_by_name(ref, safe_query) end + def search_files_by_regexp(filter, ref = 'HEAD') + gitaly_repository_client.search_files_by_regexp(ref, filter) + end + def find_commits_by_message(query, ref, path, limit, offset) wrapped_gitaly_errors do gitaly_commit_client diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index da86d6baf4a..568e894a02f 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -87,6 +87,10 @@ module Gitlab end end + def cache_key + "tag:" + Digest::SHA1.hexdigest([name, message, target, target_commit&.sha].join) + end + private def message_from_gitaly_tag diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 55ff3c6caf1..75d6b949874 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -102,12 +102,6 @@ module Gitlab end end - def file(name, version) - wrapped_gitaly_errors do - gitaly_find_file(name, version) - end - end - # options: # :page - The Integer page number. # :per_page - The number of items per page. @@ -161,13 +155,6 @@ module Gitlab nil end - def gitaly_find_file(name, version) - wiki_file = gitaly_wiki_client.find_file(name, version) - return unless wiki_file - - Gitlab::Git::WikiFile.new(wiki_file) - end - def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) params = { limit: limit, sort: sort, direction_desc: direction_desc } diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb index 7f09173f05c..c56a17c52f3 100644 --- a/lib/gitlab/git/wiki_file.rb +++ b/lib/gitlab/git/wiki_file.rb @@ -5,25 +5,11 @@ module Gitlab class WikiFile attr_reader :mime_type, :raw_data, :name, :path - # This class wraps Gitlab::GitalyClient::WikiFile - def initialize(gitaly_file) - @mime_type = gitaly_file.mime_type - @raw_data = gitaly_file.raw_data - @name = gitaly_file.name - @path = gitaly_file.path - end - - def self.from_blob(blob) - hash = { - name: File.basename(blob.name), - mime_type: blob.mime_type, - path: blob.path, - raw_data: blob.data - } - - gitaly_file = Gitlab::GitalyClient::WikiFile.new(hash) - - Gitlab::Git::WikiFile.new(gitaly_file) + def initialize(blob) + @mime_type = blob.mime_type + @raw_data = blob.data + @name = File.basename(blob.name) + @path = blob.path end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c5ca46827cb..31e4755192e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -91,6 +91,7 @@ module Gitlab when *PUSH_COMMANDS check_push_access! end + check_additional_conditions! success_result end @@ -530,6 +531,10 @@ module Gitlab def size_checker container.repository_size_checker end + + # overriden in EE + def check_additional_conditions! + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e3788814dd5..f4a89edecd1 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -215,7 +215,7 @@ module Gitlab 'client_name' => CLIENT_NAME } - context_data = Labkit::Context.current&.to_h + context_data = Gitlab::ApplicationContext.current feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb index f935281ac2e..74e6279708e 100644 --- a/lib/gitlab/gitaly_client/attributes_bag.rb +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -3,7 +3,7 @@ module Gitlab module GitalyClient # This module expects an `ATTRS` const to be defined on the subclass - # See GitalyClient::WikiFile for an example + # See GitalyClient::WikiPage for an example module AttributesBag extend ActiveSupport::Concern diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index c66b3335d89..19a473e4785 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -78,17 +78,7 @@ module Gitlab end def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) - request = Gitaly::GetNewLFSPointersRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision), - limit: limit || 0 - ) - - if not_in.nil? || not_in == :all - request.not_in_all = true - else - request.not_in_refs += not_in - end + request, rpc = create_new_lfs_pointers_request(revision, limit, not_in) timeout = if dynamic_timeout @@ -100,7 +90,7 @@ module Gitlab response = GitalyClient.call( @gitaly_repo.storage_name, :blob_service, - :get_new_lfs_pointers, + rpc, request, timeout: timeout ) @@ -108,16 +98,51 @@ module Gitlab end def get_all_lfs_pointers - request = Gitaly::GetAllLFSPointersRequest.new( - repository: @gitaly_repo + request = Gitaly::ListLFSPointersRequest.new( + repository: @gitaly_repo, + revisions: [encode_binary("--all")] ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end private + def create_new_lfs_pointers_request(revision, limit, not_in) + # If the check happens for a change which is using a quarantine + # environment for incoming objects, then we can avoid doing the + # necessary graph walk to detect only new LFS pointers and instead scan + # through all quarantined objects. + git_env = ::Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository) + if Feature.enabled?(:lfs_integrity_inspect_quarantined_objects, @project, default_enabled: :yaml) && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + repository = @gitaly_repo.dup + repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) + + request = Gitaly::ListAllLFSPointersRequest.new( + repository: repository, + limit: limit || 0 + ) + + [request, :list_all_lfs_pointers] + else + revisions = [revision] + revisions += if not_in.nil? || not_in == :all + ["--not", "--all"] + else + not_in.prepend "--not" + end + + request = Gitaly::ListLFSPointersRequest.new( + repository: @gitaly_repo, + limit: limit || 0, + revisions: revisions.map { |rev| encode_binary(rev) } + ) + + [request, :list_lfs_pointers] + end + end + def consume_blob_response(response) data = [] blob = nil diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb index 9d4d86997ad..4bb184bee2f 100644 --- a/lib/gitlab/gitaly_client/call.rb +++ b/lib/gitlab/gitaly_client/call.rb @@ -50,11 +50,11 @@ module Gitlab end def recording_request - start = Gitlab::Metrics::System.monotonic_time + @start = Gitlab::Metrics::System.monotonic_time yield ensure - @duration += Gitlab::Metrics::System.monotonic_time - start + @duration += Gitlab::Metrics::System.monotonic_time - @start end def store_timings @@ -64,8 +64,14 @@ module Gitlab request_hash = @request.is_a?(Google::Protobuf::MessageExts) ? @request.to_h : {} - GitalyClient.add_call_details(feature: "#{@service}##{@rpc}", duration: @duration, request: request_hash, rpc: @rpc, - backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) + GitalyClient.add_call_details( + start: @start, + feature: "#{@service}##{@rpc}", + duration: @duration, + request: request_hash, + rpc: @rpc, + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) + ) end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ef5221a8042..3d24b4d53a4 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -107,6 +107,8 @@ module Gitlab entry.data = data.join entry unless entry.oid.blank? + rescue GRPC::NotFound + nil end def tree_entries(repository, revision, path, recursive) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 6f302b2c4e7..5ce1b1f0c87 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -103,7 +103,7 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) + def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index bd450249355..a93f4071efc 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -339,6 +339,11 @@ module Gitlab search_results_from_response(response, options) end + def search_files_by_regexp(ref, filter) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter) + GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) + end + def disconnect_alternates request = Gitaly::DisconnectGitAlternatesRequest.new( repository: @gitaly_repo diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 7edd42f9ef7..dd9e3d5d28b 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -11,7 +11,7 @@ module Gitlab DirectPathAccessError = Class.new(StandardError) InvalidConfigurationError = Class.new(StandardError) - INVALID_STORAGE_MESSAGE = <<~MSG.freeze + INVALID_STORAGE_MESSAGE = <<~MSG Storage is invalid because it has no `path` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb deleted file mode 100644 index ef2b23732d1..00000000000 --- a/lib/gitlab/gitaly_client/wiki_file.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class WikiFile - ATTRS = %i(name mime_type path raw_data).freeze - - include AttributesBag - end - end -end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 9034edb6263..fecc2b7023d 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -153,32 +153,6 @@ module Gitlab versions end - def find_file(name, revision) - request = Gitaly::WikiFindFileRequest.new( - repository: @gitaly_repo, - name: encode_binary(name), - revision: encode_binary(revision) - ) - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout) - wiki_file = nil - - response.each do |message| - next unless message.name.present? || wiki_file - - if wiki_file - wiki_file.raw_data = "#{wiki_file.raw_data}#{message.raw_data}" - else - wiki_file = GitalyClient::WikiFile.new(message.to_h) - # All gRPC strings in a response are frozen, so we get - # an unfrozen version here so appending in the else clause below doesn't blow up. - wiki_file.raw_data = wiki_file.raw_data.dup - end - end - - wiki_file - end - private # If a block is given and the yielded value is truthy, iteration will be diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index f2dc668c482..31b7a198b92 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -2,10 +2,12 @@ module Gitlab module Golang + PseudoVersion = Struct.new(:semver, :timestamp, :commit_id) + extend self def local_module_prefix - @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/".freeze + @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/" end def semver_tag?(tag) @@ -37,11 +39,11 @@ module Gitlab end # This pattern is intentionally more forgiving than the patterns - # above. Correctness is verified by #pseudo_version_commit. + # above. Correctness is verified by #validate_pseudo_version. /\A\d{14}-\h+\z/.freeze.match? pre end - def pseudo_version_commit(project, semver) + def parse_pseudo_version(semver) # Per Go's implementation of pseudo-versions, a tag should be # considered a pseudo-version if it matches one of the patterns # listed in #pseudo_version?, regardless of the content of the @@ -55,9 +57,14 @@ module Gitlab # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go) # Go ignores anything before '.' or after the second '-', so we will do the same - timestamp, sha = semver.prerelease.split('-').last 2 + timestamp, commit_id = semver.prerelease.split('-').last 2 timestamp = timestamp.split('.').last - commit = project.repository.commit_by(oid: sha) + + PseudoVersion.new(semver, timestamp, commit_id) + end + + def validate_pseudo_version(project, version, commit = nil) + commit ||= project.repository.commit_by(oid: version.commit_id) # Error messages are based on the responses of proxy.golang.org @@ -65,10 +72,10 @@ module Gitlab raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit # Require the SHA fragment to be 12 characters long - raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12 + raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 # Require the timestamp to match that of the commit - raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp + raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp commit end @@ -77,6 +84,14 @@ module Gitlab Packages::SemVer.parse(str, prefixed: true) end + def go_path(project, path = nil) + if path.blank? + "#{local_module_prefix}/#{project.full_path}" + else + "#{local_module_prefix}/#{project.full_path}/#{path}" + end + end + def pkg_go_dev_url(name, version = nil) if version "https://pkg.go.dev/#{name}@#{version}" diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c7e215c143f..08c17058fcb 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -45,7 +45,7 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) - push_frontend_feature_flag(:usage_data_api, default_enabled: true) + push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) end diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb index 0a8f0872fbe..468a296886e 100644 --- a/lib/gitlab/grape_logging/loggers/context_logger.rb +++ b/lib/gitlab/grape_logging/loggers/context_logger.rb @@ -6,7 +6,7 @@ module Gitlab module Loggers class ContextLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) - Labkit::Context.current.to_h + Gitlab::ApplicationContext.current end end end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb deleted file mode 100644 index e83b567308b..00000000000 --- a/lib/gitlab/graphql/authorize.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - # Allow fields to declare permissions their objects must have. The field - # will be set to nil unless all required permissions are present. - module Authorize - extend ActiveSupport::Concern - - def self.use(schema_definition) - schema_definition.instrument(:field, Gitlab::Graphql::Authorize::Instrumentation.new, after_built_ins: true) - end - end - end -end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb deleted file mode 100644 index e8db619f88a..00000000000 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Authorize - class AuthorizeFieldService - def initialize(field) - @field = field - @old_resolve_proc = @field.resolve_proc - end - - def authorizations? - authorizations.present? - end - - def authorized_resolve - proc do |parent_typed_object, args, ctx| - resolved_type = @old_resolve_proc.call(parent_typed_object, args, ctx) - authorizing_object = authorize_against(parent_typed_object, resolved_type) - - filter_allowed(ctx[:current_user], resolved_type, authorizing_object) - end - end - - private - - def authorizations - @authorizations ||= (type_authorizations + field_authorizations).uniq - end - - # Returns any authorize metadata from the return type of @field - def type_authorizations - type = @field.type - - # When the return type of @field is a collection, find the singular type - if @field.connection? - type = node_type_for_relay_connection(type) - elsif type.list? - type = node_type_for_basic_connection(type) - end - - type = type.unwrap if type.kind.non_null? - - Array.wrap(type.metadata[:authorize]) - end - - # Returns any authorize metadata from @field - def field_authorizations - return [] if @field.metadata[:authorize] == true - - Array.wrap(@field.metadata[:authorize]) - end - - def authorize_against(parent_typed_object, resolved_type) - if scalar_type? - # The field is a built-in/scalar type, or a list of scalars - # authorize using the parent's object - parent_typed_object.object - elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array) - # The field is a connection or a list of non-built-in types, we'll - # authorize each element when rendering - nil - elsif resolved_type.respond_to?(:object) - # The field is a type representing a single object, we'll authorize - # against the object directly - resolved_type.object - else - # Resolved type is a single object that might not be loaded yet by - # the batchloader, we'll authorize that - resolved_type - end - end - - def filter_allowed(current_user, resolved_type, authorizing_object) - if resolved_type.nil? - # We're not rendering anything, for example when a record was not found - # no need to do anything - elsif authorizing_object - # Authorizing fields representing scalars, or a simple field with an object - ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object| - resolved_type if allowed_access?(current_user, object) - end - elsif @field.connection? - ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type| - # A connection with pagination, modify the visible nodes on the - # connection type in place - nodes = to_nodes(type) - nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes - type - end - elsif @field.type.list? || resolved_type.is_a?(Array) - # A simple list of rendered types each object being an object to authorize - ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items| - items.select do |single_object_type| - object_type = realized(single_object_type) - object = object_type.try(:object) || object_type - allowed_access?(current_user, object) - end - end - else - raise "Can't authorize #{@field}" - end - end - - # Ensure that we are dealing with realized objects, not delayed promises - def realized(thing) - ::Gitlab::Graphql::Lazy.force(thing) - end - - # Try to get the connection - # can be at type.object or at type - def to_nodes(type) - if type.respond_to?(:nodes) - type.nodes - elsif type.respond_to?(:object) - to_nodes(type.object) - else - nil - end - end - - def allowed_access?(current_user, object) - object = realized(object) - - authorizations.all? do |ability| - Ability.allowed?(current_user, ability, object) - end - end - - # Returns the singular type for relay connections. - # This will be the type class of edges.node - def node_type_for_relay_connection(type) - type.unwrap.get_field('edges').type.unwrap.get_field('node').type - end - - # Returns the singular type for basic connections, for example `[Types::ProjectType]` - def node_type_for_basic_connection(type) - type.unwrap - end - - def scalar_type? - node_type_for_basic_connection(@field.type).kind.scalar? - end - end - end - end -end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 6ee446011d4..4d575b964e5 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -5,15 +5,17 @@ module Gitlab module Authorize module AuthorizeResource extend ActiveSupport::Concern + ConfigurationError = Class.new(StandardError) - RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does not exist or you don't have permission to perform this action" + RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does " \ + "not exist or you don't have permission to perform this action" class_methods do def required_permissions # If the `#authorize` call is used on multiple classes, we add the # permissions specified on a subclass, to the ones that were specified - # on it's superclass. - @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions) + # on its superclass. + @required_permissions ||= if respond_to?(:superclass) && superclass.respond_to?(:required_permissions) superclass.required_permissions.dup else [] @@ -23,6 +25,18 @@ module Gitlab def authorize(*permissions) required_permissions.concat(permissions) end + + def authorizes_object? + defined?(@authorizes_object) ? @authorizes_object : false + end + + def authorizes_object! + @authorizes_object = true + end + + def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) + raise ::Gitlab::Graphql::Errors::ResourceNotAvailable, msg + end end def find_object(*args) @@ -37,33 +51,21 @@ module Gitlab object end + # authorizes the object using the current class authorization. def authorize!(object) - unless authorized_resource?(object) - raise_resource_not_available_error! - end + raise_resource_not_available_error! unless authorized_resource?(object) end - # this was named `#authorized?`, however it conflicts with the native - # graphql gem version - # TODO consider adopting the gem's built in authorization system - # https://gitlab.com/gitlab-org/gitlab/issues/13984 def authorized_resource?(object) # Sanity check. We don't want to accidentally allow a developer to authorize # without first adding permissions to authorize against - if self.class.required_permissions.empty? - raise Gitlab::Graphql::Errors::ArgumentError, "#{self.class.name} has no authorizations" - end + raise ConfigurationError, "#{self.class.name} has no authorizations" if self.class.authorization.none? - self.class.required_permissions.all? do |ability| - # The actions could be performed across multiple objects. In which - # case the current user is common, and we could benefit from the - # caching in `DeclarativePolicy`. - Ability.allowed?(current_user, ability, object, scope: :user) - end + self.class.authorization.ok?(object, current_user) end - def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) - raise Gitlab::Graphql::Errors::ResourceNotAvailable, msg + def raise_resource_not_available_error!(*args) + self.class.raise_resource_not_available_error!(*args) end end end diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb new file mode 100644 index 00000000000..c75510df3e3 --- /dev/null +++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Authorize + class ConnectionFilterExtension < GraphQL::Schema::FieldExtension + class Redactor + include ::Gitlab::Graphql::Laziness + + def initialize(type, context) + @type = type + @context = context + end + + def redact(nodes) + remove_unauthorized(nodes) + + nodes + end + + def active? + # some scalar types (such as integers) do not respond to :authorized? + return false unless @type.respond_to?(:authorized?) + + auth = @type.try(:authorization) + + auth.nil? || auth.any? + end + + private + + def remove_unauthorized(nodes) + nodes + .map! { |lazy| force(lazy) } + .keep_if { |forced| @type.authorized?(forced, @context) } + end + end + + def after_resolve(value:, context:, **rest) + return value if value.is_a?(GraphQL::Execution::Execute::Skip) + + if @field.connection? + redact_connection(value, context) + elsif @field.type.list? + redact_list(value.to_a, context) unless value.nil? + end + + value + end + + def redact_connection(conn, context) + redactor = Redactor.new(@field.type.unwrap.node_type, context) + return unless redactor.active? + + conn.redactor = redactor if conn.respond_to?(:redactor=) + end + + def redact_list(list, context) + redactor = Redactor.new(@field.type.unwrap, context) + redactor.redact(list) if redactor.active? + end + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb deleted file mode 100644 index 15ecc3b04f0..00000000000 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Authorize - class Instrumentation - # Replace the resolver for the field with one that will only return the - # resolved object if the permissions check is successful. - def instrument(_type, field) - service = AuthorizeFieldService.new(field) - - if service.authorizations? - field.redefine { resolve(service.authorized_resolve) } - else - field - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/authorize/object_authorization.rb b/lib/gitlab/graphql/authorize/object_authorization.rb new file mode 100644 index 00000000000..0bc87108871 --- /dev/null +++ b/lib/gitlab/graphql/authorize/object_authorization.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Authorize + class ObjectAuthorization + attr_reader :abilities + + def initialize(abilities) + @abilities = Array.wrap(abilities).flatten + end + + def none? + abilities.empty? + end + + def any? + abilities.present? + end + + def ok?(object, current_user) + return true if none? + + subject = object.try(:declarative_policy_subject) || object + abilities.all? do |ability| + Ability.allowed?(current_user, ability, subject) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb new file mode 100644 index 00000000000..e0176e2d6e0 --- /dev/null +++ b/lib/gitlab/graphql/deprecation.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class Deprecation + REASONS = { + renamed: 'This was renamed.', + discouraged: 'Use of this is not recommended.' + }.freeze + + include ActiveModel::Validations + + validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' } + validates :reason, presence: true + validates :reason, + format: { with: /.*[^.]\z/, message: 'must not end with a period' }, + if: :reason_is_string? + validate :milestone_is_string + validate :reason_known_or_string + + def self.parse(options) + new(**options) if options + end + + def initialize(reason: nil, milestone: nil, replacement: nil) + @reason = reason.presence + @milestone = milestone.presence + @replacement = replacement.presence + end + + def ==(other) + return false unless other.is_a?(self.class) + + [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr| + other.send(attr) # rubocop: disable GitlabSecurity/PublicSend + end + end + alias_method :eql, :== + + def markdown(context: :inline) + parts = [ + "#{deprecated_in(format: :markdown)}.", + reason_text, + replacement.then { |r| "Use: `#{r}`." if r } + ].compact + + case context + when :block + ['WARNING:', *parts].join("\n") + when :inline + parts.join(' ') + end + end + + def edit_description(original_description) + @original_description = original_description + return unless original_description + + original_description + description_suffix + end + + def original_description + return unless @original_description + return @original_description if @original_description.ends_with?('.') + + "#{@original_description}." + end + + def deprecation_reason + [ + reason_text, + replacement && "Please use `#{replacement}`.", + "#{deprecated_in}." + ].compact.join(' ') + end + + private + + attr_reader :reason, :milestone, :replacement + + def milestone_is_string + return if milestone.is_a?(String) + + errors.add(:milestone, 'must be a string') + end + + def reason_known_or_string + return if REASONS.key?(reason) + return if reason_is_string? + + errors.add(:reason, 'must be a known reason or a string') + end + + def reason_is_string? + reason.is_a?(String) + end + + def reason_text + @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}." + end + + def description_suffix + " #{deprecated_in}: #{reason_text}" + end + + def deprecated_in(format: :plain) + case format + when :plain + "Deprecated in #{milestone}" + when :markdown + "**Deprecated** in #{milestone}" + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index e9ff85d9ca9..f4173e26224 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -27,7 +27,10 @@ module Gitlab MD end - def render_name_and_description(object, level = 3) + # Template methods: + # Methods that return chunks of Markdown for insertion into the document + + def render_name_and_description(object, owner: nil, level: 3) content = [] content << "#{'#' * level} `#{object[:name]}`" @@ -35,10 +38,22 @@ module Gitlab if object[:description].present? desc = object[:description].strip desc += '.' unless desc.ends_with?('.') + end + + if object[:is_deprecated] + owner = Array.wrap(owner) + deprecation = schema_deprecation(owner, object[:name]) + content << (deprecation&.original_description || desc) + content << render_deprecation(object, owner, :block) + else content << desc end - content.join("\n\n") + content.compact.join("\n\n") + end + + def render_return_type(query) + "Returns #{render_field_type(query[:type])}.\n" end def sorted_by_name(objects) @@ -47,39 +62,25 @@ module Gitlab objects.sort_by { |o| o[:name] } end - def render_field(field) - row(render_name(field), render_field_type(field[:type]), render_description(field)) + def render_field(field, owner) + render_row( + render_name(field, owner), + render_field_type(field[:type]), + render_description(field, owner, :inline) + ) end - def render_enum_value(value) - row(render_name(value), render_description(value)) + def render_enum_value(enum, value) + render_row(render_name(value, enum[:name]), render_description(value, enum[:name], :inline)) end - def row(*values) - "| #{values.join(' | ')} |" + def render_union_member(member) + "- [`#{member}`](##{member.downcase})" end - def render_name(object) - rendered_name = "`#{object[:name]}`" - rendered_name += ' **{warning-solid}**' if object[:is_deprecated] - rendered_name - end + # QUERIES: - # Returns the object description. If the object has been deprecated, - # the deprecation reason will be returned in place of the description. - def render_description(object) - return object[:description] unless object[:is_deprecated] - - "**Deprecated:** #{object[:deprecation_reason]}" - end - - def render_field_type(type) - "[`#{type[:info]}`](##{type[:name].downcase})" - end - - def render_return_type(query) - "Returns #{render_field_type(query[:type])}.\n" - end + # Methods that return parts of the schema, or related information: # We are ignoring connections and built in types for now, # they should be added when queries are generated. @@ -103,6 +104,83 @@ module Gitlab !enum_type[:name].in?(%w[__DirectiveLocation __TypeKind]) end end + + private # DO NOT CALL THESE METHODS IN TEMPLATES + + # Template methods + + def render_row(*values) + "| #{values.map { |val| val.to_s.squish }.join(' | ')} |" + end + + def render_name(object, owner = nil) + rendered_name = "`#{object[:name]}`" + rendered_name += ' **{warning-solid}**' if object[:is_deprecated] + rendered_name + end + + # Returns the object description. If the object has been deprecated, + # the deprecation reason will be returned in place of the description. + def render_description(object, owner = nil, context = :block) + owner = Array.wrap(owner) + return render_deprecation(object, owner, context) if object[:is_deprecated] + return if object[:description].blank? + + desc = object[:description].strip + desc += '.' unless desc.ends_with?('.') + desc + end + + def render_deprecation(object, owner, context) + deprecation = schema_deprecation(owner, object[:name]) + return deprecation.markdown(context: context) if deprecation + + reason = object[:deprecation_reason] || 'Use of this is deprecated.' + "**Deprecated:** #{reason}" + end + + def render_field_type(type) + "[`#{type[:info]}`](##{type[:name].downcase})" + end + + # Queries + + # returns the deprecation information for a field or argument + # See: Gitlab::Graphql::Deprecation + def schema_deprecation(type_name, field_name) + schema_member(type_name, field_name)&.deprecation + end + + # Return a part of the schema. + # + # This queries the Schema by owner and name to find: + # + # - fields (e.g. `schema_member('Query', 'currentUser')`) + # - arguments (e.g. `schema_member(['Query', 'project], 'fullPath')`) + def schema_member(type_name, field_name) + type_name = Array.wrap(type_name) + if type_name.size == 2 + arg_name = field_name + type_name, field_name = type_name + else + type_name = type_name.first + arg_name = nil + end + + return if type_name.nil? || field_name.nil? + + type = schema.types[type_name] + return unless type && type.kind.fields? + + field = type.fields[field_name] + return field if arg_name.nil? + + args = field.arguments + is_mutation = field.mutation && field.mutation <= ::Mutations::BaseMutation + args = args['input'].type.unwrap.arguments if is_mutation + + args[arg_name] + end end end end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb index 6abd56c89c6..497567f9389 100644 --- a/lib/gitlab/graphql/docs/renderer.rb +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -10,17 +10,20 @@ module Gitlab # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. # # Arguments: - # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema.graphql_definition + # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema # output_dir: The folder where the markdown files will be saved # template: The path of the haml template to be parsed class Renderer include Gitlab::Graphql::Docs::Helper + attr_reader :schema + def initialize(schema, output_dir:, template:) @output_dir = output_dir @template = template @layout = Haml::Engine.new(File.read(template)) - @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse + @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse + @schema = schema end def contents diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 847f1777b08..fe73297d0d9 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -27,7 +27,7 @@ \ - sorted_by_name(queries).each do |query| - = render_name_and_description(query) + = render_name_and_description(query, owner: 'Query') \ = render_return_type(query) - unless query[:arguments].empty? @@ -35,7 +35,7 @@ ~ "| Name | Type | Description |" ~ "| ---- | ---- | ----------- |" - sorted_by_name(query[:arguments]).each do |argument| - = render_field(argument) + = render_field(argument, query[:type][:name]) \ :plain @@ -58,7 +58,7 @@ ~ "| Field | Type | Description |" ~ "| ----- | ---- | ----------- |" - sorted_by_name(type[:fields]).each do |field| - = render_field(field) + = render_field(field, type[:name]) \ :plain @@ -79,7 +79,7 @@ ~ "| Value | Description |" ~ "| ----- | ----------- |" - sorted_by_name(enum[:values]).each do |value| - = render_enum_value(value) + = render_enum_value(enum, value) \ :plain @@ -121,12 +121,12 @@ \ - graphql_union_types.each do |type| - = render_name_and_description(type, 4) + = render_name_and_description(type, level: 4) \ One of: \ - - type[:possible_types].each do |type_name| - ~ "- [`#{type_name}`](##{type_name.downcase})" + - type[:possible_types].each do |member| + = render_union_member(member) \ :plain @@ -134,7 +134,7 @@ \ - graphql_interface_types.each do |type| - = render_name_and_description(type, 4) + = render_name_and_description(type, level: 4) \ Implementations: \ @@ -144,5 +144,5 @@ ~ "| Field | Type | Description |" ~ "| ----- | ---- | ----------- |" - sorted_by_name(type[:fields] + type[:connections]).each do |field| - = render_field(field) + = render_field(field, type[:name]) \ diff --git a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb index 67511c124e4..1945388cdd4 100644 --- a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb @@ -5,7 +5,8 @@ module Gitlab module Loaders class BatchLfsOidLoader def initialize(repository, blob_id) - @repository, @blob_id = repository, blob_id + @repository = repository + @blob_id = blob_id end def find diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb index 9b85ba164d4..805864cdd4c 100644 --- a/lib/gitlab/graphql/loaders/batch_model_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -7,7 +7,8 @@ module Gitlab attr_reader :model_class, :model_id def initialize(model_class, model_id) - @model_class, @model_id = model_class, model_id + @model_class = model_class + @model_id = model_id end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb index 0aa237c78de..26c1ce64a83 100644 --- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -9,7 +9,8 @@ module Gitlab attr_reader :model_class, :full_path def initialize(model_class, full_path) - @model_class, @full_path = model_class, full_path + @model_class = model_class + @full_path = full_path end def find diff --git a/lib/gitlab/graphql/negatable_arguments.rb b/lib/gitlab/graphql/negatable_arguments.rb new file mode 100644 index 00000000000..b4ab31ed51a --- /dev/null +++ b/lib/gitlab/graphql/negatable_arguments.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module NegatableArguments + class TypeDefiner + def initialize(resolver_class, type_definition) + @resolver_class = resolver_class + @type_definition = type_definition + end + + def define! + negated_params_type.instance_eval(&@type_definition) + end + + def negated_params_type + @negated_params_type ||= existing_type || build_type + end + + private + + def existing_type + ::Types.const_get(type_class_name, false) if ::Types.const_defined?(type_class_name) + end + + def build_type + klass = Class.new(::Types::BaseInputObject) + ::Types.const_set(type_class_name, klass) + klass + end + + def type_class_name + @type_class_name ||= begin + base_name = @resolver_class.name.sub('Resolvers::', '') + base_name + 'NegatedParamsType' + end + end + end + + def negated(param_key: :not, &block) + definer = ::Gitlab::Graphql::NegatableArguments::TypeDefiner.new(self, block) + definer.define! + + argument param_key, definer.negated_params_type, + required: false, + description: <<~MD + List of negated arguments. + Warning: this argument is experimental and a subject to change in future. + MD + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb index bd785880b57..6645dac36fa 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb @@ -13,7 +13,11 @@ module Gitlab # @param [Symbol] before_or_after indicates whether we want # items :before the cursor or :after the cursor def initialize(arel_table, order_list, values, operators, before_or_after) - @arel_table, @order_list, @values, @operators, @before_or_after = arel_table, order_list, values, operators, before_or_after + @arel_table = arel_table + @order_list = order_list + @values = values + @operators = operators + @before_or_after = before_or_after @before_or_after = :after unless [:after, :before].include?(@before_or_after) end diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb index 3164598b7b9..ec70f5c5a24 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb @@ -30,15 +30,13 @@ module Gitlab # ex: " OR (relative_position = 23 AND id > 500)" def second_attribute_condition - condition = <<~SQL + <<~SQL OR ( #{table_condition(order_list.first, values.first, '=').to_sql} AND #{table_condition(order_list[1], values[1], operators[1]).to_sql} ) SQL - - condition end # ex: " OR (relative_position IS NULL)" diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb index fa25181d663..1aae1020e79 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb @@ -14,15 +14,13 @@ module Gitlab # ex: "(relative_position IS NULL AND id > 500)" def first_attribute_condition - condition = <<~SQL + <<~SQL ( #{table_condition(order_list.first, nil, 'is_null').to_sql} AND #{table_condition(order_list[1], values[1], operators[1]).to_sql} ) SQL - - condition end # ex: " OR (relative_position IS NOT NULL)" diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb index 29169449843..ee9c902c735 100644 --- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb +++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb @@ -6,7 +6,10 @@ module Gitlab module Keyset class QueryBuilder def initialize(arel_table, order_list, decoded_cursor, before_or_after) - @arel_table, @order_list, @decoded_cursor, @before_or_after = arel_table, order_list, decoded_cursor, before_or_after + @arel_table = arel_table + @order_list = order_list + @decoded_cursor = decoded_cursor + @before_or_after = before_or_after if order_list.empty? raise ArgumentError.new('No ordering scopes have been supplied') diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index fcf293fb13e..74f55abccbc 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -224,11 +224,9 @@ module Gitlab frag_path = frag_path.gsub(DOTS_RE) do |dots| rel_dir(dots.split('/').count) end - frag_path = frag_path.gsub(IMPLICIT_ROOT) do + frag_path.gsub(IMPLICIT_ROOT) do (Rails.root / 'app').to_s + '/' end - - frag_path end def rel_dir(n_steps_up) diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 8acd27869a9..c6f22e0bd4f 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -12,6 +12,7 @@ module Gitlab def initial_value(query) variables = process_variables(query.provided_variables) default_initial_values(query).merge({ + operation_name: query.operation_name, query_string: query.query_string, variables: variables }) @@ -20,8 +21,8 @@ module Gitlab default_initial_values(query) end - def call(memo, visit_type, irep_node) - RequestStore.store[:graphql_logs] = memo + def call(memo, *) + memo end def final_value(memo) @@ -37,6 +38,8 @@ module Gitlab memo[:used_fields] = field_usages.first memo[:used_deprecated_fields] = field_usages.second + RequestStore.store[:graphql_logs] ||= [] + RequestStore.store[:graphql_logs] << memo GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index e780bf8a986..f5f142c251f 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -5,7 +5,7 @@ module Gitlab class GitalyCheck extend BaseAbstractCheck - METRIC_PREFIX = 'gitaly_health_check'.freeze + METRIC_PREFIX = 'gitaly_health_check' class << self def readiness diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 40dee0142b9..765d3dfca56 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -20,7 +20,9 @@ module Gitlab @blob_content = blob_content end - def highlight(text, continue: true, plain: false) + def highlight(text, continue: false, plain: false, context: {}) + @context = context + plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE highlighted_text = highlight_text(text, continue: continue, plain: plain) @@ -31,13 +33,15 @@ module Gitlab def lexer @lexer ||= custom_language || begin Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new - rescue Rouge::Guesser::Ambiguous => e - e.alternatives.min_by(&:tag) + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.min_by(&:tag) end end private + attr_reader :context + def custom_language return unless @language @@ -53,13 +57,13 @@ module Gitlab end def highlight_plain(text) - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + @formatter.format(Rouge::Lexers::PlainText.lex(text), context).html_safe end def highlight_rich(text, continue: true) tag = lexer.tag tokens = lexer.lex(text, continue: continue) - Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } + Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } rescue Timeout::Error => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb new file mode 100644 index 00000000000..537245e948f --- /dev/null +++ b/lib/gitlab/hook_data/user_builder.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class UserBuilder < BaseBuilder + alias_method :user, :object + + # Sample data + # { + # :created_at=>"2021-04-02T10:00:26Z", + # :updated_at=>"2021-04-02T10:00:26Z", + # :event_name=>"user_create", + # :name=>"John Doe", + # :email=>"john@example.com", + # :user_id=>1, + # :username=>"johndoe" + # } + + def build(event) + [ + timestamps_data, + event_data(event), + user_data, + event_specific_user_data(event) + ].reduce(:merge) + end + + private + + def user_data + { + name: user.name, + email: user.email, + user_id: user.id, + username: user.username + } + end + + def event_specific_user_data(event) + case event + when :rename + { old_username: user.username_before_last_save } + when :failed_login + { state: user.state } + else + {} + end + end + end + end +end + +Gitlab::HookData::UserBuilder.prepend_if_ee('EE::Gitlab::HookData::UserBuilder') diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 37f618ae879..f7a3da53fdb 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -17,14 +17,6 @@ module Gitlab def connection @uri, hostname = validate_url!(uri) - if options.key?(:http_proxyaddr) - proxy_uri_with_port = uri_with_port(options[:http_proxyaddr], options[:http_proxyport]) - proxy_uri_validated = validate_url!(proxy_uri_with_port).first - - @options[:http_proxyaddr] = proxy_uri_validated.omit(:port).to_s - @options[:http_proxyport] = proxy_uri_validated.port - end - super.tap do |http| http.hostname_override = hostname if hostname end @@ -53,11 +45,5 @@ module Gitlab def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end - - def uri_with_port(address, port) - uri = Addressable::URI.parse(address) - uri.port = port if port.present? - uri - end end end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index d60bc79df4c..05a4a8f4c93 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -6,7 +6,7 @@ module Gitlab class RelationFactory include Gitlab::Utils::StrongMemoize - IMPORTED_OBJECT_MAX_RETRIES = 5.freeze + IMPORTED_OBJECT_MAX_RETRIES = 5 OVERRIDES = {}.freeze EXISTING_OBJECT_RELATIONS = %i[].freeze diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 778b42f4358..42d32593cbd 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -265,6 +265,7 @@ excluded_attributes: - :issue_id push_event_payload: - :event_id + - :event_id_convert_to_bigint project_badges: - :group_id resource_label_events: @@ -287,6 +288,7 @@ excluded_attributes: - :label_id events: - :target_id + - :id_convert_to_bigint timelogs: - :issue_id - :merge_request_id diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 428bcbe8dc5..2f15cdd7506 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -76,7 +76,7 @@ module Gitlab def project_uploads_except_avatar(avatar_path) return @project.uploads unless avatar_path - @project.uploads.where("path != ?", avatar_path) + @project.uploads.where.not(path: avatar_path) end def download_and_copy(upload) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 88753e80391..95c002edf0a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -28,7 +28,7 @@ module Gitlab prepend_if_ee('EE::Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule def options - Hash[import_table.map { |importer| [importer.title, importer.name] }] + import_table.to_h { |importer| [importer.title, importer.name] } end def values diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 61de6b02453..a865a6392f0 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -6,24 +6,6 @@ module Gitlab DURATION_PRECISION = 6 # microseconds - def keys - @keys ||= [ - :cpu_s, - :gitaly_calls, - :gitaly_duration_s, - :rugged_calls, - :rugged_duration_s, - :elasticsearch_calls, - :elasticsearch_duration_s, - :elasticsearch_timed_out_count, - *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, - *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS, - *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, - *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS - ] - end - def init_instrumentation_data(request_ip: nil) # Set `request_start_time` only if this is request # This is done, as `request_start_time` imply `request_deadline` diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 945ab7f40c2..6b33b60e850 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -78,7 +78,7 @@ module Gitlab # to perform the calculation more efficiently. Until then, use a shorter # timeout and return -1 as a sentinel value if it is triggered begin - ApplicationRecord.with_fast_statement_timeout do + ApplicationRecord.with_fast_read_statement_timeout do finder.count_by_state end rescue ActiveRecord::QueryCanceled => err diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb index 4415f98fc7f..ddf2cd76709 100644 --- a/lib/gitlab/jira/dvcs.rb +++ b/lib/gitlab/jira/dvcs.rb @@ -3,8 +3,8 @@ module Gitlab module Jira module Dvcs - ENCODED_SLASH = '@'.freeze - SLASH = '/'.freeze + ENCODED_SLASH = '@' + SLASH = '/' ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze def self.encode_slash(path) diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 8565f664cd4..b51c0a33457 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -186,9 +186,14 @@ module Gitlab # The `env` param is ignored because it's not needed in either our formatter or Grape's, # but it is passed through for consistency. # + # If explicitly supplied with a `PrecompiledJson` instance it will skip conversion + # and return it directly. This is mostly used in caching. + # # @param object [Object] # @return [String] def self.call(object, env = nil) + return object.to_s if object.is_a?(PrecompiledJson) + if Feature.enabled?(:grape_gitlab_json, default_enabled: true) Gitlab::Json.dump(object) else @@ -197,6 +202,34 @@ module Gitlab end end + # Wrapper class used to skip JSON dumping on Grape endpoints. + + class PrecompiledJson + UnsupportedFormatError = Class.new(StandardError) + + # @overload PrecompiledJson.new("foo") + # @param value [String] + # + # @overload PrecompiledJson.new(["foo", "bar"]) + # @param value [Array<String>] + def initialize(value) + @value = value + end + + # Convert the value to a String. This will invoke + # `#to_s` on the members of the value if it's an array. + # + # @return [String] + # @raise [NoMethodError] if the objects in an array doesn't support to_s + # @raise [PrecompiledJson::UnsupportedFormatError] if the value is neither a String or Array + def to_s + return @value if @value.is_a?(String) + return "[#{@value.join(',')}]" if @value.is_a?(Array) + + raise UnsupportedFormatError + end + end + class LimitedEncoder LimitExceeded = Class.new(StandardError) diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 329c0f221b5..7a674cb5c21 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -27,7 +27,7 @@ module Gitlab def included_in_gitlab_com_rollout?(project) return true unless ::Gitlab.com? - Feature.enabled?(:kubernetes_agent_on_gitlab_com, project) + Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml) end end end diff --git a/lib/gitlab/kubernetes/deployment.rb b/lib/gitlab/kubernetes/deployment.rb index 55ed9a7517e..f2e3a0e6810 100644 --- a/lib/gitlab/kubernetes/deployment.rb +++ b/lib/gitlab/kubernetes/deployment.rb @@ -5,7 +5,7 @@ module Gitlab class Deployment include Gitlab::Utils::StrongMemoize - STABLE_TRACK_VALUE = 'stable'.freeze + STABLE_TRACK_VALUE = 'stable' def initialize(attributes = {}, pods: []) @attributes = attributes diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index 7600e60b904..1e5edb79f10 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -20,7 +20,7 @@ module Gitlab # Newly detected languages, returned in a structure accepted by # Gitlab::Database.bulk_insert def insertions(programming_languages) - lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h + lang_to_id = programming_languages.to_h { |p| [p.name, p.id] } (languages - previous_language_names).map do |new_lang| { @@ -63,8 +63,7 @@ module Gitlab @repository .languages .first(MAX_LANGUAGES) - .map { |l| [l[:label], l] } - .to_h + .to_h { |l| [l[:label], l] } end end end diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb index 7208fe5bbc5..618ddf37b88 100644 --- a/lib/gitlab/manifest_import/manifest.rb +++ b/lib/gitlab/manifest_import/manifest.rb @@ -47,6 +47,10 @@ module Gitlab @errors << 'Make sure every <project> tag has name and path attributes.' end + unless validate_scheme + @errors << 'Make sure the url does not start with javascript' + end + @errors.empty? end @@ -64,6 +68,10 @@ module Gitlab end end + def validate_scheme + remote !~ /\Ajavascript/i + end + def repository_url(name) Gitlab::Utils.append_path(remote, name) end diff --git a/lib/gitlab/marker_range.rb b/lib/gitlab/marker_range.rb index 50a59adebdf..73e4a545679 100644 --- a/lib/gitlab/marker_range.rb +++ b/lib/gitlab/marker_range.rb @@ -24,6 +24,12 @@ module Gitlab Range.new(self.begin, self.end, self.exclude_end?) end + def ==(other) + return false unless other.is_a?(self.class) + + self.mode == other.mode && super + end + attr_reader :mode end end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index d419fa66e57..45c6205b36b 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -4,7 +4,7 @@ module Gitlab module MarkupHelper extend self - MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze + MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown rmd].freeze ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index 3dda68bf93f..a1fabe75a97 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -34,8 +34,9 @@ module Gitlab def labels @labels ||= { - endpoint_id: current_context&.get_attribute(:caller_id), - feature_category: current_context&.get_attribute(:feature_category) + endpoint_id: endpoint_id, + feature_category: feature_category, + queue: queue } end @@ -44,6 +45,21 @@ module Gitlab def current_context Labkit::Context.current end + + def feature_category + current_context&.get_attribute(:feature_category) + end + + def endpoint_id + current_context&.get_attribute(:caller_id) + end + + def queue + worker_class = endpoint_id.to_s.safe_constantize + return if worker_class.blank? || !worker_class.respond_to?(:queue) + + worker_class.queue.to_s + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index c90c1e3f0bc..55d14d6f94a 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -104,9 +104,7 @@ module Gitlab def format_query(metric) expression = remove_new_lines(metric[:expr]) expression = replace_variables(expression) - expression = replace_global_variables(expression, metric) - - expression + replace_global_variables(expression, metric) end # Accomodates instance-defined Grafana variables. @@ -135,9 +133,7 @@ module Gitlab def replace_global_variables(expression, metric) expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval] expression = expression.gsub('$__from', query_params[:from]) - expression = expression.gsub('$__to', query_params[:to]) - - expression + expression.gsub('$__to', query_params[:to]) end # Removes new lines from expression. diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 60ae22df607..c0336a4d0fb 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -32,9 +32,9 @@ module Gitlab private def init_metrics - METRIC_DESCRIPTIONS.map do |name, description| + METRIC_DESCRIPTIONS.to_h do |name, description| [name, ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)] - end.to_h + end end def host_stats diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 5eefef02507..0d1cd641ffe 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -11,13 +11,16 @@ module Gitlab DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze - DURATION_BUCKET = [0.05, 0.1, 0.25].freeze + SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze + TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ActiveRecord::Base.transaction {} block. def transaction(event) - observe(:gitlab_database_transaction_seconds, event) + observe(:gitlab_database_transaction_seconds, event) do + buckets TRANSACTION_DURATION_BUCKET + end end def sql(event) @@ -33,7 +36,9 @@ module Gitlab increment(:db_cached_count) if cached_query?(payload) increment(:db_write_count) unless select_sql_command?(payload) - observe(:gitlab_sql_duration_seconds, event) + observe(:gitlab_sql_duration_seconds, event) do + buckets SQL_DURATION_BUCKET + end end def self.db_counter_payload @@ -46,6 +51,10 @@ module Gitlab payload end + def self.known_payload_keys + DB_COUNTERS + end + private def ignored_query?(payload) @@ -66,10 +75,8 @@ module Gitlab Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 end - def observe(histogram, event) - current_transaction&.observe(histogram, event.duration / 1000.0) do - buckets DURATION_BUCKET - end + def observe(histogram, event, &block) + current_transaction&.observe(histogram, event.duration / 1000.0, &block) end def current_transaction diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index 94c5d965200..0df64f2897e 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -37,7 +37,7 @@ module Gitlab def request(event) payload = event.payload - add_to_detail_store(payload) + add_to_detail_store(event.time, payload) add_to_request_store(payload) expose_metrics(payload) end @@ -48,10 +48,11 @@ module Gitlab ::Gitlab::Metrics::Transaction.current end - def add_to_detail_store(payload) + def add_to_detail_store(start, payload) return unless Gitlab::PerformanceBar.enabled_for_request? self.class.detail_store << { + start: start, duration: payload[:duration], scheme: payload[:scheme], method: payload[:method], diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 79f1abe820f..329041e3ba2 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -31,7 +31,7 @@ module Gitlab RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' JWT_PARAM_SUFFIX = '.gitlab-workhorse-upload' JWT_PARAM_FIXED_KEY = 'upload' - REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000.freeze + REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000 class Handler def initialize(env, message) diff --git a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb new file mode 100644 index 00000000000..d16c068c3c0 --- /dev/null +++ b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class RackMultipartTempfileFactory + # Immediately unlink the created temporary file so we don't have to rely + # on Rack::TempfileReaper catching this after the fact. + FACTORY = lambda do |filename, content_type| + Rack::Multipart::Parser::TEMPFILE_FACTORY.call(filename, content_type).tap(&:unlink) + end + + def initialize(app) + @app = app + end + + def call(env) + if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1' + env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY + end + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/middleware/same_site_cookies.rb b/lib/gitlab/middleware/same_site_cookies.rb index 37ccc5abb10..405732e8015 100644 --- a/lib/gitlab/middleware/same_site_cookies.rb +++ b/lib/gitlab/middleware/same_site_cookies.rb @@ -17,7 +17,7 @@ module Gitlab module Middleware class SameSiteCookies - COOKIE_SEPARATOR = "\n".freeze + COOKIE_SEPARATOR = "\n" def initialize(app) @app = app diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index b1a1045a1f0..9a74266693b 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -68,13 +68,22 @@ module Gitlab expose_depth = hierarchy_order.present? hierarchy_order ||= :asc - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct - # if hierarchy_order is given, the calculated `depth` should be present in SELECT if expose_depth + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) else - read_only(remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order)) + recursive_query = base_and_ancestors_cte(upto).apply_to(model.all) + + if skip_ordering? + recursive_query = recursive_query.distinct + else + recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct + recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) + recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) + end + + read_only(recursive_query) end else recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) @@ -93,12 +102,21 @@ module Gitlab def base_and_descendants(with_depth: false) if use_distinct? # Always calculate `depth`, remove it later if with_depth is false - base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct - if with_depth - read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: :asc)) + base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct + read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) else - read_only(remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc)) + base_cte = base_and_descendants_cte.apply_to(model.all) + + if skip_ordering? + base_cte = base_cte.distinct + else + base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct + base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) + base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) + end + + read_only(base_cte) end else read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) @@ -161,7 +179,19 @@ module Gitlab # Use distinct on the Namespace queries to avoid bad planner behavior in PG11. def use_distinct? - (model <= Namespace) && options[:use_distinct] + return unless model <= Namespace + # Global use_distinct_for_all_object_hierarchy takes precedence over use_distinct_in_object_hierarchy + return true if Feature.enabled?(:use_distinct_for_all_object_hierarchy) + return options[:use_distinct] if options.key?(:use_distinct) + + false + end + + # Skips the extra ordering when using distinct on the namespace queries + def skip_ordering? + return options[:skip_ordering] if options.key?(:skip_ordering) + + false end # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 33e709360ad..98e87e9e915 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -3,7 +3,7 @@ module Gitlab module Pages VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze - INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request' MAX_SIZE = 1.terabyte include JwtAuthenticatable diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb new file mode 100644 index 00000000000..8f8667fafd9 --- /dev/null +++ b/lib/gitlab/pages/migration_helper.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class MigrationHelper + def initialize(logger = nil) + @logger = logger + end + + def migrate_to_remote_storage + deployments = ::PagesDeployment.with_files_stored_locally + migrate(deployments, ObjectStorage::Store::REMOTE) + end + + def migrate_to_local_storage + deployments = ::PagesDeployment.with_files_stored_remotely + migrate(deployments, ObjectStorage::Store::LOCAL) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate(deployments, store) + deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord + deployment.file.migrate!(store) + + log_success(deployment, store) + rescue => e + log_error(e, deployment) + end + end + + def log_success(deployment, store) + logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage") + end + + def log_error(err, deployment) + logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}") + end + + def storage_label(store) + if store == ObjectStorage::Store::LOCAL + 'local' + else + 'object' + end + end + end + end +end diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index 8650a80a85e..be71018e851 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -6,12 +6,28 @@ module Gitlab DiskAccessDenied = Class.new(StandardError) def path - if ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? - raise DiskAccessDenied - end + report_denied_disk_access super end + + def local_store + @local_store ||= ::Gitlab::Pages::Stores::LocalStore.new(super) + end + + private + + def disk_access_denied? + return true unless ::Settings.pages.local_store&.enabled + + ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? + end + + def report_denied_disk_access + raise DiskAccessDenied if disk_access_denied? + rescue => e + ::Gitlab::ErrorTracking.track_exception(e) + end end end end diff --git a/lib/gitlab/pages/stores/local_store.rb b/lib/gitlab/pages/stores/local_store.rb new file mode 100644 index 00000000000..68a7ebaceff --- /dev/null +++ b/lib/gitlab/pages/stores/local_store.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + module Stores + class LocalStore < ::SimpleDelegator + def enabled + return false unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + + super + end + end + end + end +end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb index c1ccfae3e1f..ae5539c03b1 100644 --- a/lib/gitlab/pages_transfer.rb +++ b/lib/gitlab/pages_transfer.rb @@ -12,7 +12,7 @@ module Gitlab class Async METHODS.each do |meth| define_method meth do |*args| - next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + next unless Settings.pages.local_store.enabled PagesTransferWorker.perform_async(meth, args) end @@ -21,7 +21,7 @@ module Gitlab METHODS.each do |meth| define_method meth do |*args| - next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + next unless Settings.pages.local_store.enabled super(*args) end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index e8e68a5c4a5..e596e1bac9d 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -55,14 +55,14 @@ module Gitlab # scope :created_at_ordered, -> { # keyset_order = Gitlab::Pagination::Keyset::Order.build([ # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - # attribute: :created_at, + # attribute_name: :created_at, # column_expression: Project.arel_table[:created_at], # order_expression: Project.arel_table[:created_at].asc, # distinct: false, # values in the column are not unique # nullable: :nulls_last # we might see NULL values (bottom) # ), # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - # attribute: :id, + # attribute_name: :id, # order_expression: Project.arel_table[:id].asc # ) # ]) @@ -93,7 +93,7 @@ module Gitlab end def cursor_attributes_for_node(node) - column_definitions.each_with_object({}) do |column_definition, hash| + column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash| field_value = node[column_definition.attribute_name] hash[column_definition.attribute_name] = if field_value.is_a?(Time) field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') @@ -162,7 +162,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_cursor_conditions(scope, values = {}) scope = apply_custom_projections(scope) - scope.where(build_where_values(values)) + scope.where(build_where_values(values.with_indifferent_access)) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/pagination/offset_header_builder.rb b/lib/gitlab/pagination/offset_header_builder.rb index 32089e40932..555f0e5a607 100644 --- a/lib/gitlab/pagination/offset_header_builder.rb +++ b/lib/gitlab/pagination/offset_header_builder.rb @@ -5,9 +5,9 @@ module Gitlab class OffsetHeaderBuilder attr_reader :request_context, :per_page, :page, :next_page, :prev_page, :total, :total_pages - delegate :params, :header, :request, to: :request_context + delegate :request, to: :request_context - def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total:, total_pages:) + def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total: nil, total_pages: nil, params: nil) @request_context = request_context @per_page = per_page @page = page @@ -15,6 +15,7 @@ module Gitlab @prev_page = prev_page @total = total @total_pages = total_pages + @params = params end def execute(exclude_total_headers: false, data_without_counts: false) @@ -56,10 +57,24 @@ module Gitlab end def page_href(next_page_params = {}) - query_params = params.merge(**next_page_params, per_page: params[:per_page]).to_query + query_params = params.merge(**next_page_params, per_page: per_page).to_query build_page_url(query_params: query_params) end + + def params + @params || request_context.params + end + + def header(name, value) + if request_context.respond_to?(:header) + # For Grape API + request_context.header(name, value) + else + # For rails controllers + request_context.response.headers[name] = value + end + end end end end diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index 380340b80be..c2a4602fd16 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -5,6 +5,12 @@ module Gitlab # This class fetches Peek stats stored in redis and logs them in a # structured log (so these can be then analyzed in Kibana) class Stats + IGNORED_BACKTRACE_LOCATIONS = %w[ + ee/lib/ee/peek + lib/peek + lib/gitlab/database + ].freeze + def initialize(redis) @redis = redis end @@ -53,7 +59,8 @@ module Gitlab end def parse_backtrace(backtrace) - return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace.first) + return unless backtrace_row = find_caller(backtrace) + return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace_row) { filename: match[:filename], @@ -65,6 +72,12 @@ module Gitlab } end + def find_caller(backtrace) + backtrace.find do |line| + !line.start_with?(*IGNORED_BACKTRACE_LOCATIONS) + end + end + def logger @logger ||= Gitlab::PerformanceBar::Logger.build end diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb index 3885a9934d5..4c9d54a93ce 100644 --- a/lib/gitlab/phabricator_import.rb +++ b/lib/gitlab/phabricator_import.rb @@ -5,7 +5,7 @@ module Gitlab BaseError = Class.new(StandardError) def self.available? - Feature.enabled?(:phabricator_import) && + Feature.enabled?(:phabricator_import, default_enabled: :yaml) && Gitlab::CurrentSettings.import_sources.include?('phabricator') end end diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb index a58438452ff..478c26af030 100644 --- a/lib/gitlab/phabricator_import/issues/importer.rb +++ b/lib/gitlab/phabricator_import/issues/importer.rb @@ -4,7 +4,8 @@ module Gitlab module Issues class Importer def initialize(project, after = nil) - @project, @after = project, after + @project = project + @after = after end def execute diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb index c17f3e1729a..9c419ecb700 100644 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -4,7 +4,8 @@ module Gitlab module Issues class TaskImporter def initialize(project, task) - @project, @task = project, task + @project = project + @task = task end def execute diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb index b37a5b44980..c842798ca74 100644 --- a/lib/gitlab/phabricator_import/project_creator.rb +++ b/lib/gitlab/phabricator_import/project_creator.rb @@ -55,12 +55,13 @@ module Gitlab end def project_feature_attributes - @project_features_attributes ||= begin - # everything disabled except for issues - ProjectFeature::FEATURES.map do |feature| - [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] - end.to_h.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) - end + @project_features_attributes ||= + begin + # everything disabled except for issues + ProjectFeature::FEATURES.to_h do |feature| + [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] + end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) + end end def import_data diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb index 4b50431e0e0..c6058d12527 100644 --- a/lib/gitlab/phabricator_import/user_finder.rb +++ b/lib/gitlab/phabricator_import/user_finder.rb @@ -4,7 +4,8 @@ module Gitlab module PhabricatorImport class UserFinder def initialize(project, phids) - @project, @phids = project, phids + @project = project + @phids = phids @loaded_phids = Set.new end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 56eeea6e746..32d3eeb8cd2 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -5,7 +5,11 @@ module Gitlab attr_reader :title, :name, :description, :preview, :logo def initialize(name, title, description, preview, logo = 'illustrations/gitlab_logo.svg') - @name, @title, @description, @preview, @logo = name, title, description, preview, logo + @name = name + @title = title + @description = description + @preview = preview + @logo = logo end def file diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index ed10ef2917f..76e65d29c7a 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -19,6 +19,10 @@ module Gitlab end def cluster_prometheus_adapter + if cluster&.integration_prometheus + return cluster.integration_prometheus + end + application = cluster&.application_prometheus application if application&.available? diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb index e4d44df3baf..73de5a11998 100644 --- a/lib/gitlab/prometheus/queries/matched_metric_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -4,7 +4,7 @@ module Gitlab module Prometheus module Queries class MatchedMetricQuery < BaseQuery - MAX_QUERY_ITEMS = 40.freeze + MAX_QUERY_ITEMS = 40 def query groups_data.map do |group, data| diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 965349ad711..0fcf63d03fc 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -140,7 +140,7 @@ module Gitlab end def mapped_options - options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h + options.keys.to_h { |k| [gitlab_http_key(k), options[k]] } end def http_options diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 02446a7953b..ce9fced9465 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -5,6 +5,7 @@ module Gitlab VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { keys: [ + :assign, :create, :description, :label, @@ -12,6 +13,7 @@ module Gitlab :remove_source_branch, :target, :title, + :unassign, :unlabel ] }, @@ -23,7 +25,9 @@ module Gitlab MULTI_VALUE_OPTIONS = [ %w[ci variable], %w[merge_request label], - %w[merge_request unlabel] + %w[merge_request unlabel], + %w[merge_request assign], + %w[merge_request unassign] ].freeze NAMESPACE_ALIASES = HashWithIndifferentAccess.new({ diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index 5e46e26e14e..03386dca141 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -6,28 +6,36 @@ module Gitlab # # This is only enabled in development and test to ensure we don't produce # any errors that users of other environments can't do anything about themselves. - def self.enable? + def self.enabled_for_env? Rails.env.development? || Rails.env.test? end + def self.enabled? + enabled_for_env? && + !Gitlab::SafeRequestStore[:query_limiting_disabled] + end + # Allows the current request to execute any number of SQL queries. # # This method should _only_ be used when there's a corresponding issue to # reduce the number of queries. # # The issue URL is only meant to push developers into creating an issue - # instead of blindly whitelisting offending blocks of code. - def self.whitelist(issue_url) - return unless enable? - + # instead of blindly disabling for offending blocks of code. + def self.disable!(issue_url) unless issue_url.start_with?('https://') raise( ArgumentError, - 'You must provide a valid issue URL in order to whitelist a block of code' + 'You must provide a valid issue URL in order to allow a block of code' ) end - Transaction&.current&.whitelisted = true + Gitlab::SafeRequestStore[:query_limiting_disabled] = true + end + + # Enables query limiting for the request. + def self.enable! + Gitlab::SafeRequestStore[:query_limiting_disabled] = nil end end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 196072dddda..643b2540c37 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -5,7 +5,7 @@ module Gitlab class Transaction THREAD_KEY = :__gitlab_query_counts_transaction - attr_accessor :count, :whitelisted + attr_accessor :count # The name of the action (e.g. `UsersController#show`) that is being # executed. @@ -45,7 +45,6 @@ module Gitlab def initialize @action = nil @count = 0 - @whitelisted = false @sql_executed = [] end @@ -59,7 +58,7 @@ module Gitlab end def increment - @count += 1 unless whitelisted + @count += 1 if enabled? end def executed_sql(sql) @@ -83,6 +82,10 @@ module Gitlab ["#{header}: #{msg}", log, ellipsis].compact.join("\n") end + + def enabled? + ::Gitlab::QueryLimiting.enabled? + end end end end diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index b17a0208f95..8ce13db4c03 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -56,15 +56,18 @@ module Gitlab end def execute(context, arg) - return unless executable?(context) + return if noop? count_commands_executed_in(context) + return unless available?(context) + execute_block(action_block, context, arg) end def execute_message(context, arg) - return unless executable?(context) + return if noop? + return _('Could not apply %{name} command.') % { name: name } unless available?(context) if execution_message.respond_to?(:call) execute_block(execution_message, context, arg) @@ -101,10 +104,6 @@ module Gitlab private - def executable?(context) - !noop? && available?(context) - end - def count_commands_executed_in(context) return unless context.respond_to?(:commands_executed_count=) diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 4934c12a339..b7d58e05651 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -182,7 +182,7 @@ module Gitlab parse_params do |raw_time_date| Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute end - command :spend do |time_spent, time_spent_date| + command :spend, :spent do |time_spent, time_spent_date| if time_spent @updates[:spend_time] = { duration: time_spent, diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 67e3a5de223..bd6d2e016b4 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -34,12 +34,16 @@ module Gitlab path =~ %r{^/-/(health|liveness|readiness|metrics)} end + def container_registry_event? + path =~ %r{^/api/v\d+/container_registry_event/} + end + def product_analytics_collector_request? path.start_with?('/-/collector/i') end def should_be_skipped? - api_internal_request? || health_check_request? + api_internal_request? || health_check_request? || container_registry_event? end def web_request? diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 00739c05386..488ba04f87c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -181,7 +181,7 @@ module Gitlab end def generic_package_version_regex - /\A\d+\.\d+\.\d+\z/ + maven_version_regex end def generic_package_name_regex @@ -385,11 +385,11 @@ module Gitlab end def merge_request_wip - /(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/ + /(?i)(\[WIP\]\s*|WIP:\s*|\AWIP\z)/ end def merge_request_draft - /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/ + /\A(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft\z)/ end def issue diff --git a/lib/gitlab/relative_positioning/closed_range.rb b/lib/gitlab/relative_positioning/closed_range.rb index 8916d1face5..11fba05edee 100644 --- a/lib/gitlab/relative_positioning/closed_range.rb +++ b/lib/gitlab/relative_positioning/closed_range.rb @@ -4,7 +4,8 @@ module Gitlab module RelativePositioning class ClosedRange < RelativePositioning::Range def initialize(lhs, rhs) - @lhs, @rhs = lhs, rhs + @lhs = lhs + @rhs = rhs raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs end diff --git a/lib/gitlab/relative_positioning/gap.rb b/lib/gitlab/relative_positioning/gap.rb index ab894141a60..2e30e598eb0 100644 --- a/lib/gitlab/relative_positioning/gap.rb +++ b/lib/gitlab/relative_positioning/gap.rb @@ -6,7 +6,8 @@ module Gitlab attr_reader :start_pos, :end_pos def initialize(start_pos, end_pos) - @start_pos, @end_pos = start_pos, end_pos + @start_pos = start_pos + @end_pos = end_pos end def ==(other) diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index eb7c9bccf96..d0230c035cc 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -60,14 +60,17 @@ module Gitlab define_method("#{name}_include?") do |value| ivar = "@#{name}_include" memoized = instance_variable_get(ivar) || {} + lookup = proc { __send__(name).include?(value) } # rubocop:disable GitlabSecurity/PublicSend next memoized[value] if memoized.key?(value) memoized[value] = - if strong_memoized?(name) || !redis_set_cache.exist?(name) - __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + if strong_memoized?(name) + lookup.call else - redis_set_cache.include?(name, value) + result, exists = redis_set_cache.try_include?(name, value) + + exists ? result : lookup.call end instance_variable_set(ivar, memoized)[value] diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index d479d3115a6..430f3e8d162 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -148,7 +148,7 @@ module Gitlab # @param hash [Hash] # @return [Hash] the stringified hash def standardize_hash(hash) - hash.map { |k, v| [k.to_s, v.to_s] }.to_h + hash.to_h { |k, v| [k.to_s, v.to_s] } end # Record metrics in Prometheus. diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 69c1688767c..f73ac628bce 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -36,10 +36,32 @@ module Gitlab end def fetch(key, &block) - if exist?(key) - read(key) - else - write(key, yield) + full_key = cache_key(key) + + smembers, exists = with do |redis| + redis.multi do + redis.smembers(full_key) + redis.exists(full_key) + end + end + + return smembers if exists + + write(key, yield) + end + + # Searches the cache set using SSCAN with the MATCH option. The MATCH + # parameter is the pattern argument. + # See https://redis.io/commands/scan#the-match-option for more information. + # Returns an Enumerator that enumerates all SSCAN hits. + def search(key, pattern, &block) + full_key = cache_key(key) + + with do |redis| + exists = redis.exists(full_key) + write(key, yield) unless exists + + redis.sscan_each(full_key, match: pattern) end end end diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb index c3bb0ff26f2..0323220690a 100644 --- a/lib/gitlab/search_context.rb +++ b/lib/gitlab/search_context.rb @@ -129,7 +129,10 @@ module Gitlab 'wiki_blobs' elsif view_context.current_controller?(:commits) 'commits' - else nil + elsif view_context.current_controller?(:groups) + if %w(issues merge_requests).include?(view_context.controller.action_name) + view_context.controller.action_name + end end end end @@ -160,3 +163,5 @@ module Gitlab end end end + +Gitlab::SearchContext::Builder.prepend_ee_mod diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 591265d014e..0f2b7b194c9 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -51,6 +51,19 @@ module Gitlab with { |redis| redis.sismember(cache_key(key), value) } end + # Like include?, but also tells us if the cache was populated when it ran + # by returning two booleans: [member_exists, set_exists] + def try_include?(key, value) + full_key = cache_key(key) + + with do |redis| + redis.multi do + redis.sismember(full_key, value) + redis.exists(full_key) + end + end + end + def ttl(key) with { |redis| redis.ttl(cache_key(key)) } end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 7561e36cc33..3ac20724403 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -98,6 +98,10 @@ module Gitlab if Rails.env.test? socket_filename = options[:gitaly_socket] || "gitaly.socket" + prometheus_listen_addr = options[:prometheus_listen_addr] + + git_bin_path = File.expand_path('../gitaly/_build/deps/git/install/bin/git') + git_bin_path = nil unless File.exist?(git_bin_path) config = { # Override the set gitaly_address since Praefect is in the loop @@ -106,8 +110,12 @@ module Gitlab # Compared to production, tests run in constrained environments. This # number is meant to grow with the number of concurrent rails requests / # sidekiq jobs, and concurrency will be low anyway in test. - git: { catfile_cache_size: 5 } - } + git: { + catfile_cache_size: 5, + bin_path: git_bin_path + }.compact, + prometheus_listen_addr: prometheus_listen_addr + }.compact storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s storages << { name: 'test_second_storage', path: storage_path } diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index e471517c50a..9490d543dd1 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -53,11 +53,11 @@ module Gitlab 'You cannot specify --queue-selector and --experimental-queue-selector together' end - all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path) - queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path) + worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path) + worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path) - queue_groups = argv.map do |queues| - next queue_names if queues == '*' + queue_groups = argv.map do |queues_or_query_string| + next worker_queues if queues_or_query_string == SidekiqConfig::WorkerMatcher::WILDCARD_MATCH # When using the queue query syntax, we treat each queue group # as a worker attribute query, and resolve the queues for the @@ -65,14 +65,14 @@ module Gitlab # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 if @queue_selector || @experimental_queue_selector - SidekiqConfig::CliMethods.query_workers(queues, all_queues) + SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas) else - SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names) + SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues) end end if @negate_queues - queue_groups.map! { |queues| queue_names - queues } + queue_groups.map! { |queues| worker_queues - queues } end if queue_groups.all?(&:empty?) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 633291dcdf3..78d45b5f3f0 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -13,10 +13,17 @@ module Gitlab (EE_QUEUE_CONFIG_PATH if Gitlab.ee?) ].compact.freeze - DEFAULT_WORKERS = [ - DummyWorker.new('default', weight: 1, tags: []), - DummyWorker.new('mailers', weight: 2, tags: []) - ].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze + # This maps workers not in our application code to queues. We need + # these queues in our YAML files to ensure we don't accidentally + # miss jobs from these queues. + # + # The default queue should be unused, which is why it maps to an + # invalid class name. We keep it in the YAML file for safety, just + # in case anything does get scheduled to run there. + DEFAULT_WORKERS = { + '_' => DummyWorker.new('default', weight: 1, tags: []), + 'ActionMailer::MailDeliveryJob' => DummyWorker.new('mailers', feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: []) + }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze class << self include Gitlab::SidekiqConfig::CliMethods @@ -40,7 +47,7 @@ module Gitlab def workers @workers ||= begin result = [] - result.concat(DEFAULT_WORKERS) + result.concat(DEFAULT_WORKERS.values) result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false)) if Gitlab.ee? diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index a256632bc12..8eef15f9ccb 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -12,35 +12,19 @@ module Gitlab # rubocop:disable Gitlab/ModuleWithInstanceVariables extend self + # The file names are misleading. Those files contain the metadata of the + # workers. They should be renamed to all_workers instead. + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1018 QUEUE_CONFIG_PATHS = begin result = %w[app/workers/all_queues.yml] result << 'ee/app/workers/all_queues.yml' if Gitlab.ee? result end.freeze - QUERY_OR_OPERATOR = '|' - QUERY_AND_OPERATOR = '&' - QUERY_CONCATENATE_OPERATOR = ',' - QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + def worker_metadatas(rails_path = Rails.root.to_s) + @worker_metadatas ||= {} - QUERY_PREDICATES = { - feature_category: :to_sym, - has_external_dependencies: lambda { |value| value == 'true' }, - name: :to_s, - resource_boundary: :to_sym, - tags: :to_sym, - urgency: :to_sym - }.freeze - - QueryError = Class.new(StandardError) - InvalidTerm = Class.new(QueryError) - UnknownOperator = Class.new(QueryError) - UnknownPredicate = Class.new(QueryError) - - def all_queues(rails_path = Rails.root.to_s) - @worker_queues ||= {} - - @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| + @worker_metadatas[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| full_path = File.join(rails_path, path) File.exist?(full_path) ? YAML.load_file(full_path) : [] @@ -49,7 +33,7 @@ module Gitlab # rubocop:enable Gitlab/ModuleWithInstanceVariables def worker_queues(rails_path = Rails.root.to_s) - worker_names(all_queues(rails_path)) + worker_names(worker_metadatas(rails_path)) end def expand_queues(queues, all_queues = self.worker_queues) @@ -62,13 +46,18 @@ module Gitlab end end - def query_workers(query_string, queues) - worker_names(queues.select(&query_string_to_lambda(query_string))) + def query_queues(query_string, worker_metadatas) + matcher = SidekiqConfig::WorkerMatcher.new(query_string) + selected_metadatas = worker_metadatas.select do |worker_metadata| + matcher.match?(worker_metadata) + end + + worker_names(selected_metadatas) end def clear_memoization! - if instance_variable_defined?('@worker_queues') - remove_instance_variable('@worker_queues') + if instance_variable_defined?('@worker_metadatas') + remove_instance_variable('@worker_metadatas') end end @@ -77,53 +66,6 @@ module Gitlab def worker_names(workers) workers.map { |queue| queue[:name] } end - - def query_string_to_lambda(query_string) - or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string| - and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term| - predicate_for_term(term) - end - - lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } } - end - - lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } } - end - - def predicate_for_term(term) - match = term.match(QUERY_TERM_REGEX) - - raise InvalidTerm.new("Invalid term: #{term}") unless match - - _, lhs, op, rhs = *match - - predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR))) - end - - def predicate_for_op(op, predicate) - case op - when '=' - predicate - when '!=' - lambda { |worker| !predicate.call(worker) } - else - # This is unreachable because InvalidTerm will be raised instead, but - # keeping it allows to guard against that changing in future. - raise UnknownOperator.new("Unknown operator: #{op}") - end - end - - def predicate_factory(lhs, values) - values_block = QUERY_PREDICATES[lhs.to_sym] - - raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block - - lambda do |queue| - comparator = Array(queue[lhs.to_sym]).to_set - - values.map(&values_block).to_set.intersect?(comparator) - end - end end end end diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb new file mode 100644 index 00000000000..fe5ac10c65a --- /dev/null +++ b/lib/gitlab/sidekiq_config/worker_matcher.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqConfig + class WorkerMatcher + WILDCARD_MATCH = '*' + QUERY_OR_OPERATOR = '|' + QUERY_AND_OPERATOR = '&' + QUERY_CONCATENATE_OPERATOR = ',' + QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + + QUERY_PREDICATES = { + feature_category: :to_sym, + has_external_dependencies: lambda { |value| value == 'true' }, + name: :to_s, + resource_boundary: :to_sym, + tags: :to_sym, + urgency: :to_sym + }.freeze + + QueryError = Class.new(StandardError) + InvalidTerm = Class.new(QueryError) + UnknownOperator = Class.new(QueryError) + UnknownPredicate = Class.new(QueryError) + + def initialize(query_string) + @match_lambda = query_string_to_lambda(query_string) + end + + def match?(worker_metadata) + @match_lambda.call(worker_metadata) + end + + private + + def query_string_to_lambda(query_string) + return lambda { |_worker| true } if query_string.strip == WILDCARD_MATCH + + or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string| + and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term| + predicate_for_term(term) + end + + lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } } + end + + lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } } + end + + def predicate_for_term(term) + match = term.match(QUERY_TERM_REGEX) + + raise InvalidTerm.new("Invalid term: #{term}") unless match + + _, lhs, op, rhs = *match + + predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR))) + end + + def predicate_for_op(op, predicate) + case op + when '=' + predicate + when '!=' + lambda { |worker| !predicate.call(worker) } + else + # This is unreachable because InvalidTerm will be raised instead, but + # keeping it allows to guard against that changing in future. + raise UnknownOperator.new("Unknown operator: #{op}") + end + end + + def predicate_factory(lhs, values) + values_block = QUERY_PREDICATES[lhs.to_sym] + + raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block + + lambda do |queue| + comparator = Array(queue[lhs.to_sym]).to_set + + values.map(&values_block).to_set.intersect?(comparator) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 654b17c5740..b1fb3771c78 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -39,9 +39,7 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - instrumentation_values = job.slice(*::Gitlab::InstrumentationHelper.keys).stringify_keys - - output_payload.merge!(instrumentation_values) + output_payload.merge!(job[:instrumentation].stringify_keys) end def add_logging_extras!(job, output_payload) diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index a2696e17078..563a105484d 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -43,3 +43,5 @@ module Gitlab end end end + +Gitlab::SidekiqMiddleware.singleton_class.prepend_if_ee('EE::Gitlab::SidekiqMiddleware') diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb index 36204e1bee0..1b33743a0e9 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb @@ -8,7 +8,8 @@ module Gitlab # If enabled then it injects a job field that persists through the job execution class Client def call(_worker_class, job, _queue, _redis_pool) - return yield unless ::Feature.enabled?(:user_mode_in_session) + # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware + # Only when admin mode application setting is enabled might the admin_mode_user_id be non-nil here # Admin mode enabled in the original request or in a nested sidekiq job admin_mode_user_id = find_admin_user_id diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb index 6366867a0fa..c4e64705d6e 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb @@ -5,7 +5,8 @@ module Gitlab module AdminMode class Server def call(_worker, job, _queue) - return yield unless Feature.enabled?(:user_mode_in_session) + # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware + # Only when admin_mode setting is enabled can it be true here admin_mode_user_id = job['admin_mode_user_id'] diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index a66a4de4655..b542aa4fe4c 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -3,6 +3,24 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger + def self.keys + @keys ||= [ + :cpu_s, + :gitaly_calls, + :gitaly_duration_s, + :rugged_calls, + :rugged_duration_s, + :elasticsearch_calls, + :elasticsearch_duration_s, + :elasticsearch_timed_out_count, + *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, + *::Gitlab::Instrumentation::Redis.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, + *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS + ] + end + def call(worker, job, queue) ::Gitlab::InstrumentationHelper.init_instrumentation_data @@ -17,7 +35,10 @@ module Gitlab # because Sidekiq keeps a pristine copy of the original hash # before sending it to the middleware: # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 - ::Gitlab::InstrumentationHelper.add_instrumentation_data(job) + job[:instrumentation] = {}.tap do |instrumentation_values| + ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values) + instrumentation_values.slice!(*self.class.keys) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 60e79ee1188..66930a34319 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -10,6 +10,7 @@ module Gitlab def create_labels(worker_class, queue, job) worker_name = (job['wrapped'].presence || worker_class).to_s + worker = find_worker(worker_name, worker_class) labels = { queue: queue.to_s, worker: worker_name, @@ -18,15 +19,15 @@ module Gitlab feature_category: "", boundary: "" } - return labels unless worker_class && worker_class.include?(WorkerAttributes) + return labels unless worker.respond_to?(:get_urgency) - labels[:urgency] = worker_class.get_urgency.to_s - labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?) + labels[:urgency] = worker.get_urgency.to_s + labels[:external_dependencies] = bool_as_label(worker.worker_has_external_dependencies?) - feature_category = worker_class.get_feature_category + feature_category = worker.get_feature_category labels[:feature_category] = feature_category.to_s - resource_boundary = worker_class.get_worker_resource_boundary + resource_boundary = worker.get_worker_resource_boundary labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s labels @@ -35,6 +36,10 @@ module Gitlab def bool_as_label(value) value ? TRUE_LABEL : FALSE_LABEL end + + def find_worker(worker_name, worker_class) + Gitlab::SidekiqConfig::DEFAULT_WORKERS.fetch(worker_name, worker_class) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index cf768811ffd..f5fee8050ac 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -21,6 +21,16 @@ module Gitlab Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME labels = create_labels(worker.class, queue, job) + instrument(job, labels) do + yield + end + end + + protected + + attr_reader :metrics + + def instrument(job, labels) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration @@ -50,19 +60,18 @@ module Gitlab # job_status: done, fail match the job_status attribute in structured logging labels[:job_status] = job_succeeded ? "done" : "fail" + instrumentation = job[:instrumentation] || {} @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) - @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job)) - @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job)) - @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job)) - @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job)) - @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job)) + @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) end end - private - def init_metrics { sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), @@ -81,29 +90,33 @@ module Gitlab } end + private + def get_thread_cputime defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end - def get_redis_time(job) - job.fetch(:redis_duration_s, 0) + def get_redis_time(payload) + payload.fetch(:redis_duration_s, 0) end - def get_redis_calls(job) - job.fetch(:redis_calls, 0) + def get_redis_calls(payload) + payload.fetch(:redis_calls, 0) end - def get_elasticsearch_time(job) - job.fetch(:elasticsearch_duration_s, 0) + def get_elasticsearch_time(payload) + payload.fetch(:elasticsearch_duration_s, 0) end - def get_elasticsearch_calls(job) - job.fetch(:elasticsearch_calls, 0) + def get_elasticsearch_calls(payload) + payload.fetch(:elasticsearch_calls, 0) end - def get_gitaly_time(job) - job.fetch(:gitaly_duration_s, 0) + def get_gitaly_time(payload) + payload.fetch(:gitaly_duration_s, 0) end end end end + +Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics') diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb index 807c27a71ff..4b71dfc0c1b 100644 --- a/lib/gitlab/sidekiq_queue.rb +++ b/lib/gitlab/sidekiq_queue.rb @@ -21,7 +21,7 @@ module Gitlab job_search_metadata = search_metadata .stringify_keys - .slice(*Labkit::Context::KNOWN_KEYS) + .slice(*Gitlab::ApplicationContext::KNOWN_KEYS) .transform_keys { |key| "meta.#{key}" } .compact diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index fcc120112f2..e184afa0032 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -36,7 +36,9 @@ module Gitlab attr_accessor :project, :current_user, :params, :chat_name def initialize(project, chat_name, params = {}) - @project, @current_user, @params = project, chat_name.user, params.dup + @project = project + @current_user = chat_name.user + @params = params.dup @chat_name = chat_name end diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 552456f5836..8841fef702e 100644 --- a/lib/gitlab/slash_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -12,16 +12,18 @@ module Gitlab private - def fallback_message - "New issue #{issue.to_reference}: #{issue.title}" + def pretext + "I created an issue on #{author_profile_link}'s behalf: *#{issue_link}* in #{project_link}" end - def fields_with_markdown - %i(title pretext text fields) + def issue_link + "[#{issue.to_reference}](#{project_issue_url(issue.project, issue)})" end - def pretext - "I created an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}" + def response_message(custom_pretext: pretext) + { + text: pretext + } end end end diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb index 10a545e28ac..40fd7ee4f20 100644 --- a/lib/gitlab/slash_commands/run.rb +++ b/lib/gitlab/slash_commands/run.rb @@ -5,7 +5,7 @@ module Gitlab # Slash command for triggering chatops jobs. class Run < BaseCommand def self.match(text) - /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/.match(text) + /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/m.match(text) end def self.help_message diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb index 1b87d3bb626..fd70def8e7c 100644 --- a/lib/gitlab/slug/environment.rb +++ b/lib/gitlab/slug/environment.rb @@ -26,16 +26,13 @@ module Gitlab # Repeated dashes are invalid (OpenShift limitation) slugified.squeeze!('-') - slugified = - if slugified.size > 24 || slugified != name - # Maximum length: 24 characters (OpenShift limitation) - shorten_and_add_suffix(slugified) - else - # Cannot end with a dash (Kubernetes label limitation) - slugified.chomp('-') - end - - slugified + if slugified.size > 24 || slugified != name + # Maximum length: 24 characters (OpenShift limitation) + shorten_and_add_suffix(slugified) + else + # Cannot end with a dash (Kubernetes label limitation) + slugified.chomp('-') + end end private diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb index 7817a2a1ce2..8f37602aeaa 100644 --- a/lib/gitlab/sql/cte.rb +++ b/lib/gitlab/sql/cte.rb @@ -15,20 +15,27 @@ module Gitlab # Namespace # with(cte.to_arel). # from(cte.alias_to(ns)) + # + # To skip materialization of the CTE query by passing materialized: false + # More context: https://www.postgresql.org/docs/12/queries-with.html + # + # cte = CTE.new(:my_cte_name, materialized: false) + # class CTE attr_reader :table, :query # name - The name of the CTE as a String or Symbol. - def initialize(name, query) + def initialize(name, query, materialized: true) @table = Arel::Table.new(name) @query = query + @materialized = materialized end # Returns the Arel relation for this CTE. def to_arel sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})") - Arel::Nodes::As.new(table, sql) + Gitlab::Database::AsWithMaterialized.new(table, sql, materialized: @materialized) end # Returns an "AS" statement that aliases the CTE name as the given table diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index e45ac5d4765..607ce10d778 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -23,9 +23,11 @@ module Gitlab attr_reader :table # name - The name of the CTE as a String or Symbol. - def initialize(name) + # union_args - The arguments supplied to Gitlab::SQL::Union class when building inner recursive query + def initialize(name, union_args: {}) @table = Arel::Table.new(name) @queries = [] + @union_args = union_args end # Adds a query to the body of the CTE. @@ -37,7 +39,7 @@ module Gitlab # Returns the Arel relation for this CTE. def to_arel - sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql) + sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries, **@union_args).to_sql) Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql)) end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb index d58a1415493..59a808eafa9 100644 --- a/lib/gitlab/sql/set_operator.rb +++ b/lib/gitlab/sql/set_operator.rb @@ -8,6 +8,9 @@ module Gitlab # ORDER BYs are dropped from the relations as the final sort order is not # guaranteed any way. # + # remove_order: false option can be used in special cases where the + # ORDER BY is necessary for the query. + # # Example usage: # # union = Gitlab::SQL::Union.new([user.personal_projects, user.projects]) @@ -15,9 +18,10 @@ module Gitlab # # Project.where("id IN (#{sql})") class SetOperator - def initialize(relations, remove_duplicates: true) + def initialize(relations, remove_duplicates: true, remove_order: true) @relations = relations @remove_duplicates = remove_duplicates + @remove_order = remove_order end def self.operator_keyword @@ -30,7 +34,9 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) + relations.map do |rel| + remove_order ? rel.reorder(nil).to_sql : rel.to_sql + end.reject(&:blank?) end if fragments.any? @@ -47,7 +53,7 @@ module Gitlab private - attr_reader :relations, :remove_duplicates + attr_reader :relations, :remove_duplicates, :remove_order end end end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 7fb3487a5e5..c4e95284c50 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -4,8 +4,8 @@ module Gitlab module SQL # Class for building SQL UNION statements. # - # ORDER BYs are dropped from the relations as the final sort order is not - # guaranteed any way. + # By default ORDER BYs are dropped from the relations as the final sort + # order is not guaranteed any way. # # Example usage: # diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb index 315c603c1dd..4180f6ccf00 100644 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -28,7 +28,7 @@ module Gitlab def to_hash_with_defaults # NOTE: The current approach of simply mapping all the descendents' keys and values ('config') # into a flat hash may need to be enhanced as we add more complex, non-scalar entries. - @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h + @global.descendants.to_h { |descendant| [descendant.key, descendant.config] } end private diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index a918e7bec80..3072210d7c8 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -6,6 +6,11 @@ module Gitlab ::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com' end - SUBSCRIPTIONS_URL = ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url).freeze + def self.subscriptions_url + ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url) + end end end + +Gitlab::SubscriptionPortal.prepend_if_jh('JH::Gitlab::SubscriptionPortal') +Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index dc006877129..31e11f73fe7 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -130,10 +130,10 @@ module Gitlab return [] if project && !project.repository.exists? if categories.any? - categories.keys.map do |category| + categories.keys.to_h do |category| files = self.by_category(category, project) [category, files.map { |t| { key: t.key, name: t.name, content: t.content } }] - end.to_h + end else files = self.all(project) files.map { |t| { key: t.key, name: t.name, content: t.content } } diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 9bb793a75cc..b16ae39bcee 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -4,35 +4,18 @@ module Gitlab module Tracking SNOWPLOW_NAMESPACE = 'gl' - module ControllerConcern - extend ActiveSupport::Concern - - protected - - def track_event(action = action_name, **args) - category = args.delete(:category) || self.class.name - Gitlab::Tracking.event(category, action.to_s, **args) - end - - def track_self_describing_event(schema_url, data:, **args) - Gitlab::Tracking.self_describing_event(schema_url, data: data, **args) - end - end - class << self def enabled? Gitlab::CurrentSettings.snowplow_enabled? end - def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists - contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context, *context] + def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists + contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] snowplow.event(category, action, label: label, property: property, value: value, context: contexts) product_analytics.event(category, action, label: label, property: property, value: value, context: contexts) - end - - def self_describing_event(schema_url, data:, context: nil) - snowplow.self_describing_event(schema_url, data: data, context: context) + rescue => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end def snowplow_options(group) diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index 4fa844de325..e548532e061 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -15,13 +15,6 @@ module Gitlab tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) end - def self_describing_event(schema_url, data:, context: nil) - return unless enabled? - - event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, data) - tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) - end - private def enabled? diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 8ce16c11267..da030649f76 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,11 +3,11 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze - GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-4' + GITLAB_RAILS_SOURCE = 'gitlab-rails' - def initialize(namespace: nil, project: nil, user: nil, **data) - @data = data + def initialize(namespace: nil, project: nil, user: nil, **extra) + @extra = extra end def to_context @@ -35,8 +35,9 @@ module Gitlab def to_h { environment: environment, - source: source - }.merge(@data) + source: source, + extra: @extra + } end end end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 6a3e2062070..706c0925302 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -35,6 +35,10 @@ module Gitlab matches end + def match(text) + scan_regexp.match(text) + end + def match?(text) text.present? && scan(text).present? end diff --git a/lib/gitlab/updated_notes_paginator.rb b/lib/gitlab/updated_notes_paginator.rb index 3d3d0e5bf9e..d5c01bde6b3 100644 --- a/lib/gitlab/updated_notes_paginator.rb +++ b/lib/gitlab/updated_notes_paginator.rb @@ -37,8 +37,8 @@ module Gitlab end def fetch_page(relation) - relation = relation.by_updated_at - notes = relation.at_most(LIMIT + 1).to_a + relation = relation.order_updated_asc.with_order_id_asc + notes = relation.limit(LIMIT + 1).to_a return [notes, false] unless notes.size > LIMIT diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index 1dc660e574b..6b185a5a1e9 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -33,6 +33,10 @@ module Gitlab object[:description] end + def render_object_schema(object) + "[Object JSON schema](#{object.json_schema_path})" + end + def render_yaml_link(yaml_path) "[YAML definition](#{yaml_path})" end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 19ad668019e..26f1aa4396d 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -27,6 +27,9 @@ = render_name(name) \ = render_description(object.attributes) + - if object.has_json_schema? + \ + = render_object_schema(object) \ = render_yaml_link(object.yaml_path) \ diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 4cb83348478..9c4255a7c92 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -5,6 +5,7 @@ module Gitlab class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' + SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze attr_reader :path attr_reader :attributes @@ -22,6 +23,16 @@ module Gitlab attributes end + def json_schema_path + return '' unless has_json_schema? + + "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}" + end + + def has_json_schema? + attributes[:value_type] == 'object' && attributes[:object_json_schema].present? + end + def yaml_path "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" end @@ -29,7 +40,15 @@ module Gitlab def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + error_message = <<~ERROR_MSG + Error type: #{error['type']} + Data: #{error['data']} + Path: #{error['data_pointer']} + Details: #{error['details']} + Metric file: #{path} + ERROR_MSG + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message)) end end end @@ -38,10 +57,11 @@ module Gitlab class << self def paths - @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')] + @paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')] end - def definitions + def definitions(skip_validation: false) + @skip_validation = skip_validation @definitions ||= load_all! end @@ -49,6 +69,10 @@ module Gitlab @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) end + def dump_metrics_yaml + @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml + end + private def load_all! @@ -87,7 +111,7 @@ module Gitlab end def skip_validation? - !!attributes[:skip_validation] + !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status]) end end end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 1aeca87d849..f77c8cab39c 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -7,7 +7,7 @@ module Gitlab UNION_OF_AGGREGATED_METRICS = 'OR' INTERSECTION_OF_AGGREGATED_METRICS = 'AND' ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml') + AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml') AggregatedMetricError = Class.new(StandardError) UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 33f025770e0..49581169452 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -6,6 +6,8 @@ module Gitlab module NamesSuggestions class Generator < ::Gitlab::UsageData FREE_TEXT_METRIC_NAME = "<please fill metric name>" + REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" + CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" class << self def generate(key_path) @@ -23,7 +25,7 @@ module Gitlab end def redis_usage_counter - FREE_TEXT_METRIC_NAME + REDIS_EVENT_METRIC_NAME end def alt_usage_data(*) @@ -31,7 +33,7 @@ module Gitlab end def redis_usage_data_totals(counter) - counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME} + counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME } end def sum(relation, column, *rest) @@ -47,49 +49,160 @@ module Gitlab end def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) - parts = [prefix] + # rubocop: disable CodeReuse/ActiveRecord + relation = relation.unscope(where: :created_at) + # rubocop: enable CodeReuse/ActiveRecord - if column - parts << parse_target(column) + parts = [prefix] + arel_column = arelize_column(relation, column) + + # nil as column indicates that the counting would use fallback value of primary key. + # Because counting primary key from relation is the conceptual equal to counting all + # records from given relation, in order to keep name suggestion more condensed + # primary key column is skipped. + # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not + # as count_id_from_issues since it does not add more information to the name suggestion + if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] + parts << arel_column.name parts << 'from' end - source = parse_source(relation) - constraints = parse_constraints(relation: relation, column: column, distinct: distinct) + arel = arel_query(relation: relation, column: arel_column, distinct: distinct) + constraints = parse_constraints(relation: relation, arel: arel) + + # In some cases due to performance reasons metrics are instrumented with joined relations + # where relation listed in FROM statement is not the one that includes counted attribute + # in such situations to make name suggestion more intuitive source should be inferred based + # on the relation that provide counted attribute + # EG: SELECT COUNT(deployments.environment_id) FROM clusters + # JOIN deployments ON deployments.cluster_id = cluster.id + # should be translated into: + # count_environment_id_from_deployments_with_clusters + # instead of + # count_environment_id_from_clusters_with_deployments + actual_source = parse_source(relation, arel_column) + + append_constraints_prompt(actual_source, [constraints], parts) + + parts << actual_source + parts += process_joined_relations(actual_source, arel, relation, constraints) + parts.compact.join('_').delete('"') + end - if constraints.include?(source) - parts << "<adjective describing: '#{constraints}'>" - end + def append_constraints_prompt(target, constraints, parts) + applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + return unless applicable_constraints.any? - parts << source - parts.compact.join('_') + parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } end - def parse_constraints(relation:, column: nil, distinct: nil) + def parse_constraints(relation:, arel:) connection = relation.connection ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints .new(connection) - .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection)) + .accept(arel, collector(connection)) .value end - def parse_target(column) - if column.is_a?(Arel::Attribute) - "#{column.relation.name}.#{column.name}" - else + # TODO: joins with `USING` keyword + def process_joined_relations(actual_source, arel, relation, where_constraints) + joins = parse_joins(connection: relation.connection, arel: arel) + return [] unless joins.any? + + sources = [relation.table_name, *joins.map { |join| join[:source] }] + joins = extract_joins_targets(joins, sources) + + relations = if actual_source != relation.table_name + build_relations_tree(joins + [{ source: relation.table_name }], actual_source) + else + # in case where counter attribute comes from joined relations, the relations + # diagram has to be built bottom up, thus source and target are reverted + build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) + end + + collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) + end + + def parse_joins(connection:, arel:) + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins + .new(connection) + .accept(arel) + end + + def extract_joins_targets(joins, sources) + joins.map do |join| + source_regex = /(#{join[:source]})\.(\w+_)*id/i + + tables_except_src = (sources - [join[:source]]).join('|') + target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i + + join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i + matched = join_cond_regex.match(join[:constraints]) + + if matched + join[:target] = matched[:target] + join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') + end + + join + end + end + + def build_relations_tree(joins, parent, source_key: :source, target_key: :target) + return [] if joins.blank? + + tree = {} + tree[parent] = [] + + joins.each do |join| + if join[source_key] == parent + tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) + end + end + tree + end + + def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) + conjunction = conjunctions.next + relations.each do |subtree| + subtree.each do |parent, children| + parts << "<#{conjunction}>" + join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) + append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + parts << parent + collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) + end + end + parts + end + + def arelize_column(relation, column) + case column + when Arel::Attribute column + when NilClass + Arel::Table.new(relation.table_name)[relation.primary_key] + when String + if column.include?('.') + table, col = column.split('.') + Arel::Table.new(table)[col] + else + Arel::Table.new(relation.table_name)[column] + end + when Symbol + arelize_column(relation, column.to_s) end end - def parse_source(relation) - relation.table_name + def parse_source(relation, column) + column.relation.name || relation.table_name end def collector(connection) Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) end - def arel(relation:, column: nil, distinct: nil) + def arel_query(relation:, column: nil, distinct: nil) column ||= relation.primary_key if column.is_a?(Arel::Attribute) diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb new file mode 100644 index 00000000000..d52e4903f3c --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + module RelationParsers + class Joins < ::Arel::Visitors::PostgreSQL + def accept(object) + object.source.right.map do |join| + visit(join, collector) + end + end + + private + + # rubocop:disable Naming/MethodName + def visit_Arel_Nodes_StringJoin(object, collector) + result = visit(object.left, collector) + source, constraints = result.value.split('ON') + { + source: source.split('JOIN').last&.strip, + constraints: constraints&.strip + }.compact + end + + def visit_Arel_Nodes_FullOuterJoin(object, _) + parse_join(object) + end + + def visit_Arel_Nodes_OuterJoin(object, _) + parse_join(object) + end + + def visit_Arel_Nodes_RightOuterJoin(object, _) + parse_join(object) + end + + def visit_Arel_Nodes_InnerJoin(object, _) + { + source: visit(object.left, collector).value, + constraints: object.right ? visit(object.right.expr, collector).value : nil + }.compact + end + # rubocop:enable Naming/MethodName + + def parse_join(object) + { + source: visit(object.left, collector).value, + constraints: visit(object.right.expr, collector).value + } + end + + def quote(value) + "#{value}" + end + + def quote_table_name(name) + "#{name}" + end + + def quote_column_name(name) + "#{name}" + end + + def collector + Arel::Collectors::SubstituteBinds.new(@connection, Arel::Collectors::SQLString.new) + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5dc3f71329d..b36ca38cd64 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -87,7 +87,7 @@ module Gitlab # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) + issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) { counts: { @@ -138,7 +138,7 @@ module Gitlab in_review_folder: count(::Environment.in_review_folder), grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), - issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id), + issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)), issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), @@ -146,9 +146,9 @@ module Gitlab issues_created_from_alerts: total_alert_issues, issues_created_gitlab_alerts: issues_created_manually_from_alerts, issues_created_manually_from_alerts: issues_created_manually_from_alerts, - incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id), - alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id), - incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id), + incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)), + alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), + incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -389,8 +389,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def container_expiration_policies_usage results = {} - start = ::Project.minimum(:id) - finish = ::Project.maximum(:id) + start = minimum_id(Project) + finish = maximum_id(Project) results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) # rubocop: disable UsageData/LargeTable @@ -591,7 +591,7 @@ module Gitlab { events: distinct_count(::Event.where(time_period), :author_id), groups: distinct_count(::GroupMember.where(time_period), :user_id), - users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), + users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), unique_users_all_imports: unique_users_all_imports(time_period), @@ -636,8 +636,8 @@ module Gitlab clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id), clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), - start: user_minimum_id, - finish: user_maximum_id), + start: minimum_id(User), + finish: maximum_id(User)), projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), @@ -691,12 +691,12 @@ module Gitlab def usage_activity_by_stage_verify(time_period) { ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id), - ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), - ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), - ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), - ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), + ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), + ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), + ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id), - ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id), clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period) } @@ -711,6 +711,8 @@ module Gitlab end def redis_hll_counters + return {} unless Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) + { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end @@ -799,8 +801,8 @@ module Gitlab end def distinct_count_service_desk_enabled_projects(time_period) - project_creator_id_start = user_minimum_id - project_creator_id_finish = user_maximum_id + project_creator_id_start = minimum_id(User) + project_creator_id_finish = maximum_id(User) distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord end @@ -832,57 +834,9 @@ module Gitlab def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. - add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), - count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id), - count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id) - end - - def user_minimum_id - strong_memoize(:user_minimum_id) do - ::User.minimum(:id) - end - end - - def user_maximum_id - strong_memoize(:user_maximum_id) do - ::User.maximum(:id) - end - end - - def issue_minimum_id - strong_memoize(:issue_minimum_id) do - ::Issue.minimum(:id) - end - end - - def issue_maximum_id - strong_memoize(:issue_maximum_id) do - ::Issue.maximum(:id) - end - end - - def deployment_minimum_id - strong_memoize(:deployment_minimum_id) do - ::Deployment.minimum(:id) - end - end - - def deployment_maximum_id - strong_memoize(:deployment_maximum_id) do - ::Deployment.maximum(:id) - end - end - - def project_minimum_id - strong_memoize(:project_minimum_id) do - ::Project.minimum(:id) - end - end - - def project_maximum_id - strong_memoize(:project_maximum_id) do - ::Project.maximum(:id) - end + add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)), + count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)), + count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)) end def self_monitoring_project @@ -916,7 +870,7 @@ module Gitlab end def deployment_count(relation) - count relation, start: deployment_minimum_id, finish: deployment_maximum_id + count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment) end def project_imports(time_period) diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml deleted file mode 100644 index 4c2355d526a..00000000000 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml +++ /dev/null @@ -1,108 +0,0 @@ -# code_review_extension_category_monthly_active_users -# This is only metrics related to the VS Code Extension for now. -# -# code_review_category_monthly_active_users -# This is the user based metrics. These should only be user based metrics and only be related to the Code Review things inside of GitLab. -# -# code_review_group_monthly_active_users -# This is an aggregation of both of the above aggregations. It's intended to represent all users who interact with our group across all of our categories. ---- -- name: code_review_group_monthly_active_users - operator: OR - feature_flag: usage_data_code_review_aggregation - source: redis - time_frame: [7d, 28d] - events: [ - 'i_code_review_user_single_file_diffs', - 'i_code_review_user_create_mr', - 'i_code_review_user_close_mr', - 'i_code_review_user_reopen_mr', - 'i_code_review_user_resolve_thread', - 'i_code_review_user_unresolve_thread', - 'i_code_review_edit_mr_title', - 'i_code_review_edit_mr_desc', - 'i_code_review_user_merge_mr', - 'i_code_review_user_create_mr_comment', - 'i_code_review_user_edit_mr_comment', - 'i_code_review_user_remove_mr_comment', - 'i_code_review_user_create_review_note', - 'i_code_review_user_publish_review', - 'i_code_review_user_create_multiline_mr_comment', - 'i_code_review_user_edit_multiline_mr_comment', - 'i_code_review_user_remove_multiline_mr_comment', - 'i_code_review_user_add_suggestion', - 'i_code_review_user_apply_suggestion', - 'i_code_review_user_assigned', - 'i_code_review_user_review_requested', - 'i_code_review_user_approve_mr', - 'i_code_review_user_unapprove_mr', - 'i_code_review_user_marked_as_draft', - 'i_code_review_user_unmarked_as_draft', - 'i_code_review_user_approval_rule_added', - 'i_code_review_user_approval_rule_deleted', - 'i_code_review_user_approval_rule_edited', - 'i_code_review_user_vs_code_api_request', - 'i_code_review_user_toggled_task_item_status', - 'i_code_review_user_create_mr_from_issue', - 'i_code_review_user_mr_discussion_locked', - 'i_code_review_user_mr_discussion_unlocked', - 'i_code_review_user_time_estimate_changed', - 'i_code_review_user_time_spent_changed', - 'i_code_review_user_assignees_changed', - 'i_code_review_user_reviewers_changed', - 'i_code_review_user_milestone_changed', - 'i_code_review_user_labels_changed' - ] -- name: code_review_category_monthly_active_users - operator: OR - feature_flag: usage_data_code_review_aggregation - source: redis - time_frame: [7d, 28d] - events: [ - 'i_code_review_user_single_file_diffs', - 'i_code_review_user_create_mr', - 'i_code_review_user_close_mr', - 'i_code_review_user_reopen_mr', - 'i_code_review_user_resolve_thread', - 'i_code_review_user_unresolve_thread', - 'i_code_review_edit_mr_title', - 'i_code_review_edit_mr_desc', - 'i_code_review_user_merge_mr', - 'i_code_review_user_create_mr_comment', - 'i_code_review_user_edit_mr_comment', - 'i_code_review_user_remove_mr_comment', - 'i_code_review_user_create_review_note', - 'i_code_review_user_publish_review', - 'i_code_review_user_create_multiline_mr_comment', - 'i_code_review_user_edit_multiline_mr_comment', - 'i_code_review_user_remove_multiline_mr_comment', - 'i_code_review_user_add_suggestion', - 'i_code_review_user_apply_suggestion', - 'i_code_review_user_assigned', - 'i_code_review_user_review_requested', - 'i_code_review_user_approve_mr', - 'i_code_review_user_unapprove_mr', - 'i_code_review_user_marked_as_draft', - 'i_code_review_user_unmarked_as_draft', - 'i_code_review_user_approval_rule_added', - 'i_code_review_user_approval_rule_deleted', - 'i_code_review_user_approval_rule_edited', - 'i_code_review_user_toggled_task_item_status', - 'i_code_review_user_create_mr_from_issue', - 'i_code_review_user_mr_discussion_locked', - 'i_code_review_user_mr_discussion_unlocked', - 'i_code_review_user_time_estimate_changed', - 'i_code_review_user_time_spent_changed', - 'i_code_review_user_assignees_changed', - 'i_code_review_user_reviewers_changed', - 'i_code_review_user_milestone_changed', - 'i_code_review_user_labels_changed' - ] -- name: code_review_extension_category_monthly_active_users - operator: OR - feature_flag: usage_data_code_review_aggregation - source: redis - time_frame: [7d, 28d] - events: [ - 'i_code_review_user_vs_code_api_request' - ] diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml deleted file mode 100644 index 73a55b5d5fa..00000000000 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Aggregated metrics that include EE only event names within `events:` attribute have to be defined at ee/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml -# instead of this file. -#- name: unique name of aggregated metric -# operator: aggregation operator. Valid values are: -# - "OR": counts unique elements that were observed triggering any of following events -# - "AND": counts unique elements that were observed triggering all of following events -# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes -# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. -# source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are: -# - database -# - redis -# time_frame: defines time frames for aggregated metrics: -# - 7d - last 7 days -# - 28d - last 28 days -# - all - all historical available data, this time frame is not available for redis source -# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. -# Corresponding feature flag should have `default_enabled` attribute set to `false`. -# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. ---- -- name: compliance_features_track_unique_visits_union - operator: OR - source: redis - time_frame: [7d, 28d] - events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] -- name: product_analytics_test_metrics_union - operator: OR - source: redis - time_frame: [7d, 28d] - events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] -- name: product_analytics_test_metrics_intersection - operator: AND - source: redis - time_frame: [7d, 28d] - events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] -- name: incident_management_alerts_total_unique_counts - operator: OR - source: redis - time_frame: [7d, 28d] - events: [ - 'incident_management_alert_status_changed', - 'incident_management_alert_assigned', - 'incident_management_alert_todo', - 'incident_management_alert_create_incident' - ] -- name: incident_management_incidents_total_unique_counts - operator: OR - source: redis - time_frame: [7d, 28d] - events: [ - 'incident_management_incident_created', - 'incident_management_incident_reopened', - 'incident_management_incident_closed', - 'incident_management_incident_assigned', - 'incident_management_incident_todo', - 'incident_management_incident_comment', - 'incident_management_incident_zoom_meeting', - 'incident_management_incident_published', - 'incident_management_incident_relate', - 'incident_management_incident_unrelate', - 'incident_management_incident_change_confidential' - ] -- name: i_testing_paid_monthly_active_user_total - operator: OR - source: redis - time_frame: [7d, 28d] - events: [ - 'i_testing_web_performance_widget_total', - 'i_testing_full_code_quality_report_total', - 'i_testing_group_code_coverage_visit_total', - 'i_testing_load_performance_widget_total', - 'i_testing_metrics_report_widget_total' - ] diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index d28fd17a989..4ab310a2519 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -22,11 +22,11 @@ module Gitlab::UsageDataCounters end def totals - known_events.map { |event| [counter_key(event), read(event)] }.to_h + known_events.to_h { |event| [counter_key(event), read(event)] } end def fallback_totals - known_events.map { |event| [counter_key(event), -1] }.to_h + known_events.to_h { |event| [counter_key(event), -1] } end def fetch_supported_event(event_name) diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 772a4623280..c9106d7c6b8 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -2,7 +2,7 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter - REDIS_SLOT = 'ci_templates'.freeze + REDIS_SLOT = 'ci_templates' # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_` TEMPLATE_TO_EVENT = { @@ -20,8 +20,6 @@ module Gitlab::UsageDataCounters class << self def track_unique_project_event(project_id:, template:, config_source:) - return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml) - if event = unique_project_event(template, config_source) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 336bef081a6..a8691169fb8 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -151,13 +151,16 @@ module Gitlab aggregation = events.first[:aggregation] keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) + + return FALLBACK unless keys.any? + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end def feature_enabled?(event) return true if event[:feature_flag].blank? - Feature.enabled?(event[:feature_flag], default_enabled: :yaml) + Feature.enabled?(event[:feature_flag], default_enabled: :yaml) && Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) end # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index c2662a74432..6f5f878501f 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -34,120 +34,120 @@ module Gitlab ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self - def track_issue_created_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CREATED, author, time) + def track_issue_created_action(author:) + track_unique_action(ISSUE_CREATED, author) end - def track_issue_title_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_TITLE_CHANGED, author, time) + def track_issue_title_changed_action(author:) + track_unique_action(ISSUE_TITLE_CHANGED, author) end - def track_issue_description_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time) + def track_issue_description_changed_action(author:) + track_unique_action(ISSUE_DESCRIPTION_CHANGED, author) end - def track_issue_assignee_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time) + def track_issue_assignee_changed_action(author:) + track_unique_action(ISSUE_ASSIGNEE_CHANGED, author) end - def track_issue_made_confidential_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time) + def track_issue_made_confidential_action(author:) + track_unique_action(ISSUE_MADE_CONFIDENTIAL, author) end - def track_issue_made_visible_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MADE_VISIBLE, author, time) + def track_issue_made_visible_action(author:) + track_unique_action(ISSUE_MADE_VISIBLE, author) end - def track_issue_closed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CLOSED, author, time) + def track_issue_closed_action(author:) + track_unique_action(ISSUE_CLOSED, author) end - def track_issue_reopened_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_REOPENED, author, time) + def track_issue_reopened_action(author:) + track_unique_action(ISSUE_REOPENED, author) end - def track_issue_label_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_LABEL_CHANGED, author, time) + def track_issue_label_changed_action(author:) + track_unique_action(ISSUE_LABEL_CHANGED, author) end - def track_issue_milestone_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) + def track_issue_milestone_changed_action(author:) + track_unique_action(ISSUE_MILESTONE_CHANGED, author) end - def track_issue_cross_referenced_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CROSS_REFERENCED, author, time) + def track_issue_cross_referenced_action(author:) + track_unique_action(ISSUE_CROSS_REFERENCED, author) end - def track_issue_moved_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MOVED, author, time) + def track_issue_moved_action(author:) + track_unique_action(ISSUE_MOVED, author) end - def track_issue_related_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_RELATED, author, time) + def track_issue_related_action(author:) + track_unique_action(ISSUE_RELATED, author) end - def track_issue_unrelated_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_UNRELATED, author, time) + def track_issue_unrelated_action(author:) + track_unique_action(ISSUE_UNRELATED, author) end - def track_issue_marked_as_duplicate_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author, time) + def track_issue_marked_as_duplicate_action(author:) + track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author) end - def track_issue_locked_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_LOCKED, author, time) + def track_issue_locked_action(author:) + track_unique_action(ISSUE_LOCKED, author) end - def track_issue_unlocked_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_UNLOCKED, author, time) + def track_issue_unlocked_action(author:) + track_unique_action(ISSUE_UNLOCKED, author) end - def track_issue_designs_added_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESIGNS_ADDED, author, time) + def track_issue_designs_added_action(author:) + track_unique_action(ISSUE_DESIGNS_ADDED, author) end - def track_issue_designs_modified_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESIGNS_MODIFIED, author, time) + def track_issue_designs_modified_action(author:) + track_unique_action(ISSUE_DESIGNS_MODIFIED, author) end - def track_issue_designs_removed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESIGNS_REMOVED, author, time) + def track_issue_designs_removed_action(author:) + track_unique_action(ISSUE_DESIGNS_REMOVED, author) end - def track_issue_due_date_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DUE_DATE_CHANGED, author, time) + def track_issue_due_date_changed_action(author:) + track_unique_action(ISSUE_DUE_DATE_CHANGED, author) end - def track_issue_time_estimate_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author, time) + def track_issue_time_estimate_changed_action(author:) + track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author) end - def track_issue_time_spent_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) + def track_issue_time_spent_changed_action(author:) + track_unique_action(ISSUE_TIME_SPENT_CHANGED, author) end - def track_issue_comment_added_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_COMMENT_ADDED, author, time) + def track_issue_comment_added_action(author:) + track_unique_action(ISSUE_COMMENT_ADDED, author) end - def track_issue_comment_edited_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_COMMENT_EDITED, author, time) + def track_issue_comment_edited_action(author:) + track_unique_action(ISSUE_COMMENT_EDITED, author) end - def track_issue_comment_removed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_COMMENT_REMOVED, author, time) + def track_issue_comment_removed_action(author:) + track_unique_action(ISSUE_COMMENT_REMOVED, author) end - def track_issue_cloned_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CLONED, author, time) + def track_issue_cloned_action(author:) + track_unique_action(ISSUE_CLONED, author) end private - def track_unique_action(action, author, time) + def track_unique_action(action, author) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end end end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 9c19c9e8b8c..3c692f2b1af 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -3,89 +3,74 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_auto_devops_build category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_auto_devops_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects # Explicit include:template pipeline events - name: p_ci_templates_5_min_production_app category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_aws_cf_deploy_ec2 category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_aws_deploy_ecs category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops_build category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops_deploy_latest category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_terraform_base_latest category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 80a79682338..077864032e8 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -91,6 +91,11 @@ redis_slot: analytics aggregation: weekly feature_flag: track_unique_visits +- name: i_analytics_dev_ops_adoption + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_merge_request category: analytics redis_slot: analytics @@ -242,6 +247,12 @@ category: incident_management_alerts aggregation: weekly feature_flag: usage_data_incident_management_alert_create_incident +# Incident management on-call +- name: i_incident_management_oncall_notification_sent + redis_slot: incident_management + category: incident_management_oncall + aggregation: weekly + feature_flag: usage_data_i_incident_management_oncall_notification_sent # Testing category - name: i_testing_test_case_parsed category: testing @@ -283,6 +294,11 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders +- name: i_testing_summary_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_summary_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -444,13 +460,19 @@ redis_slot: pipeline_authoring aggregation: weekly feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile -# Epic events -# -# We are using the same slot of issue events 'project_management' for -# epic events to allow data aggregation. -# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 -- name: g_project_management_epic_created - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity +# Merge request widgets +- name: users_expanding_secure_security_report + redis_slot: secure + category: secure + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data +- name: users_expanding_testing_code_quality_report + redis_slot: testing + category: testing + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data +- name: users_expanding_testing_accessibility_report + redis_slot: testing + category: testing + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml new file mode 100644 index 00000000000..80460dbe4d2 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -0,0 +1,142 @@ +# Epic events +# +# We are using the same slot of issue events 'project_management' for +# epic events to allow data aggregation. +# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 +- name: g_project_management_epic_created + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_epic_titles + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_epic_descriptions + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +# epic notes + +- name: g_project_management_users_creating_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_destroying_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +# start date events + +- name: g_project_management_users_setting_epic_start_date_as_fixed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_fixed_epic_start_date + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_start_date_as_inherited + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +# due date events + +- name: g_project_management_users_setting_epic_due_date_as_fixed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_fixed_epic_due_date + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_due_date_as_inherited + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_issue_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_issue_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_issue_moved_from_project + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_closed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_reopened + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: 'g_project_management_issue_promoted_to_epic' + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_confidential + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_visible + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_users_changing_labels + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_destroyed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb index 7a76180cb08..aae2d144c5b 100644 --- a/lib/gitlab/usage_data_counters/note_counter.rb +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -24,13 +24,13 @@ module Gitlab::UsageDataCounters end def totals - COUNTABLE_TYPES.map do |countable_type| + COUNTABLE_TYPES.to_h do |countable_type| [counter_key(countable_type), read(:create, countable_type)] - end.to_h + end end def fallback_totals - COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h + COUNTABLE_TYPES.to_h { |counter_key| [counter_key(counter_key), -1] } end private diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index 15c68fb3945..ed3df7dcf75 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -28,7 +28,7 @@ module Gitlab 'unassign_reviewer' when 'request_review', 'reviewer' 'assign_reviewer' - when 'spend' + when 'spend', 'spent' event_name_for_spend(args) when 'unassign' event_name_for_unassign(args) diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb new file mode 100644 index 00000000000..1f72bf4ce26 --- /dev/null +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + class UsageDataNonSqlMetrics < UsageData + SQL_METRIC_DEFAULT = -3 + + class << self + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def sum(relation, column, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def histogram(relation, column, buckets:, bucket_size: buckets.size) + SQL_METRIC_DEFAULT + end + + def maximum_id(model) + end + + def minimum_id(model) + end + end + end +end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c00e7a2aa13..c0dfae88fc7 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,11 +5,11 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self - def count(relation, column = nil, *rest) + def count(relation, column = nil, *args, **kwargs) raw_sql(relation, column) end - def distinct_count(relation, column = nil, *rest) + def distinct_count(relation, column = nil, *args, **kwargs) raw_sql(relation, column, :distinct) end @@ -21,14 +21,14 @@ module Gitlab end end - def sum(relation, column, *rest) + def sum(relation, column, *args, **kwargs) relation.select(relation.all.table[column].sum).to_sql end # For estimated distinct count use exact query instead of hll # buckets query, because it can't be used to obtain estimations without # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter - def estimate_batch_distinct_count(relation, column = nil, *rest) + def estimate_batch_distinct_count(relation, column = nil, *args, **kwargs) raw_sql(relation, column, :distinct) end @@ -36,6 +36,12 @@ module Gitlab 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') end + def maximum_id(model) + end + + def minimum_id(model) + end + private def raw_sql(relation, column, distinct = nil) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 29f02a5912a..c1a57566640 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -99,6 +99,8 @@ module Gitlab end def to_boolean(value, default: nil) + value = value.to_s if [0, 1].include?(value) + return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i return false if value =~ /^(false|f|no|n|0|off)$/i diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 854fc5c917d..efa2f7a943f 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -36,6 +36,7 @@ module Gitlab module Utils module UsageData + include Gitlab::Utils::StrongMemoize extend self FALLBACK = -1 @@ -209,6 +210,20 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end + def maximum_id(model) + key = :"#{model.name.downcase}_maximum_id" + strong_memoize(key) do + model.maximum(:id) + end + end + + def minimum_id(model) + key = :"#{model.name.downcase}_minimum_id" + strong_memoize(key) do + model.minimum(:id) + end + end + private def prometheus_client(verify:) diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index 80caf2c6788..016c25eb94b 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -9,9 +9,9 @@ module Gitlab production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze - UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze + UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN".freeze + PACK_PATTERN = "NnnnnN" class << self def v5(name, namespace_id: default_namespace_id) diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb index 403e308d45b..ec07023461f 100644 --- a/lib/gitlab/web_ide/config/entry/terminal.rb +++ b/lib/gitlab/web_ide/config/entry/terminal.rb @@ -10,6 +10,7 @@ module Gitlab class Terminal < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable + include Gitlab::Utils::StrongMemoize # By default the build will finish in a few seconds, not giving the webide # enough time to connect to the terminal. This default script provides @@ -51,21 +52,26 @@ module Gitlab private def to_hash - { tag_list: tags || [], - yaml_variables: yaml_variables, + { + tag_list: tags || [], + yaml_variables: yaml_variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: yaml_variables, options: { image: image_value, services: services_value, before_script: before_script_value, script: script_value || DEFAULT_SCRIPT - }.compact } + }.compact + }.compact end def yaml_variables - return unless variables_value + strong_memoize(:yaml_variables) do + next unless variables_value - variables_value.map do |key, value| - { key: key.to_s, value: value, public: true } + variables_value.map do |key, value| + { key: key.to_s, value: value, public: true } + end end end end diff --git a/lib/gitlab/word_diff/chunk_collection.rb b/lib/gitlab/word_diff/chunk_collection.rb index dd388f75302..d5c3e59d405 100644 --- a/lib/gitlab/word_diff/chunk_collection.rb +++ b/lib/gitlab/word_diff/chunk_collection.rb @@ -18,6 +18,27 @@ module Gitlab def reset @chunks = [] end + + def marker_ranges + start = 0 + + @chunks.each_with_object([]) do |element, ranges| + mode = mode_for_element(element) + + ranges << Gitlab::MarkerRange.new(start, start + element.length - 1, mode: mode) if mode + + start += element.length + end + end + + private + + def mode_for_element(element) + return Gitlab::MarkerRange::DELETION if element.removed? + return Gitlab::MarkerRange::ADDITION if element.added? + + nil + end end end end diff --git a/lib/gitlab/word_diff/parser.rb b/lib/gitlab/word_diff/parser.rb index 3b6d4d4d384..e611abb5692 100644 --- a/lib/gitlab/word_diff/parser.rb +++ b/lib/gitlab/word_diff/parser.rb @@ -31,7 +31,7 @@ module Gitlab @chunks.add(segment) when Segments::Newline - yielder << build_line(@chunks.content, nil, parent_file: diff_file) + yielder << build_line(@chunks.content, nil, parent_file: diff_file).tap { |line| line.set_marker_ranges(@chunks.marker_ranges) } @chunks.reset counter.increase_pos_num diff --git a/lib/learn_gitlab.rb b/lib/learn_gitlab.rb index 771083193d1..abceb80bd30 100644 --- a/lib/learn_gitlab.rb +++ b/lib/learn_gitlab.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true class LearnGitlab - PROJECT_NAME = 'Learn GitLab'.freeze - BOARD_NAME = 'GitLab onboarding'.freeze - LABEL_NAME = 'Novice'.freeze + PROJECT_NAME = 'Learn GitLab' + BOARD_NAME = 'GitLab onboarding' + LABEL_NAME = 'Novice' def initialize(current_user) @current_user = current_user diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 523e673e9e1..4040bed50a9 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -17,22 +17,32 @@ module Peek } }.freeze - def results - super.merge(calls: detailed_calls) - end - def self.thresholds @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS) end + def results + super.merge(summary: summary) + end + private - def detailed_calls - "#{calls} (#{cached_calls} cached)" + def summary + detail_store.each_with_object({}) do |item, count| + count_summary(item, count) + end end - def cached_calls - detail_store.count { |item| item[:cached] == 'cached' } + def count_summary(item, count) + if item[:cached].present? + count[item[:cached]] ||= 0 + count[item[:cached]] += 1 + end + + if item[:transaction].present? + count[item[:transaction]] ||= 0 + count[item[:transaction]] += 1 + end end def setup_subscribers @@ -45,10 +55,12 @@ module Peek def generate_detail(start, finish, data) { + start: start, duration: finish - start, sql: data[:sql].strip, backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller), - cached: data[:cached] ? 'cached' : '' + cached: data[:cached] ? 'Cached' : '', + transaction: data[:connection].transaction_open? ? 'In a transaction' : '' } end end diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb deleted file mode 100644 index ad9de067375..00000000000 --- a/lib/quality/test_level.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true - -module Quality - class TestLevel - UnknownTestLevelError = Class.new(StandardError) - - TEST_LEVEL_FOLDERS = { - migration: %w[ - migrations - ], - background_migration: %w[ - lib/gitlab/background_migration - lib/ee/gitlab/background_migration - ], - frontend_fixture: %w[ - frontend/fixtures - ], - unit: %w[ - bin - channels - config - db - dependencies - elastic - elastic_integration - experiments - factories - finders - frontend - graphql - haml_lint - helpers - initializers - javascripts - lib - models - policies - presenters - rack_servers - replicators - routing - rubocop - serializers - services - sidekiq - spam - support_specs - tasks - uploaders - validators - views - workers - tooling - ], - integration: %w[ - controllers - mailers - requests - ], - system: ['features'] - }.freeze - - attr_reader :prefix - - def initialize(prefix = nil) - @prefix = prefix - @patterns = {} - @regexps = {} - end - - def pattern(level) - @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}" - end - - def regexp(level) - @regexps[level] ||= Regexp.new("#{prefix}spec/#{folders_regex(level)}").freeze - end - - def level_for(file_path) - case file_path - # Detect migration first since some background migration tests are under - # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default - when regexp(:migration), regexp(:background_migration) - :migration - # Detect frontend fixture before matching other unit tests - when regexp(:frontend_fixture) - :frontend_fixture - when regexp(:unit) - :unit - when regexp(:integration) - :integration - when regexp(:system) - :system - else - raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}." - end - end - - def background_migration?(file_path) - !!(file_path =~ regexp(:background_migration)) - end - - private - - def suffix(level) - case level - when :frontend_fixture - ".rb" - else - "_spec.rb" - end - end - - def migration_and_background_migration_folders - TEST_LEVEL_FOLDERS.fetch(:migration) + TEST_LEVEL_FOLDERS.fetch(:background_migration) - end - - def folders_pattern(level) - case level - when :migration - "{#{migration_and_background_migration_folders.join(',')}}" - # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally - when :all, :geo - '**' - else - "{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}" - end - end - - def folders_regex(level) - case level - when :migration - "(#{migration_and_background_migration_folders.join('|')})" - # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally - when :all, :geo - '' - else - "(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})" - end - end - end -end diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 8f18d6433e0..e0e9677fac7 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -7,10 +7,11 @@ module Rouge # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # - # [+tag+] The tag (language) of the lexer used to generate the formatted tokens + # [+tag+] The tag (language) of the lexer used to generate the formatted tokens + # [+line_number+] The line number used to populate line IDs def initialize(options = {}) - @line_number = 1 @tag = options[:tag] + @line_number = options[:line_number] || 1 end def stream(tokens) diff --git a/lib/spam/concerns/has_spam_action_response_fields.rb b/lib/spam/concerns/has_spam_action_response_fields.rb index d49f5cd6454..6688ae56cb0 100644 --- a/lib/spam/concerns/has_spam_action_response_fields.rb +++ b/lib/spam/concerns/has_spam_action_response_fields.rb @@ -23,15 +23,6 @@ module Spam captcha_site_key: Gitlab::CurrentSettings.recaptcha_site_key } end - - # with_spam_action_response_fields(spammable) { {other_fields: true} } -> hash - # - # Takes a Spammable and a block as arguments. - # - # The block passed should be a hash, which the spam_action_fields will be merged into. - def with_spam_action_response_fields(spammable) - yield.merge(spam_action_response_fields(spammable)) - end end end end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 9eafa5ef008..74eb8634d58 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -7,8 +7,8 @@ ## CONTRIBUTING ## ################################## ## -## If you change this file in a Merge Request, please also create -## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +## If you change this file in a merge request, please also create +## a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests ## ################################### ## configuration ## diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index ae5c88455e4..576c13d8d10 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -11,8 +11,8 @@ ## CONTRIBUTING ## ################################## ## -## If you change this file in a Merge Request, please also create -## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +## If you change this file in a merge request, please also create +## a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests ## ################################### ## configuration ## diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 0f4fbe4fba5..31456dc096b 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -7,7 +7,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.parse('2.29.0') + @required_version ||= Gitlab::VersionInfo.parse('2.31.0') end def self.current_version diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb index a205861b9a9..e72d8b6b04d 100644 --- a/lib/system_check/app/redis_version_check.rb +++ b/lib/system_check/app/redis_version_check.rb @@ -5,8 +5,10 @@ require 'redis' module SystemCheck module App class RedisVersionCheck < SystemCheck::BaseCheck + # Redis 4.x will be deprecated + # https://gitlab.com/gitlab-org/gitlab/-/issues/327197 MIN_REDIS_VERSION = '4.0.0' - RECOMMENDED_REDIS_VERSION = '4.0.0' # In future we may deprecate but still support Redis 4 + RECOMMENDED_REDIS_VERSION = '5.0.0' set_name "Redis version >= #{RECOMMENDED_REDIS_VERSION}?" @custom_error_message = '' diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake deleted file mode 100644 index 44d2071751f..00000000000 --- a/lib/tasks/brakeman.rake +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -desc 'Security check via brakeman' -task :brakeman do - # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge - # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) - puts 'Security check succeed' - else - puts 'Security check failed' - exit 1 - end -end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 4d698e56444..13365b9ec07 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -3,7 +3,7 @@ namespace :cache do namespace :clear do REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan + REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan desc "GitLab | Cache | Clear redis cache" task redis: :environment do diff --git a/lib/tasks/db_obsolete_ignored_columns.rake b/lib/tasks/db_obsolete_ignored_columns.rake index cf35a355ce9..a689a9bf2d8 100644 --- a/lib/tasks/db_obsolete_ignored_columns.rake +++ b/lib/tasks/db_obsolete_ignored_columns.rake @@ -20,7 +20,7 @@ task 'db:obsolete_ignored_columns' => :environment do WARNING: Removing columns is tricky because running GitLab processes may still be using the columns. - See also https://docs.gitlab.com/ee/development/what_requires_downtime.html#dropping-columns + See also https://docs.gitlab.com/ee/development/avoiding_downtime_in_migrations.html#dropping-columns TEXT end end diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake deleted file mode 100644 index 3428e3f8f53..00000000000 --- a/lib/tasks/downtime_check.rake +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -desc 'Checks if migrations in a branch require downtime' -task downtime_check: :environment do - repo = if defined?(Gitlab::License) - 'gitlab' - else - 'gitlab-foss' - end - - `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1` - - Rake::Task['gitlab:db:downtime_check'].invoke('FETCH_HEAD') -end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 541a4fc62af..3baf4e7b7c6 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -80,22 +80,6 @@ namespace :gitlab do end end - desc 'GitLab | DB | Checks if migrations require downtime or not' - task :downtime_check, [:ref] => :environment do |_, args| - abort 'You must specify a Git reference to compare with' unless args[:ref] - - require 'shellwords' - - ref = Shellwords.escape(args[:ref]) - - migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines - .map { |file| Rails.root.join(file.strip).to_s } - .select { |file| File.file?(file) } - .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) } - - Gitlab::DowntimeCheck.new.check_and_print(migrations) - end - desc 'GitLab | DB | Sets up EE specific database functionality' if Gitlab.ee? @@ -237,7 +221,8 @@ namespace :gitlab do result_file = args[:result_file] || raise("Please specify result_file argument") raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file) - verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, true + verbose_was = ActiveRecord::Migration.verbose + ActiveRecord::Migration.verbose = true ctx = ActiveRecord::Base.connection.migration_context existing_versions = ctx.get_all_versions.to_set diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake new file mode 100644 index 00000000000..0c8e0755348 --- /dev/null +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -0,0 +1,57 @@ +# frozen_string_literal: true +require 'date' +require 'pathname' + +# https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page +namespace :gitlab do + namespace :docs do + desc 'GitLab | Docs | Create a doc redirect' + task :redirect, [:old_path, :new_path] do |_, args| + if args.old_path + old_path = args.old_path + else + puts '=> Enter the path of the OLD file:' + old_path = STDIN.gets.chomp + end + + if args.new_path + new_path = args.new_path + else + puts '=> Enter the path of the NEW file:' + new_path = STDIN.gets.chomp + end + + # + # If the new path is a relative URL, find the relative path between + # the old and new paths. + # The returned path is one level deeper, so remove the leading '../'. + # + unless new_path.start_with?('http') + old_pathname = Pathname.new(old_path) + new_pathname = Pathname.new(new_path) + relative_path = new_pathname.relative_path_from(old_pathname).to_s + (_, *last) = relative_path.split('/') + new_path = last.join('/') + end + + # + # - If this is an external URL, move the date 1 year later. + # - If this is a relative URL, move the date 3 months later. + # + date = Time.now.utc.strftime('%Y-%m-%d') + date = new_path.start_with?('http') ? Date.parse(date) >> 12 : Date.parse(date) >> 3 + + puts "=> Creating new redirect from #{old_path} to #{new_path}" + File.open(old_path, 'w') do |post| + post.puts '---' + post.puts "redirect_to: '#{new_path}'" + post.puts '---' + post.puts + post.puts "This file was moved to [another location](#{new_path})." + post.puts + post.puts "<!-- This redirect file can be deleted after <#{date}>. -->" + post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->" + end + end + end +end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 9e474b00ba7..df75b3cf716 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -17,24 +17,29 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1]) - command = [] - _, status = Gitlab::Popen.popen(%w[which gmake]) - command << (status == 0 ? 'gmake' : 'make') - - if Rails.env.test? - command.push( - 'BUNDLE_FLAGS=--no-deployment', - "GEM_HOME=#{Bundler.bundle_path}") - end - storage_paths = { 'default' => args.storage_path } Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths) + + # In CI we run scripts/gitaly-test-build + next if ENV['CI'].present? + Dir.chdir(args.dir) do - # In CI we run scripts/gitaly-test-build instead of this command - unless ENV['CI'].present? - Bundler.with_original_env { Gitlab::Popen.popen(command, nil, { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }) } + Bundler.with_original_env do + env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil } + + if Rails.env.test? + env["GEM_HOME"] = Bundler.bundle_path.to_s + env["BUNDLE_DEPLOYMENT"] = 'false' + end + + Gitlab::Popen.popen([make_cmd], nil, env) end end end + + def make_cmd + _, status = Gitlab::Popen.popen(%w[which gmake]) + status == 0 ? 'gmake' : 'make' + end end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index 77377a7e0fd..27bba6aa307 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -110,7 +110,7 @@ namespace :gitlab do desc 'GitLab | GraphQL | Generate GraphQL docs' task compile_docs: [:environment, :enable_feature_flags] do - renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema, render_options) renderer.write @@ -119,7 +119,7 @@ namespace :gitlab do desc 'GitLab | GraphQL | Check if GraphQL docs are up to date' task check_docs: [:environment, :enable_feature_flags] do - renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema, render_options) doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md')) diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index b598dab901d..ee2931f0c4f 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -9,9 +9,9 @@ namespace :gitlab do logger.info('Starting to migrate legacy pages storage to zip deployments') result = ::Pages::MigrateFromLegacyStorageService.new(logger, - migration_threads: migration_threads, - batch_size: batch_size, - ignore_invalid_entries: ignore_invalid_entries).execute + ignore_invalid_entries: ignore_invalid_entries, + mark_projects_as_not_deployed: mark_projects_as_not_deployed) + .execute_with_threads(threads: migration_threads, batch_size: batch_size) logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.") logger.info("- The #{result[:migrated]} projects migrated successfully") @@ -51,5 +51,39 @@ namespace :gitlab do ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false') ) end + + def mark_projects_as_not_deployed + Gitlab::Utils.to_boolean( + ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false') + ) + end + + namespace :deployments do + task migrate_to_object_storage: :gitlab_environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of pages deployments to remote storage') + + helper = Gitlab::Pages::MigrationHelper.new(logger) + + begin + helper.migrate_to_remote_storage + rescue => e + logger.error(e.message) + end + end + + task migrate_to_local: :gitlab_environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of Pages deployments to local storage') + + helper = Gitlab::Pages::MigrationHelper.new(logger) + + begin + helper.migrate_to_local_storage + rescue => e + logger.error(e.message) + end + end + end end end diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake deleted file mode 100644 index a83ba69bc75..00000000000 --- a/lib/tasks/gitlab/test.rake +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - desc "GitLab | Run all tests" - task :test do - cmds = [ - %w(rake brakeman), - %w(rake rubocop), - %w(rake spec), - %w(rake karma) - ] - - cmds.each do |cmd| - system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") - end - end -end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index bf18332a8eb..5eed5d4dce4 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -2,30 +2,44 @@ return if Rails.env.production? +require_relative '../../tooling/merge_request_rspec_failure_rake_task' + namespace :spec do desc 'GitLab | RSpec | Run unit tests' RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args| - require_dependency 'quality/test_level' + require_test_level t.pattern = Quality::TestLevel.new.pattern(:unit) t.rspec_opts = args[:rspec_opts] end desc 'GitLab | RSpec | Run integration tests' RSpec::Core::RakeTask.new(:integration, :rspec_opts) do |t, args| - require_dependency 'quality/test_level' + require_test_level t.pattern = Quality::TestLevel.new.pattern(:integration) t.rspec_opts = args[:rspec_opts] end desc 'GitLab | RSpec | Run system tests' RSpec::Core::RakeTask.new(:system, :rspec_opts) do |t, args| - require_dependency 'quality/test_level' + require_test_level t.pattern = Quality::TestLevel.new.pattern(:system) t.rspec_opts = args[:rspec_opts] end + desc 'GitLab | RSpec | Run merge request RSpec failures' + Tooling::MergeRequestRspecFailureRakeTask.new(:merge_request_rspec_failure, :rspec_opts) do |t, args| + t.pattern = t.rspec_failures_on_merge_request + t.rspec_opts = args[:rspec_opts] + end + desc 'Run the code examples in spec/requests/api' RSpec::Core::RakeTask.new(:api) do |t| t.pattern = 'spec/requests/api/**/*_spec.rb' end + + private + + def require_test_level + require_relative '../../tooling/quality/test_level' + end end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index b24817468c6..c4eb9450b31 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -2,7 +2,16 @@ Rake::Task["test"].clear -desc "GitLab | Run all tests" +desc "GitLab | List rake tasks for tests" task :test do - Rake::Task["gitlab:test"].invoke + puts "Running the full GitLab test suite takes significant time to pass. We recommend using one of the following spec tasks:\n\n" + + spec_tasks = Rake::Task.tasks.select { |t| t.name.start_with?('spec:') } + longest_task_name = spec_tasks.map { |t| t.name.size }.max + + spec_tasks.each do |task| + puts "#{"%-#{longest_task_name}s" % task.name} | #{task.full_comment}" + end + + puts "\nLearn more at https://docs.gitlab.com/ee/development/rake_tasks.html#run-tests." end |