diff options
Diffstat (limited to 'lib')
296 files changed, 4313 insertions, 2090 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 0d74bc841b1..8cafde4fedb 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -131,7 +131,7 @@ module API # This is a specific exception raised by `rack-timeout` gem when Puma # requests surpass its timeout. Given it inherits from Exception, we # should rescue it separately. For more info, see: - # - https://github.com/sharpstone/rack-timeout/blob/master/doc/exceptions.md + # - https://github.com/zombocom/rack-timeout/blob/master/doc/exceptions.md # - https://github.com/ruby-grape/grape#exception-handling rescue_from Rack::Timeout::RequestTimeoutException do |exception| handle_api_exception(exception) @@ -229,6 +229,7 @@ module API mount ::API::ImportGithub mount ::API::Integrations mount ::API::Integrations::JiraConnect::Subscriptions + mount ::API::Integrations::Slack::Events mount ::API::Invitations mount ::API::IssueLinks mount ::API::Issues @@ -314,6 +315,7 @@ module API mount ::API::Internal::Kubernetes mount ::API::Internal::MailRoom mount ::API::Internal::ContainerRegistry::Migration + mount ::API::Internal::Workhorse version 'v3', using: :path do # Although the following endpoints are kept behind V3 namespace, diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index 1eaa4167a7d..e599abf4aaf 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -5,6 +5,7 @@ module API before { authenticated_as_admin! } feature_category :navigation + urgency :low helpers do def current_appearance diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb index bd9fb37e18b..0fb7a4cd435 100644 --- a/lib/api/avatar.rb +++ b/lib/api/avatar.rb @@ -3,7 +3,7 @@ module API class Avatar < ::API::Base feature_category :users - urgency :high + urgency :medium resource :avatar do desc 'Return avatar url for a user' do diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 68095fb2975..f969eec8431 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -32,7 +32,7 @@ module API params do use :pagination end - get ":id/badges", urgency: :default do + get ":id/badges", urgency: :low do source = find_source(source_type, params[:id]) badges = source.badges @@ -91,7 +91,7 @@ module API requires :image_url, type: String, desc: 'URL of the badge image' optional :name, type: String, desc: 'Name for the badge' end - post ":id/badges", urgency: :default do + post ":id/badges" do source = find_source_if_admin(source_type) badge = ::Badges::CreateService.new(declared_params(include_missing: false)).execute(source) diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index e081265b418..b5d68ca5de2 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -5,6 +5,7 @@ module API include PaginationParams feature_category :navigation + urgency :low resource :broadcast_messages do helpers do diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index 766e05eca23..b1cb84c97cb 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -47,7 +47,7 @@ module API requires :source_type, type: String, desc: 'Source entity type (only `group_entity` is supported)', values: %w[group_entity] requires :source_full_path, type: String, desc: 'Source full path of the entity to import' - requires :destination_name, type: String, desc: 'Destination name for the entity' + requires :destination_name, type: String, desc: 'Destination slug for the entity' requires :destination_namespace, type: String, desc: 'Destination namespace for the entity' end end diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb index 0800993602b..8b332f96be0 100644 --- a/lib/api/ci/job_artifacts.rb +++ b/lib/api/ci/job_artifacts.rb @@ -3,6 +3,8 @@ module API module Ci class JobArtifacts < ::API::Base + helpers ::API::Helpers::ProjectStatsRefreshConflictsHelpers + before { authenticate_non_get! } feature_category :build_artifacts @@ -35,7 +37,7 @@ module API latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) authorize_read_job_artifacts!(latest_build) - present_carrierwave_file!(latest_build.artifacts_file) + present_artifacts_file!(latest_build.artifacts_file) end desc 'Download a specific file from artifacts archive from a ref' do @@ -76,7 +78,7 @@ module API build = find_build!(params[:job_id]) authorize_read_job_artifacts!(build) - present_carrierwave_file!(build.artifacts_file) + present_artifacts_file!(build.artifacts_file) end desc 'Download a specific file from artifacts archive' do @@ -137,6 +139,8 @@ module API build = find_build!(params[:job_id]) authorize!(:destroy_artifacts, build) + reject_if_build_artifacts_size_refreshing!(build.project) + build.erase_erasable_artifacts! status :no_content @@ -146,6 +150,8 @@ module API delete ':id/artifacts' do authorize_destroy_artifacts! + reject_if_build_artifacts_size_refreshing!(user_project) + ::Ci::JobArtifacts::DeleteProjectArtifactsService.new(project: user_project).execute accepted! diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb index 04999b5fb44..97471d3c96e 100644 --- a/lib/api/ci/jobs.rb +++ b/lib/api/ci/jobs.rb @@ -4,6 +4,9 @@ module API module Ci class Jobs < ::API::Base include PaginationParams + + helpers ::API::Helpers::ProjectStatsRefreshConflictsHelpers + before { authenticate! } resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -137,6 +140,8 @@ module API authorize!(:erase_build, build) break forbidden!('Job is not erasable!') unless build.erasable? + reject_if_build_artifacts_size_refreshing!(build.project) + build.erase(erased_by: current_user) present build, with: Entities::Ci::Job end @@ -204,7 +209,7 @@ module API .select { |_role, role_access_level| role_access_level <= user_access_level } .map(&:first) - environment = if environment_slug = current_authenticated_job.deployment&.environment&.slug + environment = if environment_slug = current_authenticated_job.persisted_environment&.slug { slug: environment_slug } end diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 4253a9eb4d7..cd686a28dd2 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -5,6 +5,8 @@ module API class Pipelines < ::API::Base include PaginationParams + helpers ::API::Helpers::ProjectStatsRefreshConflictsHelpers + before { authenticate_non_get! } params do @@ -208,6 +210,8 @@ module API delete ':id/pipelines/:pipeline_id', urgency: :low, feature_category: :continuous_integration do authorize! :destroy_pipeline, pipeline + reject_if_build_artifacts_size_refreshing!(pipeline.project) + destroy_conditionally!(pipeline) do ::Ci::DestroyPipelineService.new(user_project, current_user).execute(pipeline) end diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 4381309fb9e..65dc002e67d 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -330,7 +330,7 @@ module API authenticate_job!(require_running: false) end - present_carrierwave_file!(current_job.artifacts_file, supports_direct_download: params[:direct_download]) + present_artifacts_file!(current_job.artifacts_file, supports_direct_download: params[:direct_download]) end end end diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb index 6c7f502b428..c1f47dd67ce 100644 --- a/lib/api/ci/secure_files.rb +++ b/lib/api/ci/secure_files.rb @@ -26,7 +26,7 @@ module API end route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true get ':id/secure_files' do - secure_files = user_project.secure_files + secure_files = user_project.secure_files.order_by_created_at present paginate(secure_files), with: Entities::Ci::SecureFile end diff --git a/lib/api/clusters/agent_tokens.rb b/lib/api/clusters/agent_tokens.rb index 1e52790f26b..1f9c8700d7a 100644 --- a/lib/api/clusters/agent_tokens.rb +++ b/lib/api/clusters/agent_tokens.rb @@ -26,9 +26,7 @@ module API use :pagination end get do - authorize! :read_cluster, user_project - - agent = user_project.cluster_agents.find(params[:agent_id]) + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) present paginate(agent.agent_tokens), with: Entities::Clusters::AgentTokenBasic end @@ -41,9 +39,8 @@ module API requires :token_id, type: Integer, desc: 'The ID of the agent token' end get ':token_id' do - authorize! :read_cluster, user_project + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) - agent = user_project.cluster_agents.find(params[:agent_id]) token = agent.agent_tokens.find(params[:token_id]) present token, with: Entities::Clusters::AgentToken @@ -62,7 +59,7 @@ module API token_params = declared_params(include_missing: false) - agent = user_project.cluster_agents.find(params[:agent_id]) + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) result = ::Clusters::AgentTokens::CreateService.new( container: agent.project, current_user: current_user, params: token_params.merge(agent_id: agent.id) @@ -82,7 +79,8 @@ module API delete ':token_id' do authorize! :admin_cluster, user_project - agent = user_project.cluster_agents.find(params[:agent_id]) + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) + token = agent.agent_tokens.find(params[:token_id]) # Skipping explicit error handling and relying on exceptions diff --git a/lib/api/clusters/agents.rb b/lib/api/clusters/agents.rb index 0fa556d2da9..2affd9680b6 100644 --- a/lib/api/clusters/agents.rb +++ b/lib/api/clusters/agents.rb @@ -22,7 +22,7 @@ module API use :pagination end get ':id/cluster_agents' do - authorize! :read_cluster, user_project + not_found!('ClusterAgents') unless can?(current_user, :read_cluster, user_project) agents = ::Clusters::AgentsFinder.new(user_project, current_user).execute @@ -37,9 +37,7 @@ module API requires :agent_id, type: Integer, desc: 'The ID of an agent' end get ':id/cluster_agents/:agent_id' do - authorize! :read_cluster, user_project - - agent = user_project.cluster_agents.find(params[:agent_id]) + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) present agent, with: Entities::Clusters::Agent end @@ -72,7 +70,7 @@ module API delete ':id/cluster_agents/:agent_id' do authorize! :admin_cluster, user_project - agent = user_project.cluster_agents.find(params.delete(:agent_id)) + agent = ::Clusters::AgentsFinder.new(user_project, current_user).find(params[:agent_id]) destroy_conditionally!(agent) end diff --git a/lib/api/entities/ci/job_request/image.rb b/lib/api/entities/ci/job_request/image.rb index 8e404a8fa02..83f64da6050 100644 --- a/lib/api/entities/ci/job_request/image.rb +++ b/lib/api/entities/ci/job_request/image.rb @@ -7,6 +7,8 @@ module API class Image < Grape::Entity expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port + + expose :pull_policy, if: ->(_) { ::Feature.enabled?(:ci_docker_image_pull_policy) } end end end diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb index 0dae5d5a933..d9da2c92ec7 100644 --- a/lib/api/entities/ci/job_request/service.rb +++ b/lib/api/entities/ci/job_request/service.rb @@ -4,7 +4,10 @@ module API module Entities module Ci module JobRequest - class Service < Entities::Ci::JobRequest::Image + class Service < Grape::Entity + expose :name, :entrypoint + expose :ports, using: Entities::Ci::JobRequest::Port + expose :alias, :command expose :variables end diff --git a/lib/api/entities/hook.rb b/lib/api/entities/hook.rb index ac813bcac3f..d176e76b321 100644 --- a/lib/api/entities/hook.rb +++ b/lib/api/entities/hook.rb @@ -5,6 +5,9 @@ module API class Hook < Grape::Entity expose :id, :url, :created_at, :push_events, :tag_push_events, :merge_requests_events, :repository_update_events expose :enable_ssl_verification + + expose :alert_status + expose :disabled_until end end end diff --git a/lib/api/entities/issue.rb b/lib/api/entities/issue.rb index f87ef093cd8..1060b2c517a 100644 --- a/lib/api/entities/issue.rb +++ b/lib/api/entities/issue.rb @@ -29,6 +29,16 @@ module API expose :project do |issue| expose_url(api_v4_projects_path(id: issue.project_id)) end + + expose :closed_as_duplicate_of do |issue| + if ::Feature.enabled?(:closed_as_duplicate_of_issues_api, issue.project) && + issue.duplicated? && + options[:current_user]&.can?(:read_issue, issue.duplicated_to) + expose_url( + api_v4_project_issue_path(id: issue.duplicated_to.project_id, issue_iid: issue.duplicated_to.iid) + ) + end + end end expose :references, with: IssuableReferences do |issue| diff --git a/lib/api/entities/personal_access_token_with_details.rb b/lib/api/entities/personal_access_token_with_details.rb new file mode 100644 index 00000000000..5654bd4a1e1 --- /dev/null +++ b/lib/api/entities/personal_access_token_with_details.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module API + module Entities + class PersonalAccessTokenWithDetails < Entities::PersonalAccessToken + expose :expired?, as: :expired + expose :expires_soon?, as: :expires_soon + expose :revoke_path do |token| + Gitlab::Routing.url_helpers.revoke_profile_personal_access_token_path(token) + end + end + end +end diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb index c1d83a8924f..5157645af69 100644 --- a/lib/api/entities/releases/link.rb +++ b/lib/api/entities/releases/link.rb @@ -7,16 +7,11 @@ module API expose :id expose :name expose :url - expose :direct_asset_url + expose :direct_asset_url do |link| + ::Releases::LinkPresenter.new(link).direct_asset_url + end expose :external?, as: :external expose :link_type - - def direct_asset_url - return object.url unless object.filepath - - release = object.release.present - release.download_url(object.filepath) - end end end end diff --git a/lib/api/entities/wiki_page.rb b/lib/api/entities/wiki_page.rb index 43af6a336d2..5bba4271396 100644 --- a/lib/api/entities/wiki_page.rb +++ b/lib/api/entities/wiki_page.rb @@ -6,7 +6,15 @@ module API include ::MarkupHelper expose :content do |wiki_page, options| - options[:render_html] ? render_wiki_content(wiki_page, ref: wiki_page.version.id) : wiki_page.content + if options[:render_html] + render_wiki_content( + wiki_page, + ref: wiki_page.version.id, + current_user: options[:current_user] + ) + else + wiki_page.content + end end expose :encoding do |wiki_page| diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 11f1cab0c72..c4b67f83941 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -22,16 +22,15 @@ module API use :pagination optional :name, type: String, desc: 'Returns the environment with this name' optional :search, type: String, desc: 'Returns list of environments matching the search criteria' + optional :states, type: String, values: Environment.valid_states.map(&:to_s), desc: 'List all environments that match a specific state' mutually_exclusive :name, :search, message: 'cannot be used together' end get ':id/environments' do authorize! :read_environment, user_project - environments = ::Environments::EnvironmentsFinder.new(user_project, current_user, params).execute + environments = ::Environments::EnvironmentsFinder.new(user_project, current_user, declared_params(include_missing: false)).execute present paginate(environments), with: Entities::Environment, current_user: current_user - rescue ::Environments::EnvironmentsFinder::InvalidStatesError => exception - bad_request!(exception.message) end desc 'Creates a new environment' do @@ -129,14 +128,14 @@ module API end params do requires :environment_id, type: Integer, desc: 'The environment ID' + optional :force, type: Boolean, default: false end post ':id/environments/:environment_id/stop' do authorize! :read_environment, user_project environment = user_project.environments.find(params[:environment_id]) - authorize! :stop_environment, environment - - environment.stop_with_actions!(current_user) + ::Environments::StopService.new(user_project, current_user, declared_params(include_missing: false)) + .execute(environment) status 200 present environment, with: Entities::Environment, current_user: current_user diff --git a/lib/api/error_tracking/client_keys.rb b/lib/api/error_tracking/client_keys.rb index d92cf220433..c1c378111a7 100644 --- a/lib/api/error_tracking/client_keys.rb +++ b/lib/api/error_tracking/client_keys.rb @@ -5,6 +5,7 @@ module API before { authenticate! } feature_category :error_tracking + urgency :low params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb index 29b213eaffb..eea0fd2bce9 100644 --- a/lib/api/error_tracking/collector.rb +++ b/lib/api/error_tracking/collector.rb @@ -6,6 +6,7 @@ module API # sentry backend. For more details see https://gitlab.com/gitlab-org/gitlab/-/issues/329596. class ErrorTracking::Collector < ::API::Base feature_category :error_tracking + urgency :low content_type :envelope, 'application/x-sentry-envelope' content_type :json, 'application/json' diff --git a/lib/api/error_tracking/project_settings.rb b/lib/api/error_tracking/project_settings.rb index 74432d1eaec..fefc2098137 100644 --- a/lib/api/error_tracking/project_settings.rb +++ b/lib/api/error_tracking/project_settings.rb @@ -5,6 +5,7 @@ module API before { authenticate! } feature_category :error_tracking + urgency :low helpers do def project_setting diff --git a/lib/api/features.rb b/lib/api/features.rb index bff2817a2ec..13a6aedc2df 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -68,10 +68,13 @@ module API requires :value, type: String, desc: '`true` or `false` to enable/disable, a float for percentage of time' optional :key, type: String, desc: '`percentage_of_actors` or the default `percentage_of_time`' optional :feature_group, type: String, desc: 'A Feature group name' - optional :user, type: String, desc: 'A GitLab username' - optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'" - optional :namespace, type: String, desc: "A GitLab group or user namespace path, such as 'gitlab-org'" - optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' + optional :user, type: String, desc: 'A GitLab username or comma-separated multiple usernames' + optional :group, type: String, + desc: "A GitLab group's path, such as 'gitlab-org', or comma-separated multiple group paths" + optional :namespace, type: String, + desc: "A GitLab group or user namespace path, such as 'john-doe', or comma-separated multiple namespace paths" + optional :project, type: String, + desc: "A projects path, such as `gitlab-org/gitlab-ce`, or comma-separated multiple project paths" optional :force, type: Boolean, desc: 'Skip feature flag validation checks, ie. YAML definition' mutually_exclusive :key, :feature_group @@ -110,6 +113,8 @@ module API present Feature.get(params[:name]), # rubocop:disable Gitlab/AvoidFeatureGet with: Entities::Feature, current_user: current_user + rescue Feature::Target::UnknowTargetError => e + bad_request!(e.message) end desc 'Remove the gate value for the given feature' diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 60bb51bf48f..c17bc432404 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -417,7 +417,7 @@ module API requires :group_access, type: Integer, values: Gitlab::Access.all_values, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end - post ":id/share", feature_category: :subgroups do + post ":id/share", feature_category: :subgroups, urgency: :low do shared_with_group = find_group!(params[:group_id]) group_link_create_params = { diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a079c591519..fc1037131d8 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -394,8 +394,7 @@ module API end def order_options_with_tie_breaker - order_by = if Feature.enabled?(:replace_order_by_created_at_with_id) && - params[:order_by] == 'created_at' + order_by = if params[:order_by] == 'created_at' 'id' else params[:order_by] @@ -409,15 +408,11 @@ module API # error helpers def forbidden!(reason = nil) - message = ['403 Forbidden'] - message << "- #{reason}" if reason - render_api_error!(message.join(' '), 403) + render_api_error_with_reason!(403, '403 Forbidden', reason) end def bad_request!(reason = nil) - message = ['400 Bad request'] - message << "- #{reason}" if reason - render_api_error!(message.join(' '), 400) + render_api_error_with_reason!(400, '400 Bad request', reason) end def bad_request_missing_attribute!(attribute) @@ -437,8 +432,8 @@ module API end end - def unauthorized! - render_api_error!('401 Unauthorized', 401) + def unauthorized!(reason = nil) + render_api_error_with_reason!(401, '401 Unauthorized', reason) end def not_allowed!(message = nil) @@ -491,6 +486,12 @@ module API model.errors.messages end + def render_api_error_with_reason!(status, message, reason) + message = [message] + message << "- #{reason}" if reason + render_api_error!(message.join(' '), status) + end + def render_api_error!(message, status) render_structured_api_error!({ 'message' => message }, status) end @@ -569,11 +570,19 @@ module API end end + def log_artifact_size(file) + Gitlab::ApplicationContext.push(artifact: file.model) + end + + def present_artifacts_file!(file, **args) + log_artifact_size(file) if file + + present_carrierwave_file!(file, **args) + end + def present_carrierwave_file!(file, supports_direct_download: true) return not_found! unless file&.exists? - log_artifact_size(file) if file.is_a?(JobArtifactUploader) - if file.file_storage? present_disk_file!(file.path, file.filename) elsif supports_direct_download && file.class.direct_download_enabled? @@ -724,7 +733,6 @@ module API # Deprecated. Use `send_artifacts_entry` instead. def legacy_send_artifacts_entry(file, entry) header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) - log_artifact_size(file) body '' end @@ -732,15 +740,10 @@ module API def send_artifacts_entry(file, entry) header(*Gitlab::Workhorse.send_artifacts_entry(file, entry)) header(*Gitlab::Workhorse.detect_content_type) - log_artifact_size(file) body '' end - def log_artifact_size(file) - Gitlab::ApplicationContext.push(artifact: file.model) - end - # The Grape Error Middleware only has access to `env` but not `params` nor # `request`. We workaround this by defining methods that returns the right # values. diff --git a/lib/api/helpers/project_stats_refresh_conflicts_helpers.rb b/lib/api/helpers/project_stats_refresh_conflicts_helpers.rb new file mode 100644 index 00000000000..db464521033 --- /dev/null +++ b/lib/api/helpers/project_stats_refresh_conflicts_helpers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Helpers + module ProjectStatsRefreshConflictsHelpers + def reject_if_build_artifacts_size_refreshing!(project) + return unless project.refreshing_build_artifacts_size? + + Gitlab::ProjectStatsRefreshConflictsLogger.warn_request_rejected_during_stats_refresh(project.id) + + conflict!('Action temporarily disabled. The project this pipeline belongs to is undergoing stats refresh.') + end + end + end +end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 7a9dd78e4ed..52cb398d6bf 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -169,7 +169,6 @@ module API :merge_commit_template, :squash_commit_template, :repository_storage, - :compliance_framework_setting, :packages_enabled, :service_desk_enabled, :keep_latest_artifact, diff --git a/lib/api/helpers/sse_helpers.rb b/lib/api/helpers/sse_helpers.rb deleted file mode 100644 index c354694f508..00000000000 --- a/lib/api/helpers/sse_helpers.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module SSEHelpers - def request_from_sse?(project) - return false if request.referer.blank? - - uri = URI.parse(request.referer) - uri.path.starts_with?(::Gitlab::Routing.url_helpers.project_root_sse_path(project)) - rescue URI::InvalidURIError - false - end - end - end -end diff --git a/lib/api/integrations/jira_connect/subscriptions.rb b/lib/api/integrations/jira_connect/subscriptions.rb index fa19dc2be3f..a6e931ba7bb 100644 --- a/lib/api/integrations/jira_connect/subscriptions.rb +++ b/lib/api/integrations/jira_connect/subscriptions.rb @@ -23,7 +23,7 @@ module API installation = JiraConnectInstallation.find_by_client_key(jwt.iss_claim) if !installation || !jwt.valid?(installation.shared_secret) || !jwt.verify_context_qsh_claim - unauthorized! + unauthorized!('JWT authentication failed') end jira_user = installation.client.user_info(jwt.sub_claim) diff --git a/lib/api/integrations/slack/events.rb b/lib/api/integrations/slack/events.rb new file mode 100644 index 00000000000..6227b75a9d7 --- /dev/null +++ b/lib/api/integrations/slack/events.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# This API endpoint handles all events sent from Slack once a Slack +# workspace has installed the GitLab Slack app. +# +# See https://api.slack.com/apis/connections/events-api. +module API + class Integrations + module Slack + class Events < ::API::Base + feature_category :integrations + + before { verify_slack_request! } + + helpers do + def verify_slack_request! + unauthorized! unless Request.verify!(request) + end + end + + namespace 'integrations/slack' do + post :events do + type = params['type'] + raise ArgumentError, "Unable to handle event type: '#{type}'" unless type == 'url_verification' + + status :ok + UrlVerification.call(params) + rescue ArgumentError => e + # Track the error, but respond with a `2xx` because we don't want to risk + # Slack rate-limiting, or disabling our app, due to error responses. + # See https://api.slack.com/apis/connections/events-api. + Gitlab::ErrorTracking.track_exception(e) + + no_content! + end + end + end + end + end +end diff --git a/lib/api/integrations/slack/events/url_verification.rb b/lib/api/integrations/slack/events/url_verification.rb new file mode 100644 index 00000000000..4628b93665d --- /dev/null +++ b/lib/api/integrations/slack/events/url_verification.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module API + class Integrations + module Slack + class Events + class UrlVerification + # When the GitLab Slack app is first configured to receive Slack events, + # Slack will issue a special request to the endpoint and expect it to respond + # with the `challenge` param. + # + # This must be done in-request, rather than on a queue. + # + # See https://api.slack.com/apis/connections/events-api. + def self.call(params) + { challenge: params[:challenge] } + end + end + end + end + end +end diff --git a/lib/api/integrations/slack/request.rb b/lib/api/integrations/slack/request.rb new file mode 100644 index 00000000000..df0109b07aa --- /dev/null +++ b/lib/api/integrations/slack/request.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module API + class Integrations + module Slack + module Request + VERIFICATION_VERSION = 'v0' + VERIFICATION_TIMESTAMP_HEADER = 'X-Slack-Request-Timestamp' + VERIFICATION_SIGNATURE_HEADER = 'X-Slack-Signature' + VERIFICATION_DELIMITER = ':' + VERIFICATION_HMAC_ALGORITHM = 'sha256' + VERIFICATION_TIMESTAMP_EXPIRY = 1.minute.to_i + + # Verify the request by comparing the given request signature in the header + # with a signature value that we compute according to the steps in: + # https://api.slack.com/authentication/verifying-requests-from-slack. + def self.verify!(request) + return false unless Gitlab::CurrentSettings.slack_app_signing_secret + + timestamp, signature = request.headers.values_at( + VERIFICATION_TIMESTAMP_HEADER, + VERIFICATION_SIGNATURE_HEADER + ) + + return false if timestamp.nil? || signature.nil? + return false if Time.current.to_i - timestamp.to_i >= VERIFICATION_TIMESTAMP_EXPIRY + + request.body.rewind + + basestring = [ + VERIFICATION_VERSION, + timestamp, + request.body.read + ].join(VERIFICATION_DELIMITER) + + hmac_digest = OpenSSL::HMAC.hexdigest( + VERIFICATION_HMAC_ALGORITHM, + Gitlab::CurrentSettings.slack_app_signing_secret, + basestring + ) + + # Signature will look like: 'v0=a2114d57b48eac39b9ad189dd8316235a7b4a8d21a10bd27519666489c69b503' + ActiveSupport::SecurityUtils.secure_compare( + signature, + "#{VERIFICATION_VERSION}=#{hmac_digest}" + ) + end + end + end + end +end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index b53f855c3a2..3edd38a0108 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -164,6 +164,18 @@ module API check_allowed(params) end + post '/error_tracking_allowed', feature_category: :error_tracking do + public_key = params[:public_key] + project_id = params[:project_id] + + unprocessable_entity! if public_key.blank? || project_id.blank? + + enabled = ::ErrorTracking::ClientKey.enabled_key_for(project_id, public_key).exists? + + status 200 + { enabled: enabled } + end + post "/lfs_authenticate", feature_category: :source_code_management, urgency: :high do not_found! unless container&.lfs_enabled? diff --git a/lib/api/internal/mail_room.rb b/lib/api/internal/mail_room.rb index 6e24cf6e7c5..1e5e8c4c4e2 100644 --- a/lib/api/internal/mail_room.rb +++ b/lib/api/internal/mail_room.rb @@ -12,6 +12,10 @@ module API class MailRoom < ::API::Base feature_category :service_desk + format :json + content_type :txt, 'text/plain' + default_format :txt + before do authenticate_gitlab_mailroom_request! end @@ -30,7 +34,7 @@ module API end post "/*mailbox_type" do worker = Gitlab::MailRoom.worker_for(params[:mailbox_type]) - raw = request.body.read + raw = Gitlab::EncodingHelper.encode_utf8(request.body.read) begin worker.perform_async(raw) rescue Gitlab::SidekiqMiddleware::SizeLimiter::ExceedLimitError diff --git a/lib/api/internal/workhorse.rb b/lib/api/internal/workhorse.rb new file mode 100644 index 00000000000..910cf52bc3b --- /dev/null +++ b/lib/api/internal/workhorse.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module API + module Internal + class Workhorse < ::API::Base + feature_category :not_owned # rubocop:todo Gitlab/AvoidFeatureCategoryNotOwned + + before do + verify_workhorse_api! + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + end + + helpers do + def request_authenticated? + authenticator = Gitlab::Auth::RequestAuthenticator.new(request) + return true if authenticator.find_authenticated_requester([:api]) + + # Look up user from warden, ignoring the absence of a CSRF token. For + # web users the CSRF token can be in the POST form data but Workhorse + # does not propagate the form data to us. + !!request.env['warden']&.authenticate + end + end + + namespace 'internal' do + namespace 'workhorse' do + post 'authorize_upload' do + unauthorized! unless request_authenticated? + + status 200 + { TempPath: File.join(::Gitlab.config.uploads.storage_path, 'uploads/tmp') } + end + end + end + end + end +end diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb index cf075af8373..c07c2c1994e 100644 --- a/lib/api/issue_links.rb +++ b/lib/api/issue_links.rb @@ -61,6 +61,22 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Get issues relation' do + detail 'This feature was introduced in GitLab 15.1.' + success Entities::IssueLink + end + params do + requires :issue_link_id, type: Integer, desc: 'The ID of an issue link' + end + get ':id/issues/:issue_iid/links/:issue_link_id' do + issue = find_project_issue(params[:issue_iid]) + issue_link = IssueLink.for_source_or_target(issue).find(declared_params[:issue_link_id]) + + find_project_issue(issue_link.target.iid.to_s, issue_link.target.project_id.to_s) + + present issue_link, with: Entities::IssueLink + end + desc 'Remove issues relation' do success Entities::IssueLink end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 730baae63a2..156a92802b0 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -9,7 +9,6 @@ module API before { authenticate_non_get! } helpers Helpers::MergeRequestsHelpers - helpers Helpers::SSEHelpers # These endpoints are defined in `TimeTrackingEndpoints` and is shared by # API::Issues. In order to be able to define the feature category of these @@ -234,8 +233,6 @@ module API handle_merge_request_errors!(merge_request) - Gitlab::UsageDataCounters::EditorUniqueCounter.track_sse_edit_action(author: current_user) if request_from_sse?(user_project) - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end @@ -458,7 +455,11 @@ module API not_allowed! if !immediately_mergeable && !automatically_mergeable - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: automatically_mergeable) + if Feature.enabled?(:change_response_code_merge_status, user_project) + render_api_error!('Branch cannot be merged', 422) unless merge_request.mergeable?(skip_ci_check: automatically_mergeable) + else + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: automatically_mergeable) + end check_sha_param!(params, merge_request) diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb index c6406bf61df..6fc90da87d4 100644 --- a/lib/api/metrics/dashboard/annotations.rb +++ b/lib/api/metrics/dashboard/annotations.rb @@ -5,6 +5,7 @@ module API module Dashboard class Annotations < ::API::Base feature_category :metrics + urgency :low desc 'Create a new monitoring dashboard annotation' do success Entities::Metrics::Dashboard::Annotation diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb index 909f7f0405d..83d95f8b062 100644 --- a/lib/api/metrics/user_starred_dashboards.rb +++ b/lib/api/metrics/user_starred_dashboards.rb @@ -4,6 +4,7 @@ module API module Metrics class UserStarredDashboards < ::API::Base feature_category :metrics + urgency :low resource :projects do desc 'Marks selected metrics dashboard as starred' do diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 4ff7096b5d9..a12fbbb9bb6 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -67,9 +67,10 @@ module API end get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS, feature_category: :subgroups, urgency: :low do namespace_path = params[:namespace] + existing_namespaces_within_the_parent = Namespace.without_project_namespaces.by_parent(params[:parent_id]) - exists = Namespace.without_project_namespaces.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists? - suggestions = exists ? [Namespace.clean_path(namespace_path)] : [] + exists = existing_namespaces_within_the_parent.filter_by_path(namespace_path).exists? + suggestions = exists ? [Namespace.clean_path(namespace_path, limited_to: existing_namespaces_within_the_parent)] : [] present :exists, exists present :suggests, suggestions diff --git a/lib/api/personal_access_tokens.rb b/lib/api/personal_access_tokens.rb index 40e6486dae9..f8b744bb14b 100644 --- a/lib/api/personal_access_tokens.rb +++ b/lib/api/personal_access_tokens.rb @@ -54,6 +54,14 @@ module API present paginate(tokens), with: Entities::PersonalAccessToken end + get ':id' do + token = PersonalAccessToken.find_by_id(params[:id]) + + unauthorized! unless token && Ability.allowed?(current_user, :read_user_personal_access_tokens, token.user) + + present token, with: Entities::PersonalAccessToken + end + delete 'self' do revoke_token(access_token) end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index 35f555e16b5..fb782b49f02 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -45,8 +45,6 @@ module API # For all projects except those in a user namespace, the `namespace` # and `group` are identical. Preload the group when it's not a user namespace. def preload_groups(projects_relation) - return unless Feature.enabled?(:group_projects_api_preload_groups) - group_projects = projects_for_group_preload(projects_relation) groups = group_projects.map(&:namespace) diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index f11270457c9..5bf3c3b8aac 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -39,6 +39,51 @@ module API params :package_name do requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' end + + def present_simple_index(group_or_project) + authorize_read_package!(group_or_project) + + packages = Packages::Pypi::PackagesFinder.new(current_user, group_or_project).execute + presenter = ::Packages::Pypi::SimpleIndexPresenter.new(packages, group_or_project) + + present_html(presenter.body) + end + + def present_simple_package(group_or_project) + authorize_read_package!(group_or_project) + track_simple_event(group_or_project, 'list_package') + + packages = Packages::Pypi::PackagesFinder.new(current_user, group_or_project, { package_name: params[:package_name] }).execute + empty_packages = packages.empty? + + redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do + not_found!('Package') if empty_packages + presenter = ::Packages::Pypi::SimplePackageVersionsPresenter.new(packages, group_or_project) + + present_html(presenter.body) + end + end + + def track_simple_event(group_or_project, event_name) + if group_or_project.is_a?(Project) + project = group_or_project + namespace = group_or_project.namespace + else + project = nil + namespace = group_or_project + end + + track_package_event(event_name, :pypi, project: project, namespace: namespace) + end + + def present_html(content) + # Adjusts grape output format + # to be HTML + content_type "text/html; charset=utf-8" + env['api.format'] = :binary + + body content + end end params do @@ -67,7 +112,18 @@ module API present_carrierwave_file!(package_file.file, supports_direct_download: true) end - desc 'The PyPi Simple Endpoint' do + desc 'The PyPi Simple Group Index Endpoint' do + detail 'This feature was introduced in GitLab 15.1' + end + + # An API entry point but returns an HTML file instead of JSON. + # PyPi simple API returns a list of packages as a simple HTML file. + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'simple', format: :txt do + present_simple_index(find_authorized_group!) + end + + desc 'The PyPi Simple Group Package Endpoint' do detail 'This feature was introduced in GitLab 12.10' end @@ -75,29 +131,11 @@ module API use :package_name end - # An Api entry point but returns an HTML file instead of JSON. + # An API entry point but returns an HTML file instead of JSON. # PyPi simple API returns the package descriptor as a simple HTML file. route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'simple/*package_name', format: :txt do - group = find_authorized_group! - authorize_read_package!(group) - - track_package_event('list_package', :pypi) - - packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute - empty_packages = packages.empty? - - redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do - not_found!('Package') if empty_packages - presenter = ::Packages::Pypi::PackagePresenter.new(packages, group) - - # Adjusts grape output format - # to be HTML - content_type "text/html; charset=utf-8" - env['api.format'] = :binary - - body presenter.body - end + present_simple_package(find_authorized_group!) end end end @@ -133,7 +171,18 @@ module API present_carrierwave_file!(package_file.file, supports_direct_download: true) end - desc 'The PyPi Simple Endpoint' do + desc 'The PyPi Simple Project Index Endpoint' do + detail 'This feature was introduced in GitLab 15.1' + end + + # An API entry point but returns an HTML file instead of JSON. + # PyPi simple API returns a list of packages as a simple HTML file. + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'simple', format: :txt do + present_simple_index(authorized_user_project) + end + + desc 'The PyPi Simple Project Package Endpoint' do detail 'This feature was introduced in GitLab 12.10' end @@ -141,28 +190,11 @@ module API use :package_name end - # An Api entry point but returns an HTML file instead of JSON. + # An API entry point but returns an HTML file instead of JSON. # PyPi simple API returns the package descriptor as a simple HTML file. route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth get 'simple/*package_name', format: :txt do - authorize_read_package!(authorized_user_project) - - track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace) - - packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute - empty_packages = packages.empty? - - redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do - not_found!('Package') if empty_packages - presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) - - # Adjusts grape output format - # to be HTML - content_type "text/html; charset=utf-8" - env['api.format'] = :binary - - body presenter.body - end + present_simple_package(authorized_user_project) end desc 'The PyPi Package upload endpoint' do diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb index bc5ffe5b21f..8b9380b332e 100644 --- a/lib/api/release/links.rb +++ b/lib/api/release/links.rb @@ -29,6 +29,7 @@ module API params do use :pagination end + route_setting :authentication, job_token_allowed: true get 'links' do authorize! :read_release, release @@ -45,6 +46,7 @@ module API optional :filepath, type: String, desc: 'The filepath of the link' optional :link_type, type: String, desc: 'The link type, one of: "runbook", "image", "package" or "other"' end + route_setting :authentication, job_token_allowed: true post 'links' do authorize! :create_release, release @@ -65,6 +67,7 @@ module API detail 'This feature was introduced in GitLab 11.7.' success Entities::Releases::Link end + route_setting :authentication, job_token_allowed: true get do authorize! :read_release, release @@ -82,6 +85,7 @@ module API optional :link_type, type: String, desc: 'The link type' at_least_one_of :name, :url end + route_setting :authentication, job_token_allowed: true put do authorize! :update_release, release @@ -96,6 +100,7 @@ module API detail 'This feature was introduced in GitLab 11.7.' success Entities::Releases::Link end + route_setting :authentication, job_token_allowed: true delete do authorize! :destroy_release, release diff --git a/lib/api/releases.rb b/lib/api/releases.rb index c69f45f1f38..aecd6f9eef8 100644 --- a/lib/api/releases.rb +++ b/lib/api/releases.rb @@ -107,9 +107,10 @@ module API end params do requires :tag_name, type: String, desc: 'The name of the tag', as: :tag + optional :tag_message, type: String, desc: 'Message to use if creating a new annotated tag' optional :name, type: String, desc: 'The name of the release' optional :description, type: String, desc: 'The release notes' - optional :ref, type: String, desc: 'The commit sha or branch name' + optional :ref, type: String, desc: 'Commit SHA or branch name to use if creating a new tag' optional :assets, type: Hash do optional :links, type: Array do requires :name, type: String, desc: 'The name of the link' diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index 8da77ba18ae..f96cffb008c 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -114,7 +114,9 @@ module API module_version: params[:module_version] ) - jwt_token = Gitlab::TerraformRegistryToken.from_token(token_from_namespace_inheritable).encoded + if token_from_namespace_inheritable + jwt_token = Gitlab::TerraformRegistryToken.from_token(token_from_namespace_inheritable).encoded + end header 'X-Terraform-Get', module_file_path.sub(%r{module_version/file$}, "#{params[:module_version]}/file?token=#{jwt_token}&archive=tgz") status :no_content diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb index 7b111451b9f..b727fbd9f65 100644 --- a/lib/api/terraform/state.rb +++ b/lib/api/terraform/state.rb @@ -13,6 +13,7 @@ module API default_format :json rescue_from( + ::Terraform::RemoteStateHandler::StateDeletedError, ::ActiveRecord::RecordNotUnique, ::PG::UniqueViolation ) do |e| @@ -24,6 +25,11 @@ module API authorize! :read_terraform_state, user_project increment_unique_values('p_terraform_state_api_unique_users', current_user.id) + + if Feature.enabled?(:route_hll_to_snowplow_phase2, user_project&.namespace) + Gitlab::Tracking.event('API::Terraform::State', 'p_terraform_state_api_unique_users', + namespace: user_project&.namespace, user: current_user) + end end params do @@ -76,7 +82,7 @@ module API authorize! :admin_terraform_state, user_project remote_state_handler.handle_with_lock do |state| - state.destroy! + ::Terraform::States::TriggerDestroyService.new(state, current_user: current_user).execute end body false diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 756901c5717..d0b1e458a27 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -3,6 +3,7 @@ module API class UserCounts < ::API::Base feature_category :navigation + urgency :low resource :user_counts do desc 'Return the user specific counts' do diff --git a/lib/api/users.rb b/lib/api/users.rb index b10458c4358..93df9413119 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -10,7 +10,7 @@ module API feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] - urgency :high, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] + urgency :medium, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do include CustomAttributesEndpoints @@ -145,7 +145,7 @@ module API use :with_custom_attributes end # rubocop: disable CodeReuse/ActiveRecord - get ":id", feature_category: :users, urgency: :medium do + get ":id", feature_category: :users, urgency: :low do forbidden!('Not authorized!') unless current_user unless current_user.admin? @@ -170,7 +170,7 @@ module API params do requires :user_id, type: String, desc: 'The ID or username of the user' end - get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users, urgency: :high do + get ":user_id/status", requirements: API::USER_REQUIREMENTS, feature_category: :users, urgency: :default do user = find_user(params[:user_id]) not_found!('User') unless user && can?(current_user, :read_user, user) @@ -346,6 +346,30 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Get the project-level Deploy keys that a specified user can access to.' do + success Entities::DeployKey + end + params do + requires :user_id, type: String, desc: 'The ID or username of the user' + use :pagination + end + get ':user_id/project_deploy_keys', requirements: API::USER_REQUIREMENTS, feature_category: :continuous_delivery do + user = find_user(params[:user_id]) + not_found!('User') unless user && can?(current_user, :read_user, user) + + project_ids = Project.visible_to_user_and_access_level(current_user, Gitlab::Access::MAINTAINER) + + unless current_user == user + # Restrict to only common projects of both current_user and user. + project_ids = project_ids.visible_to_user_and_access_level(user, Gitlab::Access::MAINTAINER) + end + + forbidden!('No common authorized project found') unless project_ids.present? + + keys = DeployKey.in_projects(project_ids) + present paginate(keys), with: Entities::DeployKey + end + desc 'Add an SSH key to a specified user. Available only for admins.' do success Entities::SSHKey end @@ -921,7 +945,7 @@ module API desc 'Get the currently authenticated user' do success Entities::UserPublic end - get feature_category: :users, urgency: :medium do + get feature_category: :users, urgency: :low do entity = if current_user.admin? Entities::UserWithAdmin @@ -1096,7 +1120,7 @@ module API requires :credit_card_mask_number, type: String, desc: 'The last 4 digits of credit card number' requires :credit_card_type, type: String, desc: 'The credit card network name' end - put ":user_id/credit_card_validation", feature_category: :purchase do + put ":user_id/credit_card_validation", urgency: :low, feature_category: :purchase do authenticated_as_admin! user = find_user(params[:user_id]) diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 12dbf4792d6..082be1f7e11 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -37,7 +37,12 @@ module API entity = params[:with_content] ? Entities::WikiPage : Entities::WikiPageBasic - present container.wiki.list_pages(load_content: params[:with_content]), with: entity + options = { + with: entity, + current_user: current_user + } + + present container.wiki.list_pages(load_content: params[:with_content]), options end desc 'Get a wiki page' do @@ -51,7 +56,13 @@ module API get ':id/wikis/:slug', urgency: :low do authorize! :read_wiki, container - present wiki_page(params[:version]), with: Entities::WikiPage, render_html: params[:render_html] + options = { + with: Entities::WikiPage, + render_html: params[:render_html], + current_user: current_user + } + + present wiki_page(params[:version]), options end desc 'Create a wiki page' do diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index b8aa2cc8ea0..3c999920d39 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -30,6 +30,8 @@ module Atlassian responses.compact end + # Fetch user information for the given account. + # https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-users/#api-rest-api-3-user-get def user_info(account_id) r = get('/rest/api/3/user', { accountId: account_id, expand: 'groups' }) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 0991177d044..16b8f21c9e9 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -11,7 +11,8 @@ module Backup LIST_ENVS = { skipped: 'SKIP', - repositories_storages: 'REPOSITORIES_STORAGES' + repositories_storages: 'REPOSITORIES_STORAGES', + repositories_paths: 'REPOSITORIES_PATHS' }.freeze TaskDefinition = Struct.new( @@ -41,29 +42,8 @@ module Backup end def create - if incremental? - unpack(ENV.fetch('PREVIOUS_BACKUP', ENV['BACKUP'])) - read_backup_information - verify_backup_version - update_backup_information - end - - build_backup_information - - definitions.keys.each do |task_name| - run_create_task(task_name) - end - - write_backup_information - - if skipped?('tar') - upload - else - pack - upload - cleanup - remove_old - end + unpack(ENV.fetch('PREVIOUS_BACKUP', ENV['BACKUP'])) if incremental? + run_all_create_tasks puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need these files to restore a backup.\n" \ @@ -95,22 +75,8 @@ module Backup end def restore - cleanup_required = unpack(ENV['BACKUP']) - read_backup_information - verify_backup_version - - definitions.keys.each do |task_name| - run_restore_task(task_name) if !skipped?(task_name) && enabled_task?(task_name) - end - - Rake::Task['gitlab:shell:setup'].invoke - Rake::Task['cache:clear'].invoke - - if cleanup_required - cleanup - end - - remove_tmp + unpack(ENV['BACKUP']) + run_all_restore_tasks puts_time "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need to restore these files manually.".color(:red) @@ -225,13 +191,59 @@ module Backup max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence strategy = Backup::GitalyBackup.new(progress, incremental: incremental?, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency) - Repositories.new(progress, strategy: strategy, storages: repositories_storages) + Repositories.new(progress, + strategy: strategy, + storages: list_env(:repositories_storages), + paths: list_env(:repositories_paths) + ) end def build_files_task(app_files_dir, excludes: []) Files.new(progress, app_files_dir, excludes: excludes) end + def run_all_create_tasks + if incremental? + read_backup_information + verify_backup_version + update_backup_information + end + + build_backup_information + + definitions.keys.each do |task_name| + run_create_task(task_name) + end + + write_backup_information + + unless skipped?('tar') + pack + upload + remove_old + end + + ensure + cleanup unless skipped?('tar') + remove_tmp + end + + def run_all_restore_tasks + read_backup_information + verify_backup_version + + definitions.keys.each do |task_name| + run_restore_task(task_name) if !skipped?(task_name) && enabled_task?(task_name) + end + + Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke + + ensure + cleanup unless skipped?('tar') + remove_tmp + end + def incremental? @incremental end @@ -259,7 +271,8 @@ module Backup tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, skipped: ENV['SKIP'], - repositories_storages: ENV['REPOSITORIES_STORAGES'] + repositories_storages: ENV['REPOSITORIES_STORAGES'], + repositories_paths: ENV['REPOSITORIES_PATHS'] } end @@ -272,7 +285,8 @@ module Backup tar_version: tar_version, installation_type: Gitlab::INSTALLATION_TYPE, skipped: list_env(:skipped).join(','), - repositories_storages: list_env(:repositories_storages).join(',') + repositories_storages: list_env(:repositories_storages).join(','), + repositories_paths: list_env(:repositories_paths).join(',') ) end @@ -299,7 +313,7 @@ module Backup def upload connection_settings = Gitlab.config.backup.upload.connection - if connection_settings.blank? || skipped?('remote') + if connection_settings.blank? || skipped?('remote') || skipped?('tar') puts_time "Uploading backup archive to remote storage #{remote_directory} ... ".color(:blue) + "[SKIPPED]".color(:cyan) return end @@ -405,8 +419,7 @@ module Backup def unpack(source_backup_id) if source_backup_id.blank? && non_tarred_backup? puts_time "Non tarred backup found in #{backup_path}, using that" - - return false + return end Dir.chdir(backup_path) do @@ -466,10 +479,6 @@ module Backup @skipped ||= list_env(:skipped) end - def repositories_storages - @repositories_storages ||= list_env(:repositories_storages) - end - def list_env(name) list = ENV.fetch(LIST_ENVS[name], '').split(',') list += backup_information[name].split(',') if backup_information[name] diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 4a31e87b969..4f4a098f374 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -3,19 +3,25 @@ require 'yaml' module Backup + # Backup and restores repositories by querying the database class Repositories < Task extend ::Gitlab::Utils::Override - def initialize(progress, strategy:, storages: []) + # @param [IO] progress IO interface to output progress + # @param [Object] :strategy Fetches backups from gitaly + # @param [Array<String>] :storages Filter by specified storage names. Empty means all storages. + # @param [Array<String>] :paths Filter by specified project paths. Empty means all projects, groups and snippets. + def initialize(progress, strategy:, storages: [], paths: []) super(progress) @strategy = strategy @storages = storages + @paths = paths end override :dump - def dump(path, backup_id) - strategy.start(:create, path, backup_id: backup_id) + def dump(destination_path, backup_id) + strategy.start(:create, destination_path, backup_id: backup_id) enqueue_consecutive ensure @@ -23,8 +29,8 @@ module Backup end override :restore - def restore(path) - strategy.start(:restore, path) + def restore(destination_path) + strategy.start(:restore, destination_path) enqueue_consecutive ensure @@ -36,7 +42,7 @@ module Backup private - attr_reader :strategy, :storages + attr_reader :strategy, :storages, :paths def enqueue_consecutive enqueue_consecutive_projects @@ -66,12 +72,26 @@ module Backup def project_relation scope = Project.includes(:route, :group, namespace: :owner) scope = scope.id_in(ProjectRepository.for_repository_storage(storages).select(:project_id)) if storages.any? + if paths.any? + scope = scope.where_full_path_in(paths).or( + Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants) + ) + end + scope end def snippet_relation scope = Snippet.all scope = scope.id_in(SnippetRepository.for_repository_storage(storages).select(:snippet_id)) if storages.any? + if paths.any? + scope = scope.joins(:project).merge( + Project.where_full_path_in(paths).or( + Project.where(namespace_id: Namespace.where_full_path_in(paths).self_and_descendants) + ) + ) + end + scope end @@ -79,7 +99,6 @@ module Backup PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." - pool.source_project ||= pool.member_projects.first&.root_of_fork_network unless pool.source_project progress.puts " - Object pool #{pool.disk_path}... " + "[SKIPPED]".color(:cyan) next diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb index 157dc696cc8..86ab8597cf5 100644 --- a/lib/banzai/filter/references/commit_reference_filter.rb +++ b/lib/banzai/filter/references/commit_reference_filter.rb @@ -19,7 +19,12 @@ module Banzai def find_object(project, id) return unless project.is_a?(Project) && project.valid_repo? - _, record = reference_cache.records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + # Optimization: try exact commit hash match first + record = reference_cache.records_per_parent[project].fetch(id, nil) + + unless record + _, record = reference_cache.records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) } + end record end diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb index 337075b7ff8..b536d900a02 100644 --- a/lib/banzai/filter/references/issue_reference_filter.rb +++ b/lib/banzai/filter/references/issue_reference_filter.rb @@ -29,6 +29,14 @@ module Banzai super + design_link_extras(issue, matches.named_captures['path']) end + def reference_class(object_sym, tooltip: false) + super + end + + def data_attributes_for(text, parent, object, **data) + super.merge(project_path: parent.full_path, iid: object.iid) + end + private def additional_object_attributes(issue) diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb index 6c5ad83d9ae..5bc18ee6985 100644 --- a/lib/banzai/filter/references/merge_request_reference_filter.rb +++ b/lib/banzai/filter/references/merge_request_reference_filter.rb @@ -17,12 +17,6 @@ module Banzai only_path: context[:only_path]) end - def object_link_title(object, matches) - # The method will return `nil` if object is not a commit - # allowing for properly handling the extended MR Tooltip - object_link_commit_title(object, matches) - end - def object_link_text_extras(object, matches) extras = super @@ -53,20 +47,16 @@ module Banzai .includes(target_project: :namespace) end - def reference_class(object_sym, options = {}) - super(object_sym, tooltip: false) + def reference_class(object_sym, tooltip: false) + super end def data_attributes_for(text, parent, object, **data) - super.merge(project_path: parent.full_path, iid: object.iid, mr_title: object.title) + super.merge(project_path: parent.full_path, iid: object.iid) end private - def object_link_commit_title(object, matches) - object_link_commit(object, matches)&.title - end - def object_link_commit_ref(object, matches) object_link_commit(object, matches)&.short_id end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index bcd9f39d1dc..7175e99f1c7 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -60,6 +60,7 @@ module Banzai highlighted = %(<div class="gl-relative markdown-code-block js-markdown-code"><pre #{sourcepos_attr} class="#{css_classes}" lang="#{language}" + #{lang != language ? "data-canonical-lang=\"#{escape_once(lang)}\"" : ""} #{lang_params} v-pre="true"><code>#{code}</code></pre><copy-code></copy-code></div>) diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index c4db53424fd..0378a9c605d 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -5,6 +5,21 @@ module BulkImports class Stage < ::BulkImports::Stage private + # To skip the execution of a pipeline in a specific source instance version, define the attributes + # `minimum_source_version` and `maximum_source_version`. + # + # Use the `minimum_source_version` to inform that the pipeline needs to run when importing from source instances + # version greater than or equal to the specified minimum source version. For example, if the + # `minimum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.0, 15.1.1, 15.2.0, 16.0.0, etc. And it won't be executed when the source instance version + # is 15.0.1, 15.0.0, 14.10.0, etc. + # + # Use the `maximum_source_version` to inform that the pipeline needs to run when importing from source instance + # versions less than or equal to the specified maximum source version. For example, if the + # `maximum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.1 (patch), 15.1.0, 15.0.1, 15.0.0, 14.10.0, etc. And it won't be executed when the source + # instance version is 15.2.0, 15.2.1, 16.0.0, etc. + def config @config ||= { group: { @@ -21,7 +36,8 @@ module BulkImports }, namespace_settings: { pipeline: BulkImports::Groups::Pipelines::NamespaceSettingsPipeline, - stage: 1 + stage: 1, + minimum_source_version: '15.0.0' }, members: { pipeline: BulkImports::Common::Pipelines::MembersPipeline, diff --git a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb new file mode 100644 index 00000000000..2d5231b0541 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class DesignBundlePipeline + include Pipeline + + file_extraction_pipeline! + relation_name BulkImports::FileTransfer::ProjectConfig::DESIGN_BUNDLE_RELATION + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle") + + BulkImports::Pipeline::ExtractedData.new(data: bundle_path) + end + + def load(_context, bundle_path) + Gitlab::Utils.check_path_traversal!(bundle_path) + Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + + return unless portable.lfs_enabled? + return unless File.exist?(bundle_path) + return if File.directory?(bundle_path) + return if File.lstat(bundle_path).symlink? + + portable.design_repository.create_from_bundle(bundle_path) + end + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(self.class.relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def tar_filename + "#{self.class.relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb index 1754f27137c..d5886d7bae7 100644 --- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -10,16 +10,9 @@ module BulkImports relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION - transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer - - def extract(_context) - download_service.execute - decompression_service.execute - - project_attributes = json_decode(json_attributes) + extractor ::BulkImports::Common::Extractors::JsonExtractor, relation: relation - BulkImports::Pipeline::ExtractedData.new(data: project_attributes) - end + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer def transform(_context, data) subrelations = config.portable_relations_tree.keys.map(&:to_s) @@ -39,51 +32,14 @@ module BulkImports end def after_run(_context) - FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) - end - - def json_attributes - @json_attributes ||= File.read(File.join(tmpdir, filename)) + extractor.remove_tmpdir end private - def tmpdir - @tmpdir ||= Dir.mktmpdir('bulk_imports') - end - def config @config ||= BulkImports::FileTransfer.config_for(portable) end - - def download_service - @download_service ||= BulkImports::FileDownloadService.new( - configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(self.class.relation), - tmpdir: tmpdir, - filename: compressed_filename - ) - end - - def decompression_service - @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: compressed_filename) - end - - def compressed_filename - "#{filename}.gz" - end - - def filename - "#{self.class.relation}.json" - end - - def json_decode(string) - Gitlab::Json.parse(string) - rescue JSON::ParserError => e - Gitlab::ErrorTracking.log_exception(e) - - raise BulkImports::Error, 'Incorrect JSON format' - end end end end diff --git a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb index 8f9c6a5749f..c77e53b9aec 100644 --- a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb @@ -9,6 +9,22 @@ module BulkImports relation_name 'releases' extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + + def after_run(_context) + super + + portable.releases.find_each do |release| + create_release_evidence(release) + end + end + + private + + def create_release_evidence(release) + return if release.historical_release? || release.upcoming_release? + + ::Releases::CreateEvidenceWorker.perform_async(release.id) + end end end end diff --git a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb new file mode 100644 index 00000000000..9a3c582642f --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class RepositoryBundlePipeline + include Pipeline + + abort_on_failure! + file_extraction_pipeline! + relation_name BulkImports::FileTransfer::ProjectConfig::REPOSITORY_BUNDLE_RELATION + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle") + + BulkImports::Pipeline::ExtractedData.new(data: bundle_path) + end + + def load(_context, bundle_path) + Gitlab::Utils.check_path_traversal!(bundle_path) + Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + + return unless File.exist?(bundle_path) + return if File.directory?(bundle_path) + return if File.lstat(bundle_path).symlink? + + portable.repository.create_from_bundle(bundle_path) + end + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def tar_filename + "#{self.class.relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(self.class.relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 229df9c410d..acfa9163eae 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -5,6 +5,21 @@ module BulkImports class Stage < ::BulkImports::Stage private + # To skip the execution of a pipeline in a specific source instance version, define the attributes + # `minimum_source_version` and `maximum_source_version`. + # + # Use the `minimum_source_version` to inform that the pipeline needs to run when importing from source instances + # version greater than or equal to the specified minimum source version. For example, if the + # `minimum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.0, 15.1.1, 15.2.0, 16.0.0, etc. And it won't be executed when the source instance version + # is 15.0.1, 15.0.0, 14.10.0, etc. + # + # Use the `maximum_source_version` to inform that the pipeline needs to run when importing from source instance + # versions less than or equal to the specified maximum source version. For example, if the + # `maximum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.1 (patch), 15.1.0, 15.0.1, 15.0.0, 14.10.0, etc. And it won't be executed when the source + # instance version is 15.2.0, 15.2.1, 16.0.0, etc. + def config @config ||= { project: { @@ -13,6 +28,12 @@ module BulkImports }, repository: { pipeline: BulkImports::Projects::Pipelines::RepositoryPipeline, + maximum_source_version: '15.0.0', + stage: 1 + }, + repository_bundle: { + pipeline: BulkImports::Projects::Pipelines::RepositoryBundlePipeline, + minimum_source_version: '15.1.0', stage: 1 }, project_attributes: { @@ -95,6 +116,11 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::LfsObjectsPipeline, stage: 5 }, + design: { + pipeline: BulkImports::Projects::Pipelines::DesignBundlePipeline, + minimum_source_version: '15.1.0', + stage: 5 + }, auto_devops: { pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, stage: 5 diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index 6cf394c5df0..b45ac139385 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -15,9 +15,6 @@ module BulkImports @pipelines ||= config .values .sort_by { |entry| entry[:stage] } - .map do |entry| - [entry[:stage], entry[:pipeline]] - end end private diff --git a/lib/constraints/repository_redirect_url_constrainer.rb b/lib/constraints/repository_redirect_url_constrainer.rb index 44df670d8d3..046b3397152 100644 --- a/lib/constraints/repository_redirect_url_constrainer.rb +++ b/lib/constraints/repository_redirect_url_constrainer.rb @@ -18,11 +18,17 @@ module Constraints end # Check if the path matches any known repository containers. - # These also cover wikis, since a `.wiki` suffix is valid in project/group paths too. def container_path?(path) - NamespacePathValidator.valid_path?(path) || + wiki_path?(path) || ProjectPathValidator.valid_path?(path) || path =~ Gitlab::PathRegex.full_snippets_repository_path_regex end + + private + + # These also cover wikis, since a `.wiki` suffix is valid in project/group paths too. + def wiki_path?(path) + NamespacePathValidator.valid_path?(path) && path.end_with?('.wiki') + end end end diff --git a/lib/container_registry/base_client.rb b/lib/container_registry/base_client.rb index 66bc934d1ef..0b24b31c4ae 100644 --- a/lib/container_registry/base_client.rb +++ b/lib/container_registry/base_client.rb @@ -8,11 +8,12 @@ module ContainerRegistry class BaseClient DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE = 'application/vnd.docker.distribution.manifest.list.v2+json' + OCI_DISTRIBUTION_INDEX_TYPE = 'application/vnd.oci.image.index.v1+json' OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json' ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze - ACCEPTED_TYPES_RAW = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE, DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE].freeze + ACCEPTED_TYPES_RAW = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE, DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE, OCI_DISTRIBUTION_INDEX_TYPE].freeze RETRY_EXCEPTIONS = [Faraday::Request::Retry::DEFAULT_EXCEPTIONS, Faraday::ConnectionFailed].flatten.freeze RETRY_OPTIONS = { @@ -107,6 +108,7 @@ module ContainerRegistry conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE conn.response :json, content_type: OCI_MANIFEST_V1_TYPE + conn.response :json, content_type: OCI_DISTRIBUTION_INDEX_TYPE end def delete_if_exists(path) diff --git a/lib/container_registry/migration.rb b/lib/container_registry/migration.rb index 8377190c83c..92a001f9c24 100644 --- a/lib/container_registry/migration.rb +++ b/lib/container_registry/migration.rb @@ -22,6 +22,7 @@ module ContainerRegistry delegate :container_registry_import_created_before, to: ::Gitlab::CurrentSettings delegate :container_registry_pre_import_timeout, to: ::Gitlab::CurrentSettings delegate :container_registry_import_timeout, to: ::Gitlab::CurrentSettings + delegate :container_registry_pre_import_tags_rate, to: ::Gitlab::CurrentSettings alias_method :max_tags_count, :container_registry_import_max_tags_count alias_method :max_retries, :container_registry_import_max_retries @@ -31,6 +32,7 @@ module ContainerRegistry alias_method :created_before, :container_registry_import_created_before alias_method :pre_import_timeout, :container_registry_pre_import_timeout alias_method :import_timeout, :container_registry_import_timeout + alias_method :pre_import_tags_rate, :container_registry_pre_import_tags_rate end def self.enabled? @@ -41,6 +43,10 @@ module ContainerRegistry Feature.enabled?(:container_registry_migration_limit_gitlab_org) end + def self.delete_container_repository_worker_support? + Feature.enabled?(:container_registry_migration_phase2_delete_container_repository_worker_support) + end + def self.enqueue_waiting_time return 0 if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_fast) return 165.minutes if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_slow) @@ -54,6 +60,7 @@ module ContainerRegistry # # TODO: See https://gitlab.com/gitlab-org/container-registry/-/issues/582 # + return 40 if Feature.enabled?(:container_registry_migration_phase2_capacity_40) return 25 if Feature.enabled?(:container_registry_migration_phase2_capacity_25) return 10 if Feature.enabled?(:container_registry_migration_phase2_capacity_10) return 5 if Feature.enabled?(:container_registry_migration_phase2_capacity_5) @@ -71,12 +78,8 @@ module ContainerRegistry Feature.enabled?(:container_registry_migration_phase2_all_plans) end - def self.enqueue_twice? - Feature.enabled?(:container_registry_migration_phase2_enqueue_twice) - end - - def self.enqueuer_loop? - Feature.enabled?(:container_registry_migration_phase2_enqueuer_loop) + def self.dynamic_pre_import_timeout_for(repository) + (repository.tags_count * pre_import_tags_rate).seconds end end end diff --git a/lib/error_tracking/stacktrace_builder.rb b/lib/error_tracking/stacktrace_builder.rb new file mode 100644 index 00000000000..4f331bc4e06 --- /dev/null +++ b/lib/error_tracking/stacktrace_builder.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module ErrorTracking + class StacktraceBuilder + attr_reader :stacktrace + + def initialize(payload) + @stacktrace = build_stacktrace(payload) + end + + private + + def build_stacktrace(payload) + raw_stacktrace = raw_stacktrace_from_payload(payload) + return [] unless raw_stacktrace + + raw_stacktrace.map do |entry| + { + 'lineNo' => entry['lineno'], + 'context' => build_stacktrace_context(entry), + 'filename' => entry['filename'], + 'function' => entry['function'], + 'colNo' => 0 # we don't support colNo yet. + } + end + end + + def raw_stacktrace_from_payload(payload) + exception_entry = payload['exception'] + return unless exception_entry + + exception_values = exception_entry['values'] + stack_trace_entry = exception_values&.detect { |h| h['stacktrace'].present? } + stack_trace_entry&.dig('stacktrace', 'frames') + end + + def build_stacktrace_context(entry) + error_line = entry['context_line'] + error_line_no = entry['lineno'] + pre_context = entry['pre_context'] + post_context = entry['post_context'] + + context = [] + context.concat lines_with_position(pre_context, error_line_no - pre_context.size) if pre_context + context.concat lines_with_position([error_line], error_line_no) + context.concat lines_with_position(post_context, error_line_no + 1) if post_context + + context.reject(&:blank?) + end + + def lines_with_position(lines, position) + return [] if lines.blank? + + lines.map.with_index do |line, index| + next unless line + + [position + index, line] + end + end + end +end diff --git a/lib/feature.rb b/lib/feature.rb index b5a97ee8f9b..3bba4be7514 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -281,6 +281,8 @@ class Feature end class Target + UnknowTargetError = Class.new(StandardError) + attr_reader :params def initialize(params) @@ -292,7 +294,7 @@ class Feature end def targets - [feature_group, user, project, group, namespace].compact + [feature_group, users, projects, groups, namespaces].flatten.compact end private @@ -305,29 +307,37 @@ class Feature end # rubocop: enable CodeReuse/ActiveRecord - def user + def users return unless params.key?(:user) - UserFinder.new(params[:user]).find_by_username! + params[:user].split(',').map do |arg| + UserFinder.new(arg).find_by_username || (raise UnknowTargetError, "#{arg} is not found!") + end end - def project + def projects return unless params.key?(:project) - Project.find_by_full_path(params[:project]) + params[:project].split(',').map do |arg| + Project.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") + end end - def group + def groups return unless params.key?(:group) - Group.find_by_full_path(params[:group]) + params[:group].split(',').map do |arg| + Group.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") + end end - def namespace + def namespaces return unless params.key?(:namespace) - # We are interested in Group or UserNamespace - Namespace.without_project_namespaces.find_by_full_path(params[:namespace]) + params[:namespace].split(',').map do |arg| + # We are interested in Group or UserNamespace + Namespace.without_project_namespaces.find_by_full_path(arg) || (raise UnknowTargetError, "#{arg} is not found!") + end end end end diff --git a/lib/generators/gitlab/usage_metric/templates/numbers_instrumentation_class.rb.template b/lib/generators/gitlab/usage_metric/templates/numbers_instrumentation_class.rb.template new file mode 100644 index 00000000000..ef9537f970e --- /dev/null +++ b/lib/generators/gitlab/usage_metric/templates/numbers_instrumentation_class.rb.template @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class <%= class_name %>Metric < NumbersMetric + operation :<%= operation%> + + data do |time_frame| + [ + # Insert numbers here + ] + end + end + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric_generator.rb index 0656dfbc312..3624a6eb5a7 100644 --- a/lib/generators/gitlab/usage_metric_generator.rb +++ b/lib/generators/gitlab/usage_metric_generator.rb @@ -12,10 +12,13 @@ module Gitlab ALLOWED_SUPERCLASSES = { generic: 'Generic', database: 'Database', - redis: 'Redis' + redis: 'Redis', + numbers: 'Numbers' }.freeze - ALLOWED_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count).freeze + ALLOWED_DATABASE_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count sum average).freeze + ALLOWED_NUMBERS_OPERATIONS = %w(add).freeze + ALLOWED_OPERATIONS = ALLOWED_DATABASE_OPERATIONS | ALLOWED_NUMBERS_OPERATIONS source_root File.expand_path('usage_metric/templates', __dir__) @@ -29,6 +32,7 @@ module Gitlab validate! template "database_instrumentation_class.rb.template", file_path if type == 'database' + template "numbers_instrumentation_class.rb.template", file_path if type == 'numbers' template "generic_instrumentation_class.rb.template", file_path if type == 'generic' template "instrumentation_class_spec.rb.template", spec_file_path @@ -39,7 +43,8 @@ module Gitlab def validate! raise ArgumentError, "Type is required, valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" unless type.present? raise ArgumentError, "Unknown type '#{type}', valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" if metric_superclass.nil? - raise ArgumentError, "Unknown operation '#{operation}' valid operations are #{ALLOWED_OPERATIONS.join(', ')}" if type == 'database' && !ALLOWED_OPERATIONS.include?(operation) + raise ArgumentError, "Unknown operation '#{operation}' valid operations for database are #{ALLOWED_DATABASE_OPERATIONS.join(', ')}" if type == 'database' && ALLOWED_DATABASE_OPERATIONS.exclude?(operation) + raise ArgumentError, "Unknown operation '#{operation}' valid operations for numbers are #{ALLOWED_NUMBERS_OPERATIONS.join(', ')}" if type == 'numbers' && ALLOWED_NUMBERS_OPERATIONS.exclude?(operation) end def ee? diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 55e421173d7..07dc4c02ba8 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -38,7 +38,7 @@ module Gitlab strong_memoize(:serialized_records) do # When RecordsFetcher is used with query sourced from # InOperatorOptimization::QueryBuilder only columns - # used in ORDER BY statement would be selected by Arel.start operation + # used in ORDER BY statement would be selected by Arel.star operation selections = [stage_event_model.arel_table[Arel.star]] selections << duration_in_seconds.as('total_time') if params[:sort] != :duration # duration sorting already exposes this data @@ -55,7 +55,9 @@ module Gitlab project_path: project.path, namespace_path: project.namespace.route.path, author: issuable.author, - total_time: record.total_time + total_time: record.total_time, + start_event_timestamp: record.start_event_timestamp, + end_event_timestamp: record.end_event_timestamp }) serializer.represent(attributes) end diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb deleted file mode 100644 index 3546a7e3ddb..00000000000 --- a/lib/gitlab/analytics/unique_visits.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Analytics - class UniqueVisits - # Returns number of unique visitors for given targets in given time frame - # - # @param [String, Array[<String>]] targets ids of targets to count visits on. Special case for :any - # @param [ActiveSupport::TimeWithZone] start_date start of time frame - # @param [ActiveSupport::TimeWithZone] end_date end of time frame - # @return [Integer] number of unique visitors - def unique_visits_for(targets:, start_date: 7.days.ago, end_date: start_date + 1.week) - events = if targets == :analytics - self.class.analytics_events - elsif targets == :compliance - self.class.compliance_events - else - Array(targets) - end - - Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, start_date: start_date, end_date: end_date) - end - - class << self - def analytics_events - Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('analytics') - end - - def compliance_events - Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('compliance') - end - end - end - end -end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 6ef5a1e2cd8..3e095585b18 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -20,7 +20,8 @@ module Gitlab :pipeline_id, :related_class, :feature_category, - :artifact_size + :artifact_size, + :root_caller_id ].freeze private_constant :KNOWN_KEYS @@ -34,7 +35,8 @@ module Gitlab Attribute.new(:job, ::Ci::Build), Attribute.new(:related_class, String), Attribute.new(:feature_category, String), - Attribute.new(:artifact, ::Ci::JobArtifact) + Attribute.new(:artifact, ::Ci::JobArtifact), + Attribute.new(:root_caller_id, String) ].freeze def self.known_keys @@ -84,10 +86,11 @@ module Gitlab hash[:project] = -> { project_path } if include_project? hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:client_id] = -> { client } if include_client? - hash[:caller_id] = caller_id if set_values.include?(:caller_id) - hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) - hash[:related_class] = related_class if set_values.include?(:related_class) - hash[:feature_category] = feature_category if set_values.include?(:feature_category) + assign_hash_if_value(hash, :caller_id) + assign_hash_if_value(hash, :root_caller_id) + assign_hash_if_value(hash, :remote_ip) + assign_hash_if_value(hash, :related_class) + assign_hash_if_value(hash, :feature_category) hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) hash[:job_id] = -> { job&.id } if set_values.include?(:job) hash[:artifact_size] = -> { artifact&.size } if set_values.include?(:artifact) @@ -108,6 +111,14 @@ module Gitlab lazy_attr_reader attr.name, type: attr.type end + def assign_hash_if_value(hash, attribute_name) + raise ArgumentError unless KNOWN_KEYS.include?(attribute_name) + + # rubocop:disable GitlabSecurity/PublicSend + hash[attribute_name] = public_send(attribute_name) if set_values.include?(attribute_name) + # rubocop:enable GitlabSecurity/PublicSend + end + def assign_attributes(values) values.slice(*APPLICATION_ATTRIBUTES.map(&:name)).each do |name, value| instance_variable_set("@#{name}", value) diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 41a6cbc2543..722ee061eba 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -32,6 +32,8 @@ module Gitlab group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, web_hook_calls: { interval: 1.minute }, + web_hook_calls_mid: { interval: 1.minute }, + web_hook_calls_low: { interval: 1.minute }, users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, username_exists: { threshold: 20, interval: 1.minute }, user_sign_up: { threshold: 20, interval: 1.minute }, @@ -42,7 +44,8 @@ module Gitlab search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, gitlab_shell_operation: { threshold: 600, interval: 1.minute }, - pipelines_create: { threshold: 25, interval: 1.minute } + pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, + temporary_email_failure: { threshold: 50, interval: 1.day } }.freeze end @@ -190,3 +193,5 @@ module Gitlab end end end + +Gitlab::ApplicationRateLimiter.prepend_mod diff --git a/lib/gitlab/audit/unauthenticated_author.rb b/lib/gitlab/audit/unauthenticated_author.rb index 84c323c1950..b811f9f8ad0 100644 --- a/lib/gitlab/audit/unauthenticated_author.rb +++ b/lib/gitlab/audit/unauthenticated_author.rb @@ -12,6 +12,10 @@ module Gitlab def name @name || _('An unauthenticated user') end + + def impersonated? + false + end end end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index d9efb6b8d2d..7d9c4c0d7c1 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -217,7 +217,11 @@ module Gitlab def build_new_user(skip_confirmation: true) user_params = user_attributes.merge(skip_confirmation: skip_confirmation) - Users::AuthorizedBuildService.new(nil, user_params).execute + new_user = Users::AuthorizedBuildService.new(nil, user_params).execute + + persist_accepted_terms_if_required(new_user) + + new_user end def user_attributes @@ -245,6 +249,15 @@ module Gitlab } end + def persist_accepted_terms_if_required(new_user) + if Feature.enabled?(:update_oauth_registration_flow) && + Gitlab::CurrentSettings.current_application_settings.enforce_terms? + + terms = ApplicationSetting::Term.latest + Users::RespondToTermsService.new(new_user, terms).execute(accepted: true) + end + end + def sync_profile_from_provider? Gitlab::Auth::OAuth::Provider.sync_profile_from_provider?(auth_hash.provider) end diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb new file mode 100644 index 00000000000..814f5a897a9 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill projectfeatures.package_registry_access_level depending on projects.packages_enabled + class BackfillProjectFeaturePackageRegistryAccessLevel < ::Gitlab::BackgroundMigration::BatchedMigrationJob + FEATURE_DISABLED = 0 # ProjectFeature::DISABLED + FEATURE_PRIVATE = 10 # ProjectFeature::PRIVATE + FEATURE_ENABLED = 20 # ProjectFeature::ENABLED + FEATURE_PUBLIC = 30 # ProjectFeature::PUBLIC + PROJECT_PRIVATE = 0 # Gitlab::VisibilityLevel::PRIVATE + PROJECT_INTERNAL = 10 # Gitlab::VisibilityLevel::INTERNAL + PROJECT_PUBLIC = 20 # Gitlab::VisibilityLevel::PUBLIC + + # Migration only version of ProjectFeature table + class ProjectFeature < ::ApplicationRecord + self.table_name = 'project_features' + end + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + ProjectFeature.connection.execute( + <<~SQL + UPDATE project_features pf + SET package_registry_access_level = (CASE p.packages_enabled + WHEN true THEN (CASE p.visibility_level + WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC} + WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED} + WHEN #{PROJECT_PRIVATE} THEN #{FEATURE_PRIVATE} + END) + WHEN false THEN #{FEATURE_DISABLED} + ELSE #{FEATURE_DISABLED} + END) + FROM projects p + WHERE pf.project_id = p.id AND + pf.project_id BETWEEN #{start_id} AND #{end_id} + SQL + ) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb b/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb new file mode 100644 index 00000000000..c2e37269b5e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_member_namespace_id.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `members.member_namespace_id` column for `type=ProjectMember` + class BackfillProjectMemberNamespaceId < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| + batch_metrics.time_operation(:update_all) do + # rubocop:disable Layout/LineLength + sub_batch.update_all('member_namespace_id = (SELECT projects.project_namespace_id FROM projects WHERE projects.id = source_id)') + # rubocop:enable Layout/LineLength + end + + pause_ms_value = [0, pause_ms].max + sleep(pause_ms_value * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ApplicationRecord.connection) + .where(source_key_column => start_id..stop_id) + .joins('INNER JOIN projects ON members.source_id = projects.id') + .where(type: 'ProjectMember', source_type: 'Project') + .where(member_namespace_id: nil) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb new file mode 100644 index 00000000000..0cd19dc5df9 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removes orphaned routes, i.e. routes that reference a namespace or project that no longer exists. + # This was possible since we were using a polymorphic association source_id, source_type. However since now + # we have project namespaces we can use a FK on routes#namespace_id to avoid orphaned records in routes. + class CleanupOrphanedRoutes < Gitlab::BackgroundMigration::BatchedMigrationJob + include Gitlab::Database::DynamicModelHelpers + + def perform + # there should really be no records to fix, there is none gitlab.com, but taking the safer route, just in case. + fix_missing_namespace_id_routes + cleanup_orphaned_routes + end + + private + + def fix_missing_namespace_id_routes + non_orphaned_namespace_routes = non_orphaned_namespace_routes_scoped_to_range(batch_column, start_id, end_id) + non_orphaned_project_routes = non_orphaned_project_routes_scoped_to_range(batch_column, start_id, end_id) + + update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size) + update_namespace_id(batch_column, non_orphaned_project_routes, sub_batch_size) + end + + def cleanup_orphaned_routes + orphaned_namespace_routes = orphaned_namespace_routes_scoped_to_range(batch_column, start_id, end_id) + orphaned_project_routes = orphaned_project_routes_scoped_to_range(batch_column, start_id, end_id) + + cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size) + cleanup_relations(batch_column, orphaned_project_routes, pause_ms, sub_batch_size) + end + + def update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size) + non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:fix_missing_namespace_id) do + ApplicationRecord.connection.execute <<~SQL + WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{sub_batch.to_sql} + ) + UPDATE routes + SET namespace_id = route_and_ns.namespace_id + FROM route_and_ns + WHERE id = route_and_ns.route_id + SQL + end + end + end + + def cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size) + orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:cleanup_orphaned_routes) do + sub_batch.delete_all + end + end + end + + def orphaned_namespace_routes_scoped_to_range(source_key_column, start_id, stop_id) + Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN namespaces ON source_id = namespaces.id") + .where(source_key_column => start_id..stop_id) + .where(source_type: 'Namespace') + .where(namespace_id: nil) + .where(namespaces: { id: nil }) + end + + def orphaned_project_routes_scoped_to_range(source_key_column, start_id, stop_id) + Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN projects ON source_id = projects.id") + .where(source_key_column => start_id..stop_id) + .where(source_type: 'Project') + .where(namespace_id: nil) + .where(projects: { id: nil }) + end + + def non_orphaned_namespace_routes_scoped_to_range(source_key_column, start_id, stop_id) + Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN namespaces ON source_id = namespaces.id") + .where(source_key_column => start_id..stop_id) + .where(source_type: 'Namespace') + .where(namespace_id: nil) + .where.not(namespaces: { id: nil }) + .select("routes.id, namespaces.id") + end + + def non_orphaned_project_routes_scoped_to_range(source_key_column, start_id, stop_id) + Gitlab::BackgroundMigration::Route.joins("LEFT OUTER JOIN projects ON source_id = projects.id") + .where(source_key_column => start_id..stop_id) + .where(source_type: 'Project') + .where(namespace_id: nil) + .where.not(projects: { id: nil }) + .select("routes.id, projects.project_namespace_id") + end + end + + # Isolated route model for the migration + class Route < ApplicationRecord + include EachBatch + + self.table_name = 'routes' + self.inheritance_column = :_type_disabled + end + end +end diff --git a/lib/gitlab/background_migration/delete_invalid_epic_issues.rb b/lib/gitlab/background_migration/delete_invalid_epic_issues.rb new file mode 100644 index 00000000000..3af59ab4931 --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_epic_issues.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class DeleteInvalidEpicIssues < BatchedMigrationJob + def perform + end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::DeleteInvalidEpicIssues.prepend_mod_with('Gitlab::BackgroundMigration::DeleteInvalidEpicIssues') diff --git a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb index ea3e56cb14a..4df55a7b02a 100644 --- a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb +++ b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb @@ -5,12 +5,6 @@ module Gitlab # Background migration for fixing merge_request_diff_commit rows that don't # have committer/author details due to # https://gitlab.com/gitlab-org/gitlab/-/issues/344080. - # - # This migration acts on a single project and corrects its data. Because - # this process needs Git/Gitaly access, and duplicating all that code is far - # too much, this migration relies on global models such as Project, - # MergeRequest, etc. - # rubocop: disable Metrics/ClassLength class FixMergeRequestDiffCommitUsers BATCH_SIZE = 100 @@ -20,137 +14,8 @@ module Gitlab end def perform(project_id) - if (project = ::Project.find_by_id(project_id)) - process(project) - end - - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixMergeRequestDiffCommitUsers', - [project_id] - ) - - schedule_next_job - end - - def process(project) - # Loading everything using one big query may result in timeouts (e.g. - # for projects the size of gitlab-org/gitlab). So instead we query - # data on a per merge request basis. - project.merge_requests.each_batch(column: :iid) do |mrs| - mrs.ids.each do |mr_id| - each_row_to_check(mr_id) do |commit| - update_commit(project, commit) - end - end - end - end - - def each_row_to_check(merge_request_id, &block) - columns = %w[merge_request_diff_id relative_order].map do |col| - Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: col, - order_expression: MergeRequestDiffCommit.arel_table[col.to_sym].asc, - nullable: :not_nullable, - distinct: false - ) - end - - order = Pagination::Keyset::Order.build(columns) - scope = MergeRequestDiffCommit - .joins(:merge_request_diff) - .where(merge_request_diffs: { merge_request_id: merge_request_id }) - .where('commit_author_id IS NULL OR committer_id IS NULL') - .order(order) - - Pagination::Keyset::Iterator - .new(scope: scope, use_union_optimization: true) - .each_batch(of: BATCH_SIZE) do |rows| - rows - .select([ - :merge_request_diff_id, - :relative_order, - :sha, - :committer_id, - :commit_author_id - ]) - .each(&block) - end - end - - # rubocop: disable Metrics/AbcSize - def update_commit(project, row) - commit = find_commit(project, row.sha) - updates = [] - - unless row.commit_author_id - author_id = find_or_create_user(commit, :author_name, :author_email) - - updates << [arel_table[:commit_author_id], author_id] if author_id - end - - unless row.committer_id - committer_id = - find_or_create_user(commit, :committer_name, :committer_email) - - updates << [arel_table[:committer_id], committer_id] if committer_id - end - - return if updates.empty? - - update = Arel::UpdateManager - .new - .table(MergeRequestDiffCommit.arel_table) - .where(matches_row(row)) - .set(updates) - .to_sql - - MergeRequestDiffCommit.connection.execute(update) - end - # rubocop: enable Metrics/AbcSize - - def schedule_next_job - job = Database::BackgroundMigrationJob - .for_migration_class('FixMergeRequestDiffCommitUsers') - .pending - .first - - return unless job - - BackgroundMigrationWorker.perform_in( - 2.minutes, - 'FixMergeRequestDiffCommitUsers', - job.arguments - ) - end - - def find_commit(project, sha) - @commits[sha] ||= (project.commit(sha)&.to_hash || {}) - end - - def find_or_create_user(commit, name_field, email_field) - name = commit[name_field] - email = commit[email_field] - - return unless name && email - - @users[[name, email]] ||= - MergeRequest::DiffCommitUser.find_or_create(name, email).id - end - - def matches_row(row) - primary_key = Arel::Nodes::Grouping - .new([arel_table[:merge_request_diff_id], arel_table[:relative_order]]) - - primary_val = Arel::Nodes::Grouping - .new([row.merge_request_diff_id, row.relative_order]) - - primary_key.eq(primary_val) - end - - def arel_table - MergeRequestDiffCommit.arel_table + # No-op, see https://gitlab.com/gitlab-org/gitlab/-/issues/344540 end end - # rubocop: enable Metrics/ClassLength end end diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb index b7a912da060..f53f2e8ee79 100644 --- a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb +++ b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb @@ -9,10 +9,7 @@ module Gitlab # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion class MigratePagesToZipStorage def perform(start_id, stop_id) - ::Pages::MigrateFromLegacyStorageService.new(Gitlab::AppLogger, - ignore_invalid_entries: false, - mark_projects_as_not_deployed: false) - .execute_for_batch(start_id..stop_id) + # no-op end end end diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb index 36d4e649271..13b66b2e02e 100644 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb @@ -10,9 +10,9 @@ module Gitlab pause_ms = 0 if pause_ms < 0 batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| + batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| batch_metrics.time_operation(:update_all) do - sub_batch.update_all(runner_id: nil) + filtered_sub_batch(sub_batch).update_all(runner_id: nil) end sleep(pause_ms * 0.001) @@ -31,9 +31,13 @@ module Gitlab def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) define_batchable_model(source_table, connection: connection) + .where(source_key_column => start_id..stop_id) + end + + def filtered_sub_batch(sub_batch) + sub_batch .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id') .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL') - .where(source_key_column => start_id..stop_id) end end end diff --git a/lib/gitlab/background_migration/purge_stale_security_scans.rb b/lib/gitlab/background_migration/purge_stale_security_scans.rb new file mode 100644 index 00000000000..8b13a0382b4 --- /dev/null +++ b/lib/gitlab/background_migration/purge_stale_security_scans.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PurgeStaleSecurityScans # rubocop:disable Migration/BackgroundMigrationBaseClass + class SecurityScan < ::ApplicationRecord + include EachBatch + + STALE_AFTER = 90.days + + self.table_name = 'security_scans' + + # Otherwise the schema_spec fails + validates :info, json_schema: { filename: 'security_scan_info', draft: 7 } + + enum status: { succeeded: 1, purged: 6 } + + scope :to_purge, -> { where('id <= ?', last_stale_record_id) } + scope :by_range, -> (range) { where(id: range) } + + def self.last_stale_record_id + where('created_at < ?', STALE_AFTER.ago).order(created_at: :desc).first + end + end + + def perform(_start_id, _end_id); end + end + end +end + +Gitlab::BackgroundMigration::PurgeStaleSecurityScans.prepend_mod diff --git a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb new file mode 100644 index 00000000000..e85b1bc402a --- /dev/null +++ b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for non-public projects + class SetLegacyOpenSourceLicenseAvailableForNonPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :set_legacy_open_source_license_available, + batching_scope: ->(relation) { relation.where.not(visibility_level: PUBLIC) } + ) do |sub_batch| + ProjectSetting.where(project_id: sub_batch).update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index 0f370850b5b..81b01395542 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -6,6 +6,9 @@ module Gitlab class BaseDoorkeeperController < ActionController::Base include Gitlab::Allowable include EnforcesTwoFactorAuthentication + include SessionsHelper + + before_action :limit_session_time, if: -> { !current_user } helper_method :can? end diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb index 48ca4951957..ddc678abdd8 100644 --- a/lib/gitlab/bitbucket_server_import/project_creator.rb +++ b/lib/gitlab/bitbucket_server_import/project_creator.rb @@ -19,7 +19,7 @@ module Gitlab ::Projects::CreateService.new( current_user, name: name, - path: name, + path: repo_slug, description: repo.description, namespace_id: namespace.id, visibility_level: repo.visibility_level, diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 2e469aabeb2..99752dc6a01 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -36,35 +36,17 @@ module Gitlab # any of the new revisions. def commits strong_memoize(:commits) do - allow_quarantine = true - newrevs = @changes.map do |change| - oldrev = change[:oldrev] newrev = change[:newrev] next if blank_rev?(newrev) - # In case any of the old revisions is blank, then we cannot reliably - # detect which commits are new for a given change when enumerating - # objects via the object quarantine directory given that the client - # may have pushed too many commits, and we don't know when to - # terminate the walk. We thus fall back to using `git rev-list --not - # --all`, which is a lot less efficient but at least can only ever - # returns commits which really are new. - allow_quarantine = false if allow_quarantine && blank_rev?(oldrev) - newrev end.compact next [] if newrevs.empty? - # When filtering quarantined commits we can enable usage of the object - # quarantine no matter whether we have an `oldrev` or not. - if Feature.enabled?(:filter_quarantined_commits) - allow_quarantine = true - end - - project.repository.new_commits(newrevs, allow_quarantine: allow_quarantine) + project.repository.new_commits(newrevs) end end diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 8e12801daee..2fd48dfbfe2 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -35,8 +35,7 @@ module Gitlab end def commits - @commits ||= project.repository.new_commits(newrev, - allow_quarantine: Feature.enabled?(:filter_quarantined_commits)) + @commits ||= project.repository.new_commits(newrev) end protected diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index a45db85301a..5dd7720b67d 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -6,7 +6,9 @@ module Gitlab ERROR_MESSAGES = { change_existing_tags: 'You are not allowed to change existing tags on this project.', update_protected_tag: 'Protected tags cannot be updated.', - delete_protected_tag: 'Protected tags cannot be deleted.', + delete_protected_tag: 'You are not allowed to delete protected tags from this project. '\ + 'Only a project maintainer or owner can delete a protected tag.', + delete_protected_tag_non_web: 'You can only delete protected tags using the web interface.', create_protected_tag: 'You are not allowed to create this tag as it is protected.' }.freeze @@ -34,7 +36,16 @@ module Gitlab return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + + if deletion? + unless user_access.user.can?(:maintainer_access, project) + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) + end + + unless updated_from_web? + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag_non_web] + end + end unless user_access.can_create_tag?(tag_name) raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 8ddcf1d523e..7dc375e05eb 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -4,7 +4,7 @@ module Gitlab module Ci module Build class Image - attr_reader :alias, :command, :entrypoint, :name, :ports, :variables + attr_reader :alias, :command, :entrypoint, :name, :ports, :variables, :pull_policy class << self def from_image(job) @@ -34,6 +34,7 @@ module Gitlab @name = image[:name] @ports = build_ports(image).select(&:valid?) @variables = build_variables(image) + @pull_policy = image[:pull_policy] end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 21c42857895..79443f69b03 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -12,11 +12,13 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name entrypoint ports].freeze + ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze + LEGACY_ALLOWED_KEYS = %i[name entrypoint ports].freeze validations do validates :config, hash_or_string: true - validates :config, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: ALLOWED_KEYS, if: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: LEGACY_ALLOWED_KEYS, unless: :ci_docker_image_pull_policy_enabled? validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? validates :name, type: String, presence: true @@ -26,7 +28,10 @@ module Gitlab entry :ports, Entry::Ports, description: 'Ports used to expose the image' - attributes :ports + entry :pull_policy, Entry::PullPolicy, + description: 'Pull policy for the image' + + attributes :ports, :pull_policy def name value[:name] @@ -37,16 +42,28 @@ module Gitlab end def value - return { name: @config } if string? - return @config if hash? - - {} + if string? + { name: @config } + elsif hash? + { + name: @config[:name], + entrypoint: @config[:entrypoint], + ports: ports_value, + pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) + }.compact + else + {} + end end def with_image_ports? opt(:with_image_ports) end + def ci_docker_image_pull_policy_enabled? + ::Feature.enabled?(:ci_docker_image_pull_policy) + end + def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/entry/pull_policy.rb b/lib/gitlab/ci/config/entry/pull_policy.rb new file mode 100644 index 00000000000..f597134dd2c --- /dev/null +++ b/lib/gitlab/ci/config/entry/pull_policy.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of the pull policies of an image. + # + class PullPolicy < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_POLICIES = %w[always never if-not-present].freeze + + validations do + validates :config, array_of_strings_or_string: true + validates :config, + allowed_array_values: { in: ALLOWED_POLICIES }, + presence: true, + if: :array? + validates :config, + inclusion: { in: ALLOWED_POLICIES }, + if: :string? + end + + def value + # We either return an array with policies or nothing + Array(@config).presence + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 4722f2e9a61..63bf1b38ac6 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -9,11 +9,13 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - CLAUSES = %i[if changes exists].freeze - ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables].freeze - ALLOWABLE_WHEN = %w[on_success on_failure always never manual delayed].freeze + ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables].freeze + ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze - attributes :if, :changes, :exists, :when, :start_in, :allow_failure + attributes :if, :exists, :when, :start_in, :allow_failure + + entry :changes, Entry::Rules::Rule::Changes, + description: 'File change condition rule.' entry :variables, Entry::Variables, description: 'Environment variables to define for rule conditions.' @@ -28,8 +30,8 @@ module Gitlab with_options allow_nil: true do validates :if, expression: true - validates :changes, :exists, array_of_strings: true, length: { maximum: 50 } - validates :when, allowed_values: { in: ALLOWABLE_WHEN } + validates :exists, array_of_strings: true, length: { maximum: 50 } + validates :when, allowed_values: { in: ALLOWED_WHEN } validates :allow_failure, boolean: true end @@ -41,6 +43,13 @@ module Gitlab end end + def value + config.merge( + changes: (changes_value if changes_defined?), + variables: (variables_value if variables_defined?) + ).compact + end + def specifies_delay? self.when == 'delayed' end diff --git a/lib/gitlab/ci/config/entry/rules/rule/changes.rb b/lib/gitlab/ci/config/entry/rules/rule/changes.rb new file mode 100644 index 00000000000..be57e089f34 --- /dev/null +++ b/lib/gitlab/ci/config/entry/rules/rule/changes.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Rules + class Rule + class Changes < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index feb2cbb19ad..36fc5c656fc 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -42,7 +42,9 @@ module Gitlab end def fetch_local_content - context.project.repository.blob_data_at(context.sha, location) + context.logger.instrument(:config_file_fetch_local_content) do + context.project.repository.blob_data_at(context.sha, location) + end rescue GRPC::InvalidArgument errors.push("Sha #{context.sha} is not valid!") diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 09c36a1bcb6..b7fef081269 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -65,7 +65,9 @@ module Gitlab return unless can_access_local_content? return unless sha - project.repository.blob_data_at(sha, location) + context.logger.instrument(:config_file_fetch_project_content) do + project.repository.blob_data_at(sha, location) + end rescue GRPC::NotFound, GRPC::Internal nil end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 7d3a2362246..3984bf9e4f8 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -40,7 +40,9 @@ module Gitlab def fetch_remote_content begin - response = Gitlab::HTTP.get(location) + response = context.logger.instrument(:config_file_fetch_remote_content) do + Gitlab::HTTP.get(location) + end rescue SocketError errors.push("Remote file `#{masked_location}` could not be fetched because of a socket error!") rescue Timeout::Error diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 58b81b259cb..5fcf7c71bdf 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -52,7 +52,9 @@ module Gitlab end def fetch_template_content - Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content + context.logger.instrument(:config_file_fetch_template_content) do + Gitlab::Template::GitlabCiYmlTemplate.find(template_name, context.project)&.content + end end def masked_raw diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 97774bc5e13..19678def666 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -73,11 +73,7 @@ module Gitlab def key @key ||= begin - key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project) - Gitlab::CurrentSettings.ci_jwt_signing_key - else - Rails.application.secrets.openid_connect_signing_key - end + key_data = Gitlab::CurrentSettings.ci_jwt_signing_key raise NoSigningKeyError unless key_data diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 4460843545e..ee7733a081d 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -55,14 +55,8 @@ module Gitlab end def schema_path - # We can't exactly error out here pre-15.0. - # If the report itself doesn't specify the schema version, - # it will be considered invalid post-15.0 but for now we will - # validate against earliest supported version. - # https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_801479803 - # describes the indended behavior in detail - # TODO: After 15.0 - pass report_type and report_data here and - # error out if no version. + # The schema version selection logic here is described in the user documentation: + # https://docs.gitlab.com/ee/user/application_security/#security-report-validation report_declared_version = File.join(root_path, report_version, file_name) return report_declared_version if File.file?(report_declared_version) diff --git a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb index 17ebf56985b..af5cc7fe523 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/rate_limit.rb @@ -7,10 +7,9 @@ module Gitlab module Limit class RateLimit < Chain::Base include Chain::Helpers + include ::Gitlab::Utils::StrongMemoize def perform! - return unless throttle_enabled? - # We exclude child-pipelines from the rate limit because they represent # sub-pipelines that would otherwise hit the rate limit due to having the # same scope (project, user, sha). @@ -19,7 +18,7 @@ module Gitlab if rate_limit_throttled? create_log_entry - error(throttle_message) unless dry_run? + error(throttle_message) if enforce_throttle? end end @@ -43,7 +42,9 @@ module Gitlab commit_sha: command.sha, current_user_id: current_user.id, subscription_plan: project.actual_plan_name, - message: 'Activated pipeline creation rate limit' + message: 'Activated pipeline creation rate limit', + throttled: enforce_throttle?, + throttle_override: throttle_override? ) end @@ -51,16 +52,17 @@ module Gitlab 'Too many pipelines created in the last minute. Try again later.' end - def throttle_enabled? - ::Feature.enabled?( - :ci_throttle_pipelines_creation, - project) + def enforce_throttle? + strong_memoize(:enforce_throttle) do + ::Feature.enabled?(:ci_enforce_throttle_pipelines_creation, project) && + !throttle_override? + end end - def dry_run? - ::Feature.enabled?( - :ci_throttle_pipelines_creation_dry_run, - project) + def throttle_override? + strong_memoize(:throttle_override) do + ::Feature.enabled?(:ci_enforce_throttle_pipelines_creation_override, project) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 85bd5f0a7c1..8177502be1d 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -83,7 +83,9 @@ module Gitlab project: { id: project.id, path: project.full_path, - created_at: project.created_at&.iso8601 + created_at: project.created_at&.iso8601, + shared_runners_enabled: project.shared_runners_enabled?, + group_runners_enabled: project.group_runners_enabled? }, user: { id: current_user.id, diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_report.rb index 31afb636d2f..cebbb9ae842 100644 --- a/lib/gitlab/ci/reports/coverage_reports.rb +++ b/lib/gitlab/ci/reports/coverage_report.rb @@ -3,13 +3,17 @@ module Gitlab module Ci module Reports - class CoverageReports + class CoverageReport attr_reader :files def initialize @files = {} end + def empty? + @files.empty? + end + def pick(keys) coverage_files = files.select do |key| keys.include?(key) diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb new file mode 100644 index 00000000000..fd73ed6fd25 --- /dev/null +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CoverageReportGenerator + include Gitlab::Utils::StrongMemoize + + def initialize(pipeline) + @pipeline = pipeline + end + + def report + coverage_report = Gitlab::Ci::Reports::CoverageReport.new + + # Return an empty report if the pipeline is a child pipeline. + # Since the coverage report is used in a merge request report, + # we are only interested in the coverage report from the root pipeline. + return coverage_report if @pipeline.child? + + coverage_report.tap do |coverage_report| + report_builds.find_each do |build| + build.each_report(::Ci::JobArtifact::COVERAGE_REPORT_FILE_TYPES) do |file_type, blob| + Gitlab::Ci::Parsers.fabricate!(file_type).parse!( + blob, + coverage_report, + project_path: @pipeline.project.full_path, + worktree_paths: @pipeline.all_worktree_paths + ) + end + end + end + end + + private + + def report_builds + if child_pipeline_feature_enabled? + @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) + else + @pipeline.latest_report_builds(::Ci::JobArtifact.coverage_reports) + end + end + + def child_pipeline_feature_enabled? + strong_memoize(:feature_enabled) do + Feature.enabled?(:ci_child_pipeline_coverage_reports, @pipeline.project) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 46b41ed3c6c..0808290fe5b 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -20,15 +20,27 @@ module Gitlab return :invalid unless runner_version releases = RunnerReleases.instance.releases - parsed_runner_version = runner_version.is_a?(::Gitlab::VersionInfo) ? runner_version : ::Gitlab::VersionInfo.parse(runner_version) + orig_runner_version = runner_version + runner_version = ::Gitlab::VersionInfo.parse(runner_version) unless runner_version.is_a?(::Gitlab::VersionInfo) - raise ArgumentError, "'#{runner_version}' is not a valid version" unless parsed_runner_version.valid? + raise ArgumentError, "'#{orig_runner_version}' is not a valid version" unless runner_version.valid? - available_releases = releases.reject { |release| release > @gitlab_version } + gitlab_minor_version = version_without_patch(@gitlab_version) - return :recommended if available_releases.any? { |available_release| patch_update?(available_release, parsed_runner_version) } - return :recommended if outside_backport_window?(parsed_runner_version, releases) - return :available if available_releases.any? { |available_release| available_release > parsed_runner_version } + available_releases = releases + .reject { |release| release.major > @gitlab_version.major } + .reject do |release| + release_minor_version = version_without_patch(release) + + # Do not reject a patch update, even if the runner is ahead of the instance version + next false if version_without_patch(runner_version) == release_minor_version + + release_minor_version > gitlab_minor_version + end + + return :recommended if available_releases.any? { |available_rel| patch_update?(available_rel, runner_version) } + return :recommended if outside_backport_window?(runner_version, releases) + return :available if available_releases.any? { |available_rel| available_rel > runner_version } :not_available end diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 856a097e6e0..8886929646d 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -9,7 +9,7 @@ image: "crystallang/crystal:latest" # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Check out: https://docs.gitlab.com/ee/ci/services/index.html # services: # - mysql:latest # - redis:latest diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index c1815baf7e6..ab4c9b701d0 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -11,7 +11,7 @@ # # -------------------- # -# Documentation: https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs +# Documentation: https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-ecs stages: - build @@ -23,5 +23,5 @@ stages: "error: Template has moved": stage: deploy script: - - echo "Deploy-ECS.gitlab-ci.yml has been moved to AWS/Deploy-ECS.gitlab-ci.yml, see https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs for more details." + - echo "Deploy-ECS.gitlab-ci.yml has been moved to AWS/Deploy-ECS.gitlab-ci.yml, see https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-ecs for more details." - exit 1 diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index 426076c84a1..acc4a9d2917 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -41,7 +41,7 @@ default: # # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. - # Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service + # Check out: https://docs.gitlab.com/ee/ci/services/index.html services: - mysql:8.0 # diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 1ceaf9fc86b..1eb920c7747 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -7,7 +7,7 @@ image: elixir:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Check out: https://docs.gitlab.com/ee/ci/services/index.html services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 6a6fc2cb702..8f1124373c4 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.28.2' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 98c4216679f..f9c0d4333ff 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.28.2' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 603be5b1cdb..36f1b6981c4 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.28.2' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.30.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index ff7bac15017..0ec67526234 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -9,7 +9,7 @@ image: php:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Check out: https://docs.gitlab.com/ee/ci/services/index.html services: - mysql:latest diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 16bc0026aa8..44370f896a7 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -9,7 +9,7 @@ image: node:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Check out: https://docs.gitlab.com/ee/ci/services/index.html services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 281bf7e3dd9..4edc003a638 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -23,7 +23,7 @@ before_script: - curl -sS https://getcomposer.org/installer | php - php composer.phar install -# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Bring in any services we need https://docs.gitlab.com/ee/ci/services/index.html # See http://docs.gitlab.com/ee/ci/services/README.html for examples. services: - mysql:5.7 diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 44f959468a8..690a5a291e1 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -9,7 +9,7 @@ image: ruby:latest # Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Check out: https://docs.gitlab.com/ee/ci/services/index.html services: - mysql:latest - redis:latest diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index 869c1782352..390f0bb8061 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -9,7 +9,7 @@ image: "rust:latest" # Optional: Pick zero or more services to be used on all builds. # Only needed when using a docker container to run your tests in. -# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service +# Check out: https://docs.gitlab.com/ee/ci/services/index.html # services: # - mysql:latest # - redis:latest diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index f7f016b5e57..d4b6a252b25 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -12,8 +12,8 @@ variables: # Which branch we want to run full fledged long running fuzzing jobs. # All others will run fuzzing regression COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH" - # This is using semantic version and will always download latest v2 gitlab-cov-fuzz release - COVFUZZ_VERSION: v2 + # This is using semantic version and will always download latest v3 gitlab-cov-fuzz release + COVFUZZ_VERSION: v3 # This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries # to their own servers COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw" diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 3f9c87b7abf..4a72f5e72b1 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -1,3 +1,8 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml + stages: - build - test @@ -6,12 +11,13 @@ stages: variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" - DAST_API_VERSION: "1" - DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION + DAST_API_VERSION: "2" + DAST_API_IMAGE_SUFFIX: "" + DAST_API_IMAGE: api-security dast: stage: dast - image: $DAST_API_IMAGE + image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true script: - /peach/analyzer-dast-api diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index e5ac5099546..10549b56856 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -48,13 +48,10 @@ dast: $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED && $DAST_WEBSITE == null && - $DAST_API_SPECIFICATION == null + $REVIEW_DISABLED when: never - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ - if: $CI_COMMIT_BRANCH && - $DAST_WEBSITE - - if: $CI_COMMIT_BRANCH && - $DAST_API_SPECIFICATION + $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index b34bfe2a53c..c414e70bfa3 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -20,7 +20,7 @@ variables: SECURE_BINARIES_ANALYZERS: >- bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, gemnasium, gemnasium-maven, gemnasium-python, license-finder, - dast, dast-runner-validation, api-fuzzing + dast, dast-runner-validation, api-security SECURE_BINARIES_DOWNLOAD_IMAGES: "true" SECURE_BINARIES_PUSH_IMAGES: "true" @@ -252,11 +252,11 @@ dast-runner-validation: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bdast-runner-validation\b/ -api-fuzzing: +api-security: extends: .download_images variables: - SECURE_BINARIES_ANALYZER_VERSION: "1" + SECURE_BINARIES_ANALYZER_VERSION: "2" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && - $SECURE_BINARIES_ANALYZERS =~ /\bapi-fuzzing\b/ + $SECURE_BINARIES_ANALYZERS =~ /\bapi-security\b/ diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 56151a6bcdf..4d0259fe678 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -1,7 +1,7 @@ # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml include: - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 49bdd4b7713..6f9a9c5133c 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -4,7 +4,7 @@ # they are able to only include the jobs that they find interesting. # # Therefore, this template is not supposed to run any jobs. The idea is to only -# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/jobs/#hide-jobs # # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index e93bd75a9fa..95a60b852b8 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -74,14 +74,14 @@ module Gitlab end def exist? - archived? || live_trace_exist? + archived? || live? end def archived? trace_artifact&.stored? end - def live_trace_exist? + def live? job.trace_chunks.any? || current_path.present? || old_trace.present? end diff --git a/lib/gitlab/ci/trace/archive.rb b/lib/gitlab/ci/trace/archive.rb index d4a451ca526..0cd8df2e2af 100644 --- a/lib/gitlab/ci/trace/archive.rb +++ b/lib/gitlab/ci/trace/archive.rb @@ -15,7 +15,7 @@ module Gitlab def execute!(stream) clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| - md5_checksum = self.class.md5_hexdigest(clone_path) + md5_checksum = self.class.md5_hexdigest(clone_path) unless Gitlab::FIPS.enabled? sha256_checksum = self.class.sha256_hexdigest(clone_path) job.transaction do @@ -24,7 +24,7 @@ module Gitlab end end - validate_archived_trace + validate_archived_trace unless Gitlab::FIPS.enabled? end private diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb index 6ce7046262b..40418a4c797 100644 --- a/lib/gitlab/config/entry/node.rb +++ b/lib/gitlab/config/entry/node.rb @@ -106,6 +106,10 @@ module Gitlab @config.is_a?(Hash) end + def array? + @config.is_a?(Array) + end + def string? @config.is_a?(String) end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 521dec110a8..574a7dceaa4 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -44,6 +44,7 @@ module Gitlab allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn allow_framed_gitlab_paths(directives) allow_customersdot(directives) if ENV['CUSTOMER_PORTAL_URL'].present? + allow_review_apps(directives) if ENV['REVIEW_APPS_ENABLED'] # The follow section contains workarounds to patch Safari's lack of support for CSP Level 3 # See https://gitlab.com/gitlab-org/gitlab/-/issues/343579 @@ -154,6 +155,11 @@ module Gitlab append_to_directive(directives, 'frame_src', Gitlab::Utils.append_path(Gitlab.config.gitlab.url, path)) end end + + def self.allow_review_apps(directives) + # Allow-listed to allow POSTs to https://gitlab.com/api/v4/projects/278964/merge_requests/:merge_request_iid/visual_review_discussions + append_to_directive(directives, 'connect_src', 'https://gitlab.com/api/v4/projects/278964/merge_requests/') + end end end end diff --git a/lib/gitlab/daemon.rb b/lib/gitlab/daemon.rb index 5f579f90629..04d13778499 100644 --- a/lib/gitlab/daemon.rb +++ b/lib/gitlab/daemon.rb @@ -2,10 +2,20 @@ module Gitlab class Daemon - def self.initialize_instance(...) - raise "#{name} singleton instance already initialized" if @instance + # Options: + # - recreate: We usually only allow a single instance per process to exist; + # this can be overridden with this switch, so that existing + # instances are stopped and recreated. + def self.initialize_instance(*args, recreate: false, **options) + if @instance + if recreate + @instance.stop + else + raise "#{name} singleton instance already initialized" + end + end - @instance = new(...) + @instance = new(*args, **options) Kernel.at_exit(&@instance.method(:stop)) @instance end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 385f1e57705..c13bb1d6a9a 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -12,7 +12,7 @@ module Gitlab def initialize(pipeline) @pipeline = pipeline - super( + attrs = { object_kind: 'pipeline', object_attributes: hook_attrs(pipeline), merge_request: pipeline.merge_request && merge_request_attrs(pipeline.merge_request), @@ -23,7 +23,13 @@ module Gitlab preload_builds(pipeline, :latest_builds) pipeline.latest_builds.map(&method(:build_hook_attrs)) end - ) + } + + if pipeline.source_pipeline.present? + attrs[:source_pipeline] = source_pipeline_attrs(pipeline.source_pipeline) + end + + super(attrs) end def with_retried_builds @@ -72,6 +78,20 @@ module Gitlab } end + def source_pipeline_attrs(source_pipeline) + project = source_pipeline.source_project + + { + project: { + id: project.id, + web_url: project.web_url, + path_with_namespace: project.full_path + }, + job_id: source_pipeline.source_job_id, + pipeline_id: source_pipeline.source_pipeline_id + } + end + def merge_request_attrs(merge_request) { id: merge_request.id, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 677b4485288..b42d164d9c4 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -68,7 +68,8 @@ module Gitlab @schemas_to_base_models ||= { gitlab_main: [self.database_base_models.fetch(:main)], gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main - gitlab_shared: self.database_base_models.values # all models + gitlab_shared: self.database_base_models.values, # all models + gitlab_internal: self.database_base_models.values # all models }.with_indifferent_access.freeze end @@ -203,8 +204,13 @@ module Gitlab # This does not look at literal connection names, but rather compares # models that are holders for a given db_config_name def self.gitlab_schemas_for_connection(connection) - db_name = self.db_config_name(connection) - primary_model = self.database_base_models.fetch(db_name.to_sym) + db_config = self.db_config_for_connection(connection) + + # connection might not be yet adopted (returning NullPool, and no connection_klass) + # in such cases it is fine to ignore such connections + return unless db_config + + primary_model = self.database_base_models.fetch(db_config.name.to_sym) self.schemas_to_base_models.select do |_, child_models| child_models.any? do |child_model| @@ -218,8 +224,11 @@ module Gitlab def self.db_config_for_connection(connection) return unless connection + # For a ConnectionProxy we want to avoid ambiguous db_config as it may + # sometimes default to replica so we always return the primary config + # instead. if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) - return connection.load_balancer.configuration.primary_db_config + return connection.load_balancer.configuration.db_config end # During application init we might receive `NullPool` diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index a90cae7aea2..d052d5adc4c 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -30,9 +30,23 @@ module Gitlab scope :created_after, ->(time) { where('created_at > ?', time) } - scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do - where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) + scope :for_configuration, ->(gitlab_schema, job_class_name, table_name, column_name, job_arguments) do + relation = where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals + + # This method is called from migrations older than the gitlab_schema column, + # check and add this filter only if the column exists. + relation = relation.for_gitlab_schema(gitlab_schema) if gitlab_schema_column_exists? + + relation + end + + def self.gitlab_schema_column_exists? + column_names.include?('gitlab_schema') + end + + scope :for_gitlab_schema, ->(gitlab_schema) do + where(gitlab_schema: gitlab_schema) end state_machine :status, initial: :paused do @@ -73,12 +87,13 @@ module Gitlab state_machine.states.map(&:name) end - def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) - for_configuration(job_class_name, table_name, column_name, job_arguments).first + def self.find_for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments) + for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments).first end - def self.active_migration - executable.queue_order.first + def self.active_migration(connection:) + for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) + .executable.queue_order.first end def self.successful_rows_counts(migrations) diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 59ff9a9744f..388eb596ce2 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -54,7 +54,10 @@ module Gitlab # in order to prevent it being picked up by the background worker. Perform all pending jobs, # then keep running until migration is finished. def finalize(job_class_name, table_name, column_name, job_arguments) - migration = BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + migration = BatchedMigration.find_for_configuration( + Gitlab::Database.gitlab_schemas_for_connection(connection), + job_class_name, table_name, column_name, job_arguments + ) configuration = { job_class_name: job_class_name, diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 49f56b5be97..92a41bb36ee 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -1,7 +1,11 @@ # frozen_string_literal: true # For large tables, PostgreSQL can take a long time to count rows due to MVCC. -# Implements a distinct and ordinary batch counter +# Implements: +# - distinct batch counter +# - ordinary batch counter +# - sum batch counter +# - average batch counter # Needs indexes on the column below to calculate max, min and range queries # For larger tables just set use higher batch_size with index optimization # @@ -22,6 +26,8 @@ # batch_distinct_count(Project.group(:visibility_level), :creator_id) # batch_sum(User, :sign_in_count) # batch_sum(Issue.group(:state_id), :weight)) +# batch_average(Ci::Pipeline, :duration) +# batch_average(MergeTrain.group(:status), :duration) module Gitlab module Database module BatchCount @@ -37,6 +43,10 @@ module Gitlab BatchCounter.new(relation, column: nil, operation: :sum, operation_args: [column]).count(batch_size: batch_size, start: start, finish: finish) end + def batch_average(relation, column, batch_size: nil, start: nil, finish: nil) + BatchCounter.new(relation, column: nil, operation: :average, operation_args: [column]).count(batch_size: batch_size, start: start, finish: finish) + end + class << self include BatchCount end diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 417511618e4..522b598cd9d 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -6,6 +6,7 @@ module Gitlab FALLBACK = -1 MIN_REQUIRED_BATCH_SIZE = 1_250 DEFAULT_SUM_BATCH_SIZE = 1_000 + DEFAULT_AVERAGE_BATCH_SIZE = 1_000 MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep ALLOWED_MODES = [:itself, :distinct].freeze @@ -26,6 +27,7 @@ module Gitlab def unwanted_configuration?(finish, batch_size, start) (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || + (@operation == :average && batch_size < DEFAULT_AVERAGE_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || start >= finish end @@ -92,6 +94,7 @@ module Gitlab def batch_size_for_mode_and_operation(mode, operation) return DEFAULT_SUM_BATCH_SIZE if operation == :sum + return DEFAULT_AVERAGE_BATCH_SIZE if operation == :average mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE end diff --git a/lib/gitlab/database/consistency_checker.rb b/lib/gitlab/database/consistency_checker.rb index e398fef744c..bf60c76a085 100644 --- a/lib/gitlab/database/consistency_checker.rb +++ b/lib/gitlab/database/consistency_checker.rb @@ -3,9 +3,9 @@ module Gitlab module Database class ConsistencyChecker - BATCH_SIZE = 1000 - MAX_BATCHES = 25 - MAX_RUNTIME = 30.seconds # must be less than the scheduling frequency of the ConsistencyCheck jobs + BATCH_SIZE = 500 + MAX_BATCHES = 20 + MAX_RUNTIME = 5.seconds # must be less than the scheduling frequency of the ConsistencyCheck jobs delegate :monotonic_time, to: :'Gitlab::Metrics::System' diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index d7ea614e2af..baf4cc48424 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -75,8 +75,8 @@ module Gitlab return gitlab_schema end - # All tables from `information_schema.` are `:gitlab_shared` - return :gitlab_shared if schema_name == 'information_schema' + # All tables from `information_schema.` are marked as `internal` + return :gitlab_internal if schema_name == 'information_schema' return :gitlab_main if table_name.start_with?('_test_gitlab_main_') @@ -85,8 +85,8 @@ module Gitlab # All tables that start with `_test_` without a following schema are shared and ignored return :gitlab_shared if table_name.start_with?('_test_') - # All `pg_` tables are marked as `shared` - return :gitlab_shared if table_name.start_with?('pg_') + # All `pg_` tables are marked as `internal` + return :gitlab_internal if table_name.start_with?('pg_') # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database :"undefined_#{table_name}" diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 036ce7d7631..71c323cb393 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -35,10 +35,11 @@ approval_project_rules_users: :gitlab_main approvals: :gitlab_main approver_groups: :gitlab_main approvers: :gitlab_main -ar_internal_metadata: :gitlab_shared +ar_internal_metadata: :gitlab_internal atlassian_identities: :gitlab_main audit_events_external_audit_event_destinations: :gitlab_main audit_events: :gitlab_main +audit_events_streaming_headers: :gitlab_main authentication_events: :gitlab_main award_emoji: :gitlab_main aws_roles: :gitlab_main @@ -121,6 +122,7 @@ ci_unit_tests: :gitlab_ci ci_variables: :gitlab_ci cluster_agents: :gitlab_main cluster_agent_tokens: :gitlab_main +cluster_enabled_grants: :gitlab_main cluster_groups: :gitlab_main cluster_platforms_kubernetes: :gitlab_main cluster_projects: :gitlab_main @@ -217,7 +219,6 @@ geo_event_log: :gitlab_main geo_events: :gitlab_main geo_hashed_storage_attachments_events: :gitlab_main geo_hashed_storage_migrated_events: :gitlab_main -geo_lfs_object_deleted_events: :gitlab_main geo_node_namespace_links: :gitlab_main geo_nodes: :gitlab_main geo_node_statuses: :gitlab_main @@ -266,6 +267,7 @@ integrations: :gitlab_main internal_ids: :gitlab_main ip_restrictions: :gitlab_main issuable_metric_images: :gitlab_main +issuable_resource_links: :gitlab_main issuable_severities: :gitlab_main issuable_slas: :gitlab_main issue_assignees: :gitlab_main @@ -465,7 +467,7 @@ routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main saved_replies: :gitlab_main -schema_migrations: :gitlab_shared +schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main security_findings: :gitlab_main @@ -491,6 +493,7 @@ software_license_policies: :gitlab_main software_licenses: :gitlab_main spam_logs: :gitlab_main sprints: :gitlab_main +ssh_signatures: :gitlab_main status_check_responses: :gitlab_main status_page_published_incidents: :gitlab_main status_page_settings: :gitlab_main @@ -503,6 +506,7 @@ term_agreements: :gitlab_main terraform_states: :gitlab_main terraform_state_versions: :gitlab_main timelogs: :gitlab_main +timelog_categories: :gitlab_main todos: :gitlab_main token_with_ivs: :gitlab_main topics: :gitlab_main @@ -549,6 +553,7 @@ vulnerability_occurrences: :gitlab_main vulnerability_reads: :gitlab_main vulnerability_remediations: :gitlab_main vulnerability_scanners: :gitlab_main +vulnerability_state_transitions: :gitlab_main vulnerability_statistics: :gitlab_main vulnerability_user_mentions: :gitlab_main webauthn_registrations: :gitlab_main @@ -556,10 +561,13 @@ web_hook_logs: :gitlab_main web_hooks: :gitlab_main wiki_page_meta: :gitlab_main wiki_page_slugs: :gitlab_main +work_item_parent_links: :gitlab_main work_item_types: :gitlab_main x509_certificates: :gitlab_main x509_commit_signatures: :gitlab_main x509_issuers: :gitlab_main zentao_tracker_data: :gitlab_main +# dingtalk_tracker_data JiHu-specific, see https://jihulab.com/gitlab-cn/gitlab/-/merge_requests/417 +dingtalk_tracker_data: :gitlab_main zoom_meetings: :gitlab_main batched_background_migration_job_transition_logs: :gitlab_shared diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 0ddc745ebae..59b08fac7e9 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -41,8 +41,6 @@ module Gitlab end end - config.reuse_primary_connection! - config end @@ -61,44 +59,17 @@ module Gitlab disconnect_timeout: 120, use_tcp: false } - - # Temporary model for GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ - # To be removed with FF - @primary_model = nil end def db_config_name @model.connection_db_config.name.to_sym end - # With connection re-use the primary connection can be overwritten - # to be used from different model - def primary_connection_specification_name - primary_model_or_model_if_enabled.connection_specification_name - end - - def primary_model_or_model_if_enabled - if use_dedicated_connection? - @model - else - @primary_model || @model - end - end - - def use_dedicated_connection? - return true unless @primary_model # We can only use dedicated connection, if re-use of connections is disabled - return false unless ::Gitlab::SafeRequestStore.active? - - ::Gitlab::SafeRequestStore.fetch(:force_no_sharing_primary_model) do - ::Feature::FlipperFeature.table_exists? && ::Feature.enabled?(:force_no_sharing_primary_model) - end - end - - def primary_db_config - primary_model_or_model_if_enabled.connection_db_config + def connection_specification_name + @model.connection_specification_name end - def replica_db_config + def db_config @model.connection_db_config end @@ -131,30 +102,6 @@ module Gitlab service_discovery[:record].present? end - - # TODO: This is temporary code to allow re-use of primary connection - # if the two connections are pointing to the same host. This is needed - # to properly support transaction visibility. - # - # This behavior is required to support [Phase 3](https://gitlab.com/groups/gitlab-org/-/epics/6160#progress). - # This method is meant to be removed as soon as it is finished. - # - # The remapping is done as-is: - # export GITLAB_LOAD_BALANCING_REUSE_PRIMARY_<name-of-connection>=<new-name-of-connection> - # - # Ex.: - # export GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci=main - # - def reuse_primary_connection! - new_connection = ENV["GITLAB_LOAD_BALANCING_REUSE_PRIMARY_#{db_config_name}"] - return unless new_connection.present? - - @primary_model = Gitlab::Database.database_base_models[new_connection.to_sym] - - unless @primary_model - raise "Invalid value for 'GITLAB_LOAD_BALANCING_REUSE_PRIMARY_#{db_config_name}=#{new_connection}'" - end - end end end end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 1be63da8896..8799f8d8af8 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -23,6 +23,7 @@ module Gitlab insert update update_all + exec_insert_all ).freeze NON_STICKY_READS = %i( diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 191ebe18b8a..40b76a1c028 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -104,12 +104,24 @@ module Gitlab # Yields a connection that can be used for both reads and writes. def read_write connection = nil + transaction_open = nil # In the event of a failover the primary may be briefly unavailable. # Instead of immediately grinding to a halt we'll retry the operation # a few times. retry_with_backoff do connection = pool.connection + transaction_open = connection.transaction_open? + yield connection + rescue StandardError => e + if transaction_open && connection_error?(e) + ::Gitlab::Database::LoadBalancing::Logger.warn( + event: :transaction_leak, + message: 'A write transaction has leaked during database fail-over' + ) + end + + raise e end end @@ -232,14 +244,14 @@ module Gitlab # host - An optional host name to use instead of the default one. # port - An optional port to connect to. def create_replica_connection_pool(pool_size, host = nil, port = nil) - db_config = @configuration.replica_db_config + db_config = @configuration.db_config env_config = db_config.configuration_hash.dup env_config[:pool] = pool_size env_config[:host] = host if host env_config[:port] = port if port - replica_db_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( db_config.env_name, db_config.name + REPLICA_SUFFIX, env_config @@ -249,7 +261,7 @@ module Gitlab # as it will rewrite ActiveRecord::Base.connection ActiveRecord::ConnectionAdapters::ConnectionHandler .new - .establish_connection(replica_db_config) + .establish_connection(db_config) end # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching, @@ -258,7 +270,7 @@ module Gitlab # rubocop:disable Database/MultipleDatabases def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( - @configuration.primary_connection_specification_name, + @configuration.connection_specification_name, role: ActiveRecord::Base.writing_role, shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 62dfe75a851..13afbd8fd37 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -39,18 +39,31 @@ module Gitlab end job['wal_locations'] = locations + job['wal_location_source'] = wal_location_source + end + + def wal_location_source + if ::Gitlab::Database::LoadBalancing.primary_only? || uses_primary? + ::Gitlab::Database::LoadBalancing::ROLE_PRIMARY + else + ::Gitlab::Database::LoadBalancing::ROLE_REPLICA + end end def wal_location_for(load_balancer) # When only using the primary there's no need for any WAL queries. return if load_balancer.primary_only? - if ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + if uses_primary? load_balancer.primary_write_location else load_balancer.host.database_replica_location end end + + def uses_primary? + ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + end end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index 038af570dbc..ab8b6988c3d 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -37,6 +37,7 @@ module Gitlab class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase include LockRetriesConcern include Gitlab::Database::MigrationHelpers::V2 + include Gitlab::Database::MigrationHelpers::AnnounceDatabase # When running migrations, the `db:migrate` switches connection of # ActiveRecord::Base depending where the migration runs. diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0453b81d67d..4bb1d71ce18 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -945,8 +945,13 @@ module Gitlab end def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) - migration = Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(job_class_name, table_name, column_name, job_arguments).first + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( + Gitlab::Database.gitlab_schemas_for_connection(connection), + job_class_name, table_name, column_name, job_arguments + ) configuration = { job_class_name: job_class_name, @@ -966,7 +971,7 @@ module Gitlab "but it is '#{migration.status_name}':" \ "\t#{configuration}" \ "\n\n" \ - "Finalize it manually by running" \ + "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ "\n\n" \ "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ "\n\n" \ @@ -1494,6 +1499,20 @@ into similar problems in the future (e.g. when new tables are created). SQL end + def drop_sequence(table_name, column_name, sequence_name) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} ALTER COLUMN #{quote_column_name(column_name)} DROP DEFAULT; + DROP SEQUENCE IF EXISTS #{quote_table_name(sequence_name)} + SQL + end + + def add_sequence(table_name, column_name, sequence_name, start_value) + execute <<~SQL + CREATE SEQUENCE #{quote_table_name(sequence_name)} START #{start_value}; + ALTER TABLE #{quote_table_name(table_name)} ALTER COLUMN #{quote_column_name(column_name)} SET DEFAULT nextval(#{quote(sequence_name)}) + SQL + end + private def create_temporary_columns_and_triggers(table, columns, primary_key: :id, data_type: :bigint) diff --git a/lib/gitlab/database/migration_helpers/announce_database.rb b/lib/gitlab/database/migration_helpers/announce_database.rb new file mode 100644 index 00000000000..28710aab717 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/announce_database.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module AnnounceDatabase + extend ActiveSupport::Concern + + def write(text = "") + if text.present? # announce/say + super("#{db_config_name}: #{text}") + else + super(text) + end + end + + def db_config_name + @db_config_name ||= Gitlab::Database.db_config_name(connection) + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb index d8d07fcaf2d..b8d1d21a0d2 100644 --- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -21,7 +21,7 @@ module Gitlab end end - def migrate(direction) + def exec_migration(conn, direction) if unmatched_schemas.any? migration_skipped return @@ -37,8 +37,9 @@ module Gitlab private def migration_skipped - say "Current migration is skipped since it modifies "\ - "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + say "The migration is skipped since it modifies the schemas: #{self.class.allowed_gitlab_schemas}." + say "This database can only apply migrations in one of the following schemas: " \ + "#{allowed_schemas_for_connection}." end def validator_class diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 7113c3686f1..4aaeaa7b365 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -67,10 +67,22 @@ module Gitlab batch_class_name: BATCH_CLASS_NAME, batch_size: BATCH_SIZE, max_batch_size: nil, - sub_batch_size: SUB_BATCH_SIZE + sub_batch_size: SUB_BATCH_SIZE, + gitlab_schema: nil ) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? + if transaction_open? + raise 'The `queue_batched_background_migration` cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ + 'your migration class.' + end + + gitlab_schema ||= gitlab_schema_from_context + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(gitlab_schema, job_class_name, batch_table_name, batch_column_name, job_arguments).exists? Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ "job_arguments: #{job_arguments.inspect}" @@ -119,24 +131,77 @@ module Gitlab end end + if migration.respond_to?(:gitlab_schema) + migration.gitlab_schema = gitlab_schema + end + migration.save! migration end def finalize_batched_background_migration(job_class_name:, table_name:, column_name:, job_arguments:) - database_name = Gitlab::Database.db_config_name(connection) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - unless ActiveRecord::Base.configurations.primary?(database_name) - raise 'The `#finalize_background_migration` is currently not supported when running in decomposed database, ' \ - 'and this database is not `main:`. For more information visit: ' \ - 'https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html' + if transaction_open? + raise 'The `finalize_batched_background_migration` cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ + 'your migration class.' end - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( + gitlab_schema_from_context, job_class_name, table_name, column_name, job_arguments) raise 'Could not find batched background migration' if migration.nil? - Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(job_class_name, table_name, column_name, job_arguments, connection: connection) + with_restored_connection_stack do |restored_connection| + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize( + job_class_name, table_name, + column_name, job_arguments, + connection: restored_connection) + end + end + end + + # Deletes batched background migration for the given configuration. + # + # job_class_name - The background migration job class as a string + # table_name - The name of the table the migration iterates over + # column_name - The name of the column the migration will batch over + # job_arguments - Migration arguments + # + # Example: + # + # delete_batched_background_migration( + # 'CopyColumnUsingBackgroundMigrationJob', + # :events, + # :id, + # ['column1', 'column2']) + def delete_batched_background_migration(job_class_name, table_name, column_name, job_arguments) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + + if transaction_open? + raise 'The `#delete_batched_background_migration` cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ + 'your migration class.' + end + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration( + gitlab_schema_from_context, job_class_name, table_name, column_name, job_arguments + ).delete_all + end + + def gitlab_schema_from_context + if respond_to?(:allowed_gitlab_schemas) # Gitlab::Database::Migration::V2_0 + Array(allowed_gitlab_schemas).first + else # Gitlab::Database::Migration::V1_0 + :gitlab_main + end end end end diff --git a/lib/gitlab/database/migrations/reestablished_connection_stack.rb b/lib/gitlab/database/migrations/reestablished_connection_stack.rb index d7cf482c32a..addc9d874af 100644 --- a/lib/gitlab/database/migrations/reestablished_connection_stack.rb +++ b/lib/gitlab/database/migrations/reestablished_connection_stack.rb @@ -17,7 +17,9 @@ module Gitlab original_handler = ActiveRecord::Base.connection_handler original_db_config = ActiveRecord::Base.connection_db_config - return yield if ActiveRecord::Base.configurations.primary?(original_db_config.name) + if ActiveRecord::Base.configurations.primary?(original_db_config.name) + return yield(ActiveRecord::Base.connection) + end # If the `ActiveRecord::Base` connection is different than `:main` # re-establish and configure `SharedModel` context accordingly @@ -43,7 +45,7 @@ module Gitlab ActiveRecord::Base.establish_connection :main # rubocop:disable Database/EstablishConnection Gitlab::Database::SharedModel.using_connection(base_model.connection) do - yield + yield(base_model.connection) end ensure ActiveRecord::Base.connection_handler = original_handler diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 9c8cccb3dc6..0f08a47d754 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -40,6 +40,10 @@ module Gitlab # No-op, required by the partition manager end + def validate_and_fix + # No-op, required by the partition manager + end + private def desired_partitions diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 3ee9a193b45..aac91eaadb1 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -22,15 +22,13 @@ module Gitlab connection_name: @connection_name ) - # Double-checking before getting the lease: - # The prevailing situation is no missing partitions and no extra partitions - return if missing_partitions.empty? && extra_partitions.empty? - only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do - partitions_to_create = missing_partitions - create(partitions_to_create) unless partitions_to_create.empty? + model.partitioning_strategy.validate_and_fix + partitions_to_create = missing_partitions partitions_to_detach = extra_partitions + + create(partitions_to_create) unless partitions_to_create.empty? detach(partitions_to_detach) unless partitions_to_detach.empty? end rescue StandardError => e @@ -119,7 +117,8 @@ module Gitlab parent_table_identifier = "#{connection.current_schema}.#{partition.table}" if (example_fk = PostgresForeignKey.by_referenced_table_identifier(parent_table_identifier).first) - raise UnsafeToDetachPartitionError, "Cannot detach #{partition.partition_name}, it would block while checking foreign key #{example_fk.name} on #{example_fk.constrained_table_identifier}" + raise UnsafeToDetachPartitionError, "Cannot detach #{partition.partition_name}, it would block while " \ + "checking foreign key #{example_fk.name} on #{example_fk.constrained_table_identifier}" end end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index e9865fb91d6..5cf32d3272c 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -48,9 +48,12 @@ module Gitlab default_value = current_default_value if extra.any? { |p| p.value == default_value } - Gitlab::AppLogger.error(message: "Inconsistent partition detected: partition with value #{current_default_value} should not be deleted because it's used as the default value.", - partition_number: current_default_value, - table_name: model.table_name) + Gitlab::AppLogger.error( + message: "Inconsistent partition detected: partition with value #{current_default_value} should " \ + "not be deleted because it's used as the default value.", + partition_number: current_default_value, + table_name: model.table_name + ) extra = extra.reject { |p| p.value == default_value } end @@ -73,6 +76,42 @@ module Gitlab current_partitions.empty? end + def validate_and_fix + return unless Feature.enabled?(:fix_sliding_list_partitioning) + return if no_partitions_exist? + + old_default_value = current_default_value + expected_default_value = active_partition.value + + if old_default_value != expected_default_value + with_lock_retries do + model.connection.execute("LOCK TABLE #{model.table_name} IN ACCESS EXCLUSIVE MODE") + + old_default_value = current_default_value + expected_default_value = active_partition.value + + if old_default_value == expected_default_value + Gitlab::AppLogger.warn( + message: "Table partitions or partition key default value have been changed by another process", + table_name: table_name, + default_value: expected_default_value + ) + raise ActiveRecord::Rollback + end + + model.connection.change_column_default(model.table_name, partitioning_key, expected_default_value) + Gitlab::AppLogger.warn( + message: "Fixed default value of sliding_list_strategy partitioning_key", + column: partitioning_key, + table_name: table_name, + connection_name: model.connection.pool.db_config.name, + old_value: old_default_value, + new_value: expected_default_value + ) + end + end + end + private def current_default_value @@ -95,6 +134,14 @@ module Gitlab raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" end end + + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new( + klass: self.class, + logger: Gitlab::AppLogger, + connection: model.connection + ).run(&block) + end end end end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index 391375d472f..06e2b114c91 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -27,15 +27,9 @@ module Gitlab # to reduce amount of labels sort schemas used gitlab_schemas = gitlab_schemas.to_a.sort.join(",") - # Temporary feature to observe relation of `gitlab_schemas` to `db_config_name` - # depending on primary model - ci_dedicated_primary_connection = ::Ci::ApplicationRecord.connection_class? && - ::Ci::ApplicationRecord.load_balancer.configuration.use_dedicated_connection? - schemas_metrics.increment({ gitlab_schemas: gitlab_schemas, - db_config_name: db_config_name, - ci_dedicated_primary_connection: ci_dedicated_primary_connection + db_config_name: db_config_name }) end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index 3f0176cb654..c51282c9a55 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -9,7 +9,7 @@ module Gitlab DMLNotAllowedError = Class.new(UnsupportedSchemaError) DMLAccessDeniedError = Class.new(UnsupportedSchemaError) - IGNORED_SCHEMAS = %i[gitlab_shared].freeze + IGNORED_SCHEMAS = %i[gitlab_shared gitlab_internal].freeze class << self def enabled? diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index f4c8fca8fa2..877866b9b23 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -20,6 +20,15 @@ module Gitlab "to '#{Gitlab::Database.db_config_name(connection)}'" end + # connection might not be yet adopted (returning nil, and no gitlab_schemas) + # in such cases it is fine to ignore such connections + gitlab_schemas = Gitlab::Database.gitlab_schemas_for_connection(connection) + + unless gitlab_schemas.nil? || gitlab_schemas.include?(:gitlab_shared) + raise "Cannot set `SharedModel` to connection from `#{Gitlab::Database.db_config_name(connection)}` " \ + "since this connection does not include `:gitlab_shared` schema." + end + self.overriding_connection = connection yield diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb index 111ea697ec2..ffd057e1d33 100644 --- a/lib/gitlab/devise_failure.rb +++ b/lib/gitlab/devise_failure.rb @@ -9,3 +9,5 @@ module Gitlab end end end + +Gitlab::DeviseFailure.prepend_mod diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb deleted file mode 100644 index 860f87a28a3..00000000000 --- a/lib/gitlab/diff/custom_diff.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module Diff - module CustomDiff - RENDERED_TIMEOUT_BACKGROUND = 20.seconds - RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds - BACKGROUND_EXECUTION = 'background' - FOREGROUND_EXECUTION = 'foreground' - LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' - LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT' - LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID' - - class << self - def preprocess_before_diff(path, old_blob, new_blob) - return unless path.ends_with? '.ipynb' - - Timeout.timeout(timeout_time) do - transformed_diff(old_blob&.data, new_blob&.data)&.tap do - transformed_for_diff(new_blob, old_blob) - log_event(LOG_IPYNBDIFF_GENERATED) - end - end - rescue Timeout::Error => e - rendered_timeout.increment(source: execution_source) - log_event(LOG_IPYNBDIFF_TIMEOUT, e) - rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e - log_event(LOG_IPYNBDIFF_INVALID, e) - end - - def transformed_diff(before, after) - transformed_diff = IpynbDiff.diff(before, after, - raise_if_invalid_nb: true, - diffy_opts: { include_diff_info: true }).to_s(:text) - - strip_diff_frontmatter(transformed_diff) - end - - def transformed_blob_language(blob) - 'md' if transformed_for_diff?(blob) - end - - def transformed_blob_data(blob) - if transformed_for_diff?(blob) - IpynbDiff.transform(blob.data, raise_errors: true, include_frontmatter: false) - end - end - - def strip_diff_frontmatter(diff_content) - diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? - end - - def blobs_with_transformed_diffs - @blobs_with_transformed_diffs ||= {} - end - - def transformed_for_diff?(blob) - blobs_with_transformed_diffs[blob] - end - - def transformed_for_diff(*blobs) - blobs.each do |b| - blobs_with_transformed_diffs[b] = true if b - end - end - - def rendered_timeout - @rendered_timeout ||= Gitlab::Metrics.counter( - :ipynb_semantic_diff_timeouts_total, - 'Counts the times notebook rendering timed out' - ) - end - - def timeout_time - Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND - end - - def execution_source - Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION - end - - def log_event(message, error = nil) - Gitlab::AppLogger.info({ message: message }) - Gitlab::ErrorTracking.track_exception(error) if error - nil - end - end - end - end -end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d6ee21b93b6..8e039d32ef5 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -43,20 +43,12 @@ module Gitlab # Ensure items are collected in the the batch new_blob_lazy old_blob_lazy - - if use_semantic_ipynb_diff? && !use_renderable_diff? - diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff - end end def use_semantic_ipynb_diff? strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project) } end - def use_renderable_diff? - strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project) } - end - def has_renderable? rendered&.has_renderable? end @@ -381,7 +373,7 @@ module Gitlab end def rendered - return unless use_semantic_ipynb_diff? && use_renderable_diff? && ipynb? && modified_file? && !too_large? + return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 316a0d2815a..75127098600 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :marker_ranges attr_writer :text, :rich_text - attr_accessor :index, :old_pos, :new_pos, :line_code, :type + attr_accessor :index, :old_pos, :new_pos, :line_code, :type, :embedded_image def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text = text diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index 1f064d8af50..0a5b2ec3890 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -3,11 +3,11 @@ module Gitlab module Diff module Rendered module Notebook - include Gitlab::Utils::StrongMemoize - class DiffFile < Gitlab::Diff::File + include Gitlab::Diff::Rendered::Notebook::DiffFileHelper + include Gitlab::Utils::StrongMemoize + RENDERED_TIMEOUT_BACKGROUND = 10.seconds - RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds BACKGROUND_EXECUTION = 'background' FOREGROUND_EXECUTION = 'foreground' LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED' @@ -49,11 +49,14 @@ module Gitlab end def highlighted_diff_lines - @highlighted_diff_lines ||= begin - removal_line_maps, addition_line_maps = compute_end_start_map - Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight.map do |line| - mutate_line(line, addition_line_maps, removal_line_maps) - end + strong_memoize(:highlighted_diff_lines) do + lines = Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight + lines_in_source = lines_in_source_diff( + source_diff.highlighted_diff_lines, source_diff.deleted_file?, source_diff.new_file? + ) + + lines.zip(line_positions_at_source_diff(lines, transformed_blocks)) + .map { |line, positions| mutate_line(line, positions, lines_in_source)} end end @@ -66,10 +69,9 @@ module Gitlab next end - Timeout.timeout(timeout_time) do + Gitlab::RenderTimeout.timeout(background: RENDERED_TIMEOUT_BACKGROUND) do IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, raise_if_invalid_nb: true, - hide_images: true, diffy_opts: { include_diff_info: true })&.tap do log_event(LOG_IPYNBDIFF_GENERATED) end @@ -90,50 +92,8 @@ module Gitlab diff end - def strip_diff_frontmatter(diff_content) - diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? - end - - def transformed_line_to_source(transformed_line, transformed_blocks) - transformed_blocks.empty? ? 0 : ( transformed_blocks[transformed_line - 1][:source_line] || -1 ) + 1 - end - - def mutate_line(line, addition_line_maps, removal_line_maps) - line.new_pos = transformed_line_to_source(line.new_pos, notebook_diff.to.blocks) - line.old_pos = transformed_line_to_source(line.old_pos, notebook_diff.from.blocks) - - line.old_pos = addition_line_maps[line.new_pos] if line.old_pos == 0 && line.new_pos != 0 - line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0 - - # Lines that do not appear on the original diff should not be commentable - line.type = "#{line.type || 'unchanged'}-nomappinginraw" unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] - - line.line_code = line_code(line) - line - end - - def compute_end_start_map - # line_codes are used for assigning notes to diffs, and these depend on the line on the new version and the - # line that would have been that one in the previous version. However, since we do a transformation on the - # file, that map gets lost. To overcome this, we look at the original source lines and build two maps: - # - For additions, we look at the latest line change for that line and pick the old line for that id - # - For removals, we look at the first line in the old version, and pick the first line on the new version - # - # - # The caveat here is that we can't have notes on lines that are not a translation of a line in the source - # diff - # - # (gitlab/diff/file.rb:75) - - removals = {} - additions = {} - - source_diff.highlighted_diff_lines.each do |line| - removals[line.old_pos] = line.new_pos unless source_diff.new_file? - additions[line.new_pos] = line.old_pos unless source_diff.deleted_file? - end - - [removals, additions] + def transformed_blocks + { from: notebook_diff.from.blocks, to: notebook_diff.to.blocks } end def rendered_timeout @@ -143,15 +103,26 @@ module Gitlab ) end - def timeout_time - Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND - end - def log_event(message, error = nil) Gitlab::AppLogger.info({ message: message }) Gitlab::ErrorTracking.log_exception(error) if error nil end + + def mutate_line(line, mapped_positions, source_diff_lines) + line.old_pos, line.new_pos = mapped_positions + + # Lines that do not appear on the original diff should not be commentable + unless source_diff_lines[:to].include?(line.new_pos) || source_diff_lines[:from].include?(line.old_pos) + line.type = "#{line.type || 'unchanged'}-nomappinginraw" + end + + line.line_code = line_code(line) + + line.rich_text = image_as_rich_text(line.text) || line.rich_text + + line + end end end end diff --git a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb new file mode 100644 index 00000000000..2e1b5ea301d --- /dev/null +++ b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true +module Gitlab + module Diff + module Rendered + module Notebook + module DiffFileHelper + require 'set' + + EMBEDDED_IMAGE_PATTERN = '  + diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? + end + + # line_positions_at_source_diff: given the transformed lines, + # what are the correct values for old_pos and new_pos? + # + # Example: + # + # Original + # from | to + # A | A + # B | D + # C | E + # F | F + # + # Original Diff + # A A + # - B + # - C + # + D + # + E + # F F + # + # Transformed + # from | to + # A | A + # C | D + # B | J + # L | E + # K | K + # F | F + # + # Transformed diff | transf old, new | OG old_pos, new_pos | + # A A | 1, 1 | 1, 1 | + # -C | 2, 2 | 3, 2 | + # -B | 3, 2 | 2, 2 | + # -L | 4, 2 | 0, 0 | + # + D | 5, 2 | 4, 2 | + # + J | 5, 3 | 0, 0 | + # + E | 5, 4 | 4, 3 | + # K K | 5, 5 | 0, 0 | + # F F | 6, 6 | 4, 4 | + def line_positions_at_source_diff(lines, blocks) + last_mapped_old_pos = 0 + last_mapped_new_pos = 0 + + lines.reverse_each.map do |line| + old_pos = source_line_from_block(line.old_pos, blocks[:from]) + new_pos = source_line_from_block(line.new_pos, blocks[:to]) + + old_has_no_mapping = old_pos == 0 + new_has_no_mapping = new_pos == 0 + + next [0, 0] if old_has_no_mapping && (new_has_no_mapping || line.type == 'old') + next [0, 0] if new_has_no_mapping && line.type == 'new' + + new_pos = last_mapped_new_pos if new_has_no_mapping && line.type == 'old' + old_pos = last_mapped_old_pos if old_has_no_mapping && line.type == 'new' + + last_mapped_old_pos = old_pos + last_mapped_new_pos = new_pos + + [old_pos, new_pos] + end.reverse + end + + def lines_in_source_diff(source_diff_lines, is_deleted_file, is_added_file) + { + from: is_added_file ? Set[] : source_diff_lines.map {|l| l.old_pos}.to_set, + to: is_deleted_file ? Set[] : source_diff_lines.map {|l| l.new_pos}.to_set + } + end + + def source_line_from_block(transformed_line, transformed_blocks) + # Blocks are the lines returned from the library and are a hash with {text:, source_line:} + # Blocks source_line are 0 indexed + return 0 if transformed_blocks.empty? + + line_in_source = transformed_blocks[transformed_line - 1][:source_line] + + return 0 unless line_in_source.present? + + line_in_source + 1 + end + + def image_as_rich_text(line_text) + return unless line_text[1..].starts_with?(EMBEDDED_IMAGE_PATTERN) + + image_body = line_text[1..].delete_prefix(EMBEDDED_IMAGE_PATTERN).delete_suffix(')') + + "<img src=\"data:image#{CGI.escapeHTML(image_body)}\">".html_safe + end + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 4da112bc5a0..ba84be6e8ca 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -32,8 +32,8 @@ module Gitlab def mail_metadata { mail_uid: mail.message_id, - from_address: mail.from, - to_address: mail.to, + from_address: from, + to_address: to, mail_key: mail_key, references: Array(mail.references), delivered_to: delivered_to.map(&:value), @@ -42,7 +42,7 @@ module Gitlab # reduced down to what looks like an email in the received headers received_recipients: recipients_from_received_headers, meta: { - client_id: "email/#{mail.from.first}", + client_id: "email/#{from.first}", project: handler&.project&.full_path } } @@ -63,6 +63,8 @@ module Gitlab end def build_mail + # See https://github.com/mikel/mail/blob/641060598f8f4be14d79bad8d703e9f2967e1cdb/spec/mail/message_spec.rb#L569 + # for mail structure Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e @@ -76,7 +78,7 @@ module Gitlab end def key_from_to_header - mail.to.find do |address| + to.find do |address| key = email_class.key_from_address(address) break key if key end @@ -110,6 +112,14 @@ module Gitlab end end + def from + Array(mail.from) + end + + def to + Array(mail.to) + end + def delivered_to Array(mail[:delivered_to]) end @@ -148,8 +158,6 @@ module Gitlab end def find_first_key_from_received_headers - return unless ::Feature.enabled?(:use_received_header_for_incoming_emails) - recipients_from_received_headers.find do |email| key = email_class.key_from_address(email) break key if key diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index d39fa139abb..c2d645138d7 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -33,10 +33,10 @@ module Gitlab l.strip.empty? || (!allow_only_quotes && l.start_with?('>')) end - encoded_body = body.force_encoding(encoding).encode("UTF-8") + encoded_body = force_utf8(body.force_encoding(encoding)) return encoded_body unless @append_reply - [encoded_body, stripped_text.force_encoding(encoding).encode("UTF-8")] + [encoded_body, force_utf8(stripped_text.force_encoding(encoding))] end private @@ -70,13 +70,29 @@ module Gitlab return if object.nil? if object.charset - object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s + # A part of a multi-part may have a different encoding. Its encoding + # is denoted in its header. For example: + # + # ``` + # ------=_Part_2192_32400445.1115745999735 + # Content-Type: text/plain; charset=ISO-8859-1 + # Content-Transfer-Encoding: 7bit + # + # Plain email. + # ``` + # So, we had to force its part to corresponding encoding before able + # to convert it to UTF-8 + force_utf8(object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8"))) else object.body.to_s end rescue StandardError nil end + + def force_utf8(str) + Gitlab::EncodingHelper.encode_utf8(str).to_s + end end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index f9959d5677b..35c1a1e73cf 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -67,7 +67,7 @@ module Gitlab # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. def track_and_raise_exception(exception, extra = {}) - process_exception(exception, sentry: true, extra: extra) + process_exception(exception, extra: extra) raise exception end @@ -87,7 +87,7 @@ module Gitlab # Provide an issue URL for follow up. # as `issue_url: 'http://gitlab.com/gitlab-org/gitlab/issues/111'` def track_and_raise_for_dev_exception(exception, extra = {}) - process_exception(exception, sentry: true, extra: extra) + process_exception(exception, extra: extra) raise exception if should_raise_for_dev? end @@ -99,7 +99,7 @@ module Gitlab # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. def track_exception(exception, extra = {}) - process_exception(exception, sentry: true, extra: extra) + process_exception(exception, extra: extra) end # This should be used when you only want to log the exception, @@ -110,7 +110,7 @@ module Gitlab # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. def log_exception(exception, extra = {}) - process_exception(exception, extra: extra) + process_exception(exception, extra: extra, trackers: [Logger]) end private @@ -136,25 +136,22 @@ module Gitlab end end - def process_exception(exception, sentry: false, logging: true, extra:) + def process_exception(exception, extra:, trackers: default_trackers) context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra) - if sentry && Raven.configuration.server - Raven.capture_exception(exception, **context_payload) + trackers.each do |tracker| + tracker.capture_exception(exception, **context_payload) end + end - # There is a possibility that this method is called before Sentry is - # configured. Since Sentry 4.0, some methods of Sentry are forwarded to - # to `nil`, hence we have to check the client as well. - if sentry && ::Sentry.get_current_client && ::Sentry.configuration.dsn - ::Sentry.capture_exception(exception, **context_payload) - end - - if logging - formatter = Gitlab::ErrorTracking::LogFormatter.new - log_hash = formatter.generate_log(exception, context_payload) - - Gitlab::ErrorTracking::Logger.error(log_hash) + def default_trackers + [].tap do |destinations| + destinations << Raven if Raven.configuration.server + # There is a possibility that this method is called before Sentry is + # configured. Since Sentry 4.0, some methods of Sentry are forwarded to + # to `nil`, hence we have to check the client as well. + destinations << ::Sentry if ::Sentry.get_current_client && ::Sentry.configuration.dsn + destinations << Logger end end diff --git a/lib/gitlab/error_tracking/logger.rb b/lib/gitlab/error_tracking/logger.rb index 1b081f943aa..f6b0a54fe22 100644 --- a/lib/gitlab/error_tracking/logger.rb +++ b/lib/gitlab/error_tracking/logger.rb @@ -3,6 +3,13 @@ module Gitlab module ErrorTracking class Logger < ::Gitlab::JsonLogger + def self.capture_exception(exception, **context_payload) + formatter = Gitlab::ErrorTracking::LogFormatter.new + log_hash = formatter.generate_log(exception, context_payload) + + self.error(log_hash) + end + def self.file_name_noext 'exceptions_json' end diff --git a/lib/gitlab/event_store/subscriber.rb b/lib/gitlab/event_store/subscriber.rb index 9f569059736..da95d3cfcfa 100644 --- a/lib/gitlab/event_store/subscriber.rb +++ b/lib/gitlab/event_store/subscriber.rb @@ -23,6 +23,7 @@ module Gitlab include ApplicationWorker loggable_arguments 0, 1 + idempotent! end def perform(event_type, data) diff --git a/lib/gitlab/event_store/subscription.rb b/lib/gitlab/event_store/subscription.rb index e5c92ab969f..01986355d2d 100644 --- a/lib/gitlab/event_store/subscription.rb +++ b/lib/gitlab/event_store/subscription.rb @@ -13,8 +13,7 @@ module Gitlab def consume_event(event) return unless condition_met?(event) - worker.perform_async(event.class.name, event.data) - # TODO: Log dispatching of events to subscriber + worker.perform_async(event.class.name, event.data.deep_stringify_keys) # We rescue and track any exceptions here because we don't want to # impact other subscribers if one is faulty. diff --git a/lib/gitlab/fips.rb b/lib/gitlab/fips.rb index 97813f13a91..b2c22182d4b 100644 --- a/lib/gitlab/fips.rb +++ b/lib/gitlab/fips.rb @@ -16,18 +16,14 @@ module Gitlab Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) ].freeze + OPENSSL_DIGESTS = %i(SHA1 SHA256 SHA384 SHA512).freeze + class << self # Returns whether we should be running in FIPS mode or not # # @return [Boolean] def enabled? - # Attempt to auto-detect FIPS mode from OpenSSL - return true if OpenSSL.fips_mode - - # Otherwise allow it to be set manually via the env vars - return true if ENV["FIPS_MODE"] == "true" - - false + ::Labkit::FIPS.enabled? end end end diff --git a/lib/gitlab/fogbugz_import/project_creator.rb b/lib/gitlab/fogbugz_import/project_creator.rb index 841f9de8d4a..a5e6356eb17 100644 --- a/lib/gitlab/fogbugz_import/project_creator.rb +++ b/lib/gitlab/fogbugz_import/project_creator.rb @@ -3,22 +3,23 @@ module Gitlab module FogbugzImport class ProjectCreator - attr_reader :repo, :fb_session, :namespace, :current_user, :user_map + attr_reader :repo, :name, :fb_session, :namespace, :current_user, :user_map - def initialize(repo, fb_session, namespace, current_user, user_map = nil) + def initialize(repo, name, namespace, current_user, fb_session, user_map = nil) @repo = repo - @fb_session = fb_session + @name = name @namespace = namespace @current_user = current_user + @fb_session = fb_session @user_map = user_map end def execute ::Projects::CreateService.new( current_user, - name: repo.safe_name, - path: repo.path, - namespace: namespace, + name: name, + path: name, + namespace_id: namespace.id, creator: current_user, visibility_level: Gitlab::VisibilityLevel::PRIVATE, import_type: 'fogbugz', diff --git a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb index e8e87a864cc..9174ca165cd 100644 --- a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb +++ b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb @@ -5,76 +5,50 @@ module Gitlab class GitlabUiFormBuilder < ActionView::Helpers::FormBuilder def gitlab_ui_checkbox_component( method, - label, + label = nil, help_text: nil, checkbox_options: {}, checked_value: '1', unchecked_value: '0', - label_options: {} + label_options: {}, + &block ) - @template.content_tag( - :div, - class: 'gl-form-checkbox custom-control custom-checkbox' - ) do - value = checkbox_options[:multiple] ? checked_value : nil - - @template.check_box( - @object_name, - method, - format_options(checkbox_options, ['custom-control-input']), - checked_value, - unchecked_value - ) + generic_label(method, label, label_options, help_text: help_text, value: value) - end + Pajamas::CheckboxComponent.new( + form: self, + method: method, + label: label, + help_text: help_text, + checkbox_options: format_options(checkbox_options), + checked_value: checked_value, + unchecked_value: unchecked_value, + label_options: format_options(label_options) + ).render_in(@template, &block) end def gitlab_ui_radio_component( method, value, - label, + label = nil, help_text: nil, radio_options: {}, - label_options: {} + label_options: {}, + &block ) - @template.content_tag( - :div, - class: 'gl-form-radio custom-control custom-radio' - ) do - @template.radio_button( - @object_name, - method, - value, - format_options(radio_options, ['custom-control-input']) - ) + generic_label(method, label, label_options, help_text: help_text, value: value) - end + Pajamas::RadioComponent.new( + form: self, + method: method, + value: value, + label: label, + help_text: help_text, + radio_options: format_options(radio_options), + label_options: format_options(label_options) + ).render_in(@template, &block) end private - def generic_label(method, label, label_options, help_text: nil, value: nil) - @template.label( - @object_name, method, format_options(label_options.merge({ value: value }), ['custom-control-label']) - ) do - if help_text - @template.content_tag( - :span, - label - ) + - @template.content_tag( - :p, - help_text, - class: 'help-text' - ) - else - label - end - end - end - - def format_options(options, classes) - classes << options[:class] - - objectify_options(options.merge({ class: classes.flatten.compact })) + def format_options(options) + objectify_options(options) end end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 5d0a638f97a..40dcac5f46f 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -31,30 +31,41 @@ module Gitlab # http://gitlab.com/some/link/#1234, and code `puts #1234`' # class ReferenceRewriter + include Gitlab::Utils::StrongMemoize + RewriteError = Class.new(StandardError) - def initialize(text, source_parent, current_user) + def initialize(text, text_html, source_parent, current_user) @text = text + + # If for some reason cached html is not present it gets rendered here + @text_html = text_html || original_html + @source_parent = source_parent @current_user = current_user - @original_html = markdown(text) @pattern = Gitlab::ReferenceExtractor.references_pattern end def rewrite(target_parent) return @text unless needs_rewrite? - @text.gsub(@pattern) do |reference| + @text.gsub!(@pattern) do |reference| unfold_reference(reference, Regexp.last_match, target_parent) end end def needs_rewrite? - @text =~ @pattern + strong_memoize(:needs_rewrite) { @text_html.include?('data-reference-type=') } end private + def original_html + strong_memoize(:original_html) do + markdown(@text) + end + end + def unfold_reference(reference, match, target_parent) before = @text[0...match.begin(0)] after = @text[match.end(0)..] @@ -89,7 +100,7 @@ module Gitlab end def substitution_valid?(substituted) - @original_html == markdown(substituted) + original_html == markdown(substituted) end def markdown(text) diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 82ef7eed56a..b0bf68f4204 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -12,7 +12,9 @@ module Gitlab # # class UploadsRewriter - def initialize(text, source_project, _current_user) + include Gitlab::Utils::StrongMemoize + + def initialize(text, _text_html, source_project, _current_user) @text = text @source_project = source_project @pattern = FileUploader::MARKDOWN_PATTERN @@ -21,7 +23,7 @@ module Gitlab def rewrite(target_parent) return @text unless needs_rewrite? - @text.gsub(@pattern) do |markdown| + @text.gsub!(@pattern) do |markdown| file = find_file($~[:secret], $~[:file]) # No file will be returned for a path traversal next if file.nil? @@ -43,15 +45,9 @@ module Gitlab end def needs_rewrite? - files.any? - end - - def files - referenced_files = @text.scan(@pattern).map do - find_file($~[:secret], $~[:file]) + strong_memoize(:needs_rewrite) do + FileUploader::MARKDOWN_PATTERN.match?(@text) end - - referenced_files.compact.select(&:exists?) end def was_embedded?(markdown) diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 2da30b88d55..505d0b8d728 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -87,7 +87,12 @@ module Gitlab length = [sha1.length, sha2.length].min return false if length < Gitlab::Git::Commit::MIN_SHA_LENGTH - sha1[0, length] == sha2[0, length] + # Optimization: prevent unnecessary substring creation + if sha1.length == sha2.length + sha1 == sha2 + else + sha1[0, length] == sha2[0, length] + end end end end diff --git a/lib/gitlab/git/cross_repo_comparer.rb b/lib/gitlab/git/cross_repo_comparer.rb index 3958373f7cb..d42b2a3bd98 100644 --- a/lib/gitlab/git/cross_repo_comparer.rb +++ b/lib/gitlab/git/cross_repo_comparer.rb @@ -45,7 +45,7 @@ module Gitlab # name that will be deleted once the method completes. This is a no-op if # fetching the source branch fails def with_commit_in_source_tmp(commit_id, &blk) - tmp_ref = "refs/tmp/#{SecureRandom.hex}" + tmp_ref = "refs/#{::Repository::REF_TMP}/#{SecureRandom.hex}" yield commit_id if source_repo.fetch_source_branch!(target_repo, commit_id, tmp_ref) ensure diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index c473fe6973d..003cc87d65a 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -44,7 +44,7 @@ module Gitlab else # Only show what is new in the source branch # compared to the target branch, not the other way - # around. The linex below with merge_base is + # around. The line below with merge_base is # equivalent to diff with three dots (git diff # branch1...branch2) From the git documentation: # "git diff A...B" is equivalent to "git diff diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ab365069adf..df744bd60b4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -366,9 +366,9 @@ module Gitlab end end - def new_commits(newrevs, allow_quarantine: false) + def new_commits(newrevs) wrapped_gitaly_errors do - gitaly_commit_client.list_new_commits(Array.wrap(newrevs), allow_quarantine: allow_quarantine) + gitaly_commit_client.list_new_commits(Array.wrap(newrevs)) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index cba63b3c6c7..66fd7aaedea 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true -# Check a user's access to perform a git action. All public methods in this -# class return an instance of `GitlabAccessStatus` +# Checks a user's access to perform a git action. +# All public methods in this class return an instance of `GitlabAccessStatus` + module Gitlab class GitAccess include Gitlab::Utils::StrongMemoize @@ -99,7 +100,7 @@ module Gitlab @logger ||= Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) end - def guest_can_download_code? + def guest_can_download? Guest.can?(download_ability, container) end @@ -107,10 +108,10 @@ module Gitlab authentication_abilities.include?(:download_code) && deploy_key? && deploy_key.has_access_to?(container) && - (project? && project&.repository_access_level != ::Featurable::DISABLED) + (project? && repository_access_level != ::Featurable::DISABLED) end - def user_can_download_code? + def user_can_download? authentication_abilities.include?(:download_code) && user_access.can_do_action?(download_ability) end @@ -125,10 +126,6 @@ module Gitlab raise NotImplementedError end - def build_can_download_code? - authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) - end - def request_from_ci_build? return false unless protocol == 'http' @@ -136,11 +133,36 @@ module Gitlab end def protocol_allowed? - Gitlab::ProtocolAccess.allowed?(protocol) + Gitlab::ProtocolAccess.allowed?(protocol, project: project) end private + # when accessing via the CI_JOB_TOKEN + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + end + + def build_can_download? + build_can_download_code? + end + + def deploy_token_can_download? + deploy_token? + end + + # When overriding this method, be careful using super + # as deploy_token_can_download? and build_can_download? + # do not consider the download_ability in the inheriting class + # for deploy tokens and builds + def can_download? + deploy_key_can_download_code? || + deploy_token_can_download? || + build_can_download? || + user_can_download? || + guest_can_download? + end + def check_container! # Strict nil check, to avoid any surprises with Object#present? # which can delegate to #empty? @@ -273,15 +295,9 @@ module Gitlab end def check_download_access! - passed = deploy_key_can_download_code? || - deploy_token? || - user_can_download_code? || - build_can_download_code? || - guest_can_download_code? - - unless passed - raise ForbiddenError, download_forbidden_message - end + return if can_download? + + raise ForbiddenError, download_forbidden_message end def download_forbidden_message @@ -517,6 +533,10 @@ module Gitlab # overriden in EE def check_additional_conditions! end + + def repository_access_level + project&.repository_access_level + end end end diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb index 7e9bab4a8e6..732e0e14257 100644 --- a/lib/gitlab/git_access_project.rb +++ b/lib/gitlab/git_access_project.rb @@ -74,3 +74,5 @@ module Gitlab end end end + +Gitlab::GitAccessProject.prepend_mod_with('Gitlab::GitAccessProject') diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 5ae17dbbb91..8c291dd56ba 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -90,13 +90,14 @@ module Gitlab super end - override :check_download_access! - def check_download_access! - passed = guest_can_download_code? || user_can_download_code? + override :can_download? + def can_download? + guest_can_download? || user_can_download? + end - unless passed - raise ForbiddenError, error_message(:read_snippet) - end + override :download_forbidden_message + def download_forbidden_message + error_message(:read_snippet) end override :check_change_access! diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index fdd7e8a8c4a..34378ac8f46 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -27,12 +27,21 @@ module Gitlab :create_wiki end - override :check_download_access! - def check_download_access! - super + private + + override :build_can_download? + def build_can_download? + super && user_access.can_do_action?(download_ability) + end - raise ForbiddenError, download_forbidden_message if build_cannot_download? - raise ForbiddenError, download_forbidden_message if deploy_token_cannot_download? + override :deploy_token_can_download? + def deploy_token_can_download? + super && deploy_token.can?(download_ability, container) + end + + override :repository_access_level + def repository_access_level + project&.wiki_access_level end override :check_change_access! @@ -53,17 +62,6 @@ module Gitlab def not_found_message error_message(:not_found) end - - private - - # when accessing via the CI_JOB_TOKEN - def build_cannot_download? - build_can_download_code? && !user_access.can_do_action?(download_ability) - end - - def deploy_token_cannot_download? - deploy_token && !deploy_token.can?(download_ability, container) - end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 5e1f92ae835..9fb34f74c82 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -258,9 +258,9 @@ module Gitlab end # List all commits which are new in the repository. If commits have been pushed into the repo - def list_new_commits(revisions, allow_quarantine: false) + def list_new_commits(revisions) git_env = Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository) - if allow_quarantine && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + if git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? # If we have a quarantine environment, then we can optimize the check # by doing a ListAllCommitsRequest. Instead of walking through # references, we just walk through all quarantined objects, which is @@ -278,32 +278,29 @@ module Gitlab response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) quarantined_commits = consume_commits_response(response) - - if Feature.enabled?(:filter_quarantined_commits) - quarantined_commit_ids = quarantined_commits.map(&:id) - - # While in general the quarantine directory would only contain objects - # which are actually new, this is not guaranteed by Git. In fact, - # git-push(1) may sometimes push objects which already exist in the - # target repository. We do not want to return those from this method - # though given that they're not actually new. - # - # To fix this edge-case we thus have to filter commits down to those - # which don't yet exist. To do so, we must check for object existence - # in the main repository, but the object directory of our repository - # points into the object quarantine. This can be fixed by unsetting - # it, which will cause us to use the normal repository as indicated by - # its relative path again. - main_repo = @gitaly_repo.dup - main_repo.git_object_directory = "" - - # Check object existence of all quarantined commits' IDs. - quarantined_commit_existence = object_existence_map(quarantined_commit_ids, gitaly_repo: main_repo) - - # And then we reject all quarantined commits which exist in the main - # repository already. - quarantined_commits.reject! { |c| quarantined_commit_existence[c.id] } - end + quarantined_commit_ids = quarantined_commits.map(&:id) + + # While in general the quarantine directory would only contain objects + # which are actually new, this is not guaranteed by Git. In fact, + # git-push(1) may sometimes push objects which already exist in the + # target repository. We do not want to return those from this method + # though given that they're not actually new. + # + # To fix this edge-case we thus have to filter commits down to those + # which don't yet exist. To do so, we must check for object existence + # in the main repository, but the object directory of our repository + # points into the object quarantine. This can be fixed by unsetting + # it, which will cause us to use the normal repository as indicated by + # its relative path again. + main_repo = @gitaly_repo.dup + main_repo.git_object_directory = "" + + # Check object existence of all quarantined commits' IDs. + quarantined_commit_existence = object_existence_map(quarantined_commit_ids, gitaly_repo: main_repo) + + # And then we reject all quarantined commits which exist in the main + # repository already. + quarantined_commits.reject! { |c| quarantined_commit_existence[c.id] } quarantined_commits else diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 4637bf2e3ff..d575c0f470d 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -101,6 +101,16 @@ module Gitlab if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error end + rescue GRPC::BadStatus => e + detailed_error = decode_detailed_error(e) + + case detailed_error&.error + when :custom_hook + raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), + fallback_message: e.details) + else + raise + end end def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false) @@ -163,6 +173,9 @@ module Gitlab access_check_error = detailed_error.access_check # These messages were returned from internal/allowed API calls raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :custom_hook + raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), + fallback_message: e.details) when :reference_update # We simply ignore any reference update errors which are typically an # indicator of multiple RPC calls trying to update the same reference @@ -299,10 +312,6 @@ module Gitlab timeout: GitalyClient.long_timeout ) - if response.git_error.presence - raise Gitlab::Git::Repository::GitError, response.git_error - end - response.squash_sha rescue GRPC::BadStatus => e detailed_error = decode_detailed_error(e) @@ -464,6 +473,21 @@ module Gitlab ) handle_cherry_pick_or_revert_response(response) + rescue GRPC::BadStatus => e + detailed_error = decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :cherry_pick_conflict + raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' + when :target_branch_diverged + raise Gitlab::Git::CommitError, 'branch diverged' + else + raise e + end end def handle_cherry_pick_or_revert_response(response) @@ -526,6 +550,14 @@ module Gitlab # Error Class might not be known to ruby yet nil end + + def custom_hook_error_message(custom_hook_error) + # Custom hooks may return messages via either stdout or stderr which have a specific prefix. If + # that prefix is present we'll want to print the hook's output, otherwise we'll want to print the + # Gitaly error as a fallback. + custom_hook_output = custom_hook_error.stderr.presence || custom_hook_error.stdout + EncodingHelper.encode!(custom_hook_output) + end end end end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 7f46615f17e..35fd4bd88a0 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -31,6 +31,7 @@ module Gitlab if (issue_id = create_issue) create_assignees(issue_id) issuable_finder.cache_database_id(issue_id) + update_search_data(issue_id) if Feature.enabled?(:issues_full_text_search) end end end @@ -77,6 +78,13 @@ module Gitlab ApplicationRecord.legacy_bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert end + + # Adds search data to database (if full_text_search feature is enabled) + # + # issue_id - The ID of the created issue. + def update_search_data(issue_id) + project.issues.find(issue_id)&.update_search_data! + end end end end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 64ec0251e54..7241e1ef703 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -27,7 +27,7 @@ module Gitlab def build(release) existing_tags.add(release.tag_name) - { + build_hash = { name: release.name, tag: release.tag_name, description: description_for(release), @@ -37,6 +37,12 @@ module Gitlab released_at: release.published_at || Time.current, project_id: project.id } + + if Feature.enabled?(:import_release_authors_from_github, project) + build_hash[:author_id] = fetch_author_id(release) + end + + build_hash end def each_release @@ -50,6 +56,18 @@ module Gitlab def object_type :release end + + private + + def fetch_author_id(release) + author_id, _author_found = user_finder.author_id_for(release) + + author_id + end + + def user_finder + @user_finder ||= GithubImport::UserFinder.new(project, client) + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 98570c02e3d..5f1802e323c 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -54,8 +54,6 @@ module Gitlab push_frontend_feature_flag(:usage_data_api, type: :ops) push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:new_header_search) - push_frontend_feature_flag(:bootstrap_confirmation_modals) - push_frontend_feature_flag(:sandboxed_mermaid) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) push_frontend_feature_flag(:mr_attention_requests, current_user) diff --git a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb index fe741a5bbe8..dde2bdd855e 100644 --- a/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb +++ b/lib/gitlab/grape_logging/loggers/queue_duration_logger.rb @@ -6,21 +6,12 @@ module Gitlab module GrapeLogging module Loggers class QueueDurationLogger < ::GrapeLogging::Loggers::Base - attr_accessor :start_time - - def before - @start_time = Time.now - end - def parameters(request, _) - proxy_start = request.env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence - - return {} unless proxy_start && start_time + duration_s = request.env[Gitlab::Middleware::RailsQueueDuration::GITLAB_RAILS_QUEUE_DURATION_KEY].presence - # Time in milliseconds since gitlab-workhorse started the request - duration = start_time.to_f * 1_000 - proxy_start.to_f / 1_000_000 + return {} unless duration_s - { 'queue_duration_s': Gitlab::Utils.ms_to_round_sec(duration) } + { 'queue_duration_s': duration_s } end end end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index dc49c806398..884fc85c4ec 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -15,11 +15,7 @@ module Gitlab # If the `#authorize` call is used on multiple classes, we add the # permissions specified on a subclass, to the ones that were specified # on its superclass. - @required_permissions ||= if respond_to?(:superclass) && superclass.respond_to?(:required_permissions) - superclass.required_permissions.dup - else - [] - end + @required_permissions ||= call_superclass_method(:required_permissions, []).dup end def authorize(*permissions) @@ -27,6 +23,8 @@ module Gitlab end def authorizes_object? + return true if call_superclass_method(:authorizes_object?, false) + defined?(@authorizes_object) ? @authorizes_object : false end @@ -37,6 +35,14 @@ module Gitlab def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) raise ::Gitlab::Graphql::Errors::ResourceNotAvailable, msg end + + private + + def call_superclass_method(method_name, or_else) + return or_else unless respond_to?(:superclass) && superclass.respond_to?(method_name) + + superclass.send(method_name) # rubocop: disable GitlabSecurity/PublicSend + end end def find_object(*args) diff --git a/lib/gitlab/graphql/board/issues_connection_extension.rb b/lib/gitlab/graphql/board/issues_connection_extension.rb index 9dcd8c92592..b909cb021de 100644 --- a/lib/gitlab/graphql/board/issues_connection_extension.rb +++ b/lib/gitlab/graphql/board/issues_connection_extension.rb @@ -2,7 +2,7 @@ module Gitlab module Graphql module Board - class IssuesConnectionExtension < GraphQL::Schema::Field::ConnectionExtension + class IssuesConnectionExtension < GraphQL::Schema::FieldExtension def after_resolve(value:, object:, context:, **rest) ::Boards::Issues::ListService .initialize_relative_positions(object.list.board, context[:current_user], value.nodes) diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb index 3335e511714..d30751fe46e 100644 --- a/lib/gitlab/graphql/deprecation.rb +++ b/lib/gitlab/graphql/deprecation.rb @@ -3,9 +3,12 @@ module Gitlab module Graphql class Deprecation + REASON_RENAMED = :renamed + REASON_ALPHA = :alpha + REASONS = { - renamed: 'This was renamed.', - alpha: 'This feature is in Alpha, and can be removed or changed at any point.' + REASON_RENAMED => 'This was renamed.', + REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.' }.freeze include ActiveModel::Validations @@ -39,7 +42,7 @@ module Gitlab def markdown(context: :inline) parts = [ - "#{deprecated_in(format: :markdown)}.", + "#{changed_in_milestone(format: :markdown)}.", reason_text, replacement_markdown.then { |r| "Use: #{r}." if r } ].compact @@ -77,7 +80,7 @@ module Gitlab [ reason_text, replacement && "Please use `#{replacement}`.", - "#{deprecated_in}." + "#{changed_in_milestone}." ].compact.join(' ') end @@ -107,15 +110,24 @@ module Gitlab end def description_suffix - " #{deprecated_in}: #{reason_text}" + " #{changed_in_milestone}: #{reason_text}" end - def deprecated_in(format: :plain) + # Returns 'Deprecated in <milestone>' for proper deprecations. + # Retruns 'Introduced in <milestone>' for :alpha deprecations. + # Formatted to markdown or plain format. + def changed_in_milestone(format: :plain) + verb = if reason == REASON_ALPHA + 'Introduced' + else + 'Deprecated' + end + case format when :plain - "Deprecated in #{milestone}" + "#{verb} in #{milestone}" when :markdown - "**Deprecated** in #{milestone}" + "**#{verb}** in #{milestone}" end end end diff --git a/lib/gitlab/graphql/generic_tracing.rb b/lib/gitlab/graphql/generic_tracing.rb index 936b22d5afa..d3de9c714f4 100644 --- a/lib/gitlab/graphql/generic_tracing.rb +++ b/lib/gitlab/graphql/generic_tracing.rb @@ -23,6 +23,14 @@ module Gitlab "#{type.name}.#{field.name}" end + def platform_authorized_key(type) + "#{type.graphql_name}.authorized" + end + + def platform_resolve_type_key(type) + "#{type.graphql_name}.resolve_type" + end + def platform_trace(platform_key, key, data, &block) tags = { platform_key: platform_key, key: key } start = Gitlab::Metrics::System.monotonic_time diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb index 805864cdd4c..41c3af33909 100644 --- a/lib/gitlab/graphql/loaders/batch_model_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -4,20 +4,27 @@ module Gitlab module Graphql module Loaders class BatchModelLoader - attr_reader :model_class, :model_id + attr_reader :model_class, :model_id, :preloads - def initialize(model_class, model_id) + def initialize(model_class, model_id, preloads = nil) @model_class = model_class @model_id = model_id + @preloads = preloads || [] end # rubocop: disable CodeReuse/ActiveRecord def find - BatchLoader::GraphQL.for(model_id.to_i).batch(key: model_class) do |ids, loader, args| + BatchLoader::GraphQL.for([model_id.to_i, preloads]).batch(key: model_class) do |for_params, loader, args| model = args[:key] + keys_by_id = for_params.group_by(&:first) + ids = for_params.map(&:first) + preloads = for_params.flat_map(&:second).uniq results = model.where(id: ids) + results = results.preload(*preloads) unless preloads.empty? - results.each { |record| loader.call(record.id, record) } + results.each do |record| + keys_by_id.fetch(record.id, []).each { |k| loader.call(k, record) } + end end end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 6188d860aba..43dddf4c4bc 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -22,8 +22,10 @@ module Gitlab field name, GraphQL::Types::String, **kwargs define_method resolver_method do + markdown_object = block_given? ? yield(object) : object + # We need to `dup` the context so the MarkdownHelper doesn't modify it - ::MarkupHelper.markdown_field(object, method_name.to_sym, context.to_h.dup) + ::MarkupHelper.markdown_field(markdown_object, method_name.to_sym, context.to_h.dup) end end end diff --git a/lib/gitlab/graphql/present/field_extension.rb b/lib/gitlab/graphql/present/field_extension.rb index 050a3a276ea..bc6d0c6fd35 100644 --- a/lib/gitlab/graphql/present/field_extension.rb +++ b/lib/gitlab/graphql/present/field_extension.rb @@ -21,6 +21,7 @@ module Gitlab # TODO: remove this when resolve procs are removed from the # graphql-ruby library, and all field instrumentation is removed. # See: https://github.com/rmosolgo/graphql-ruby/issues/3385 + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/363131 presented = field.owner.try(:present, object, attrs) || object yield(presented, arguments) end diff --git a/lib/gitlab/graphql/query_analyzers/ast/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/ast/logger_analyzer.rb new file mode 100644 index 00000000000..9a7069249ec --- /dev/null +++ b/lib/gitlab/graphql/query_analyzers/ast/logger_analyzer.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module QueryAnalyzers + module AST + class LoggerAnalyzer < GraphQL::Analysis::AST::Analyzer + COMPLEXITY_ANALYZER = GraphQL::Analysis::AST::QueryComplexity + DEPTH_ANALYZER = GraphQL::Analysis::AST::QueryDepth + FIELD_USAGE_ANALYZER = GraphQL::Analysis::AST::FieldUsage + ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze + + def initialize(query) + super + + @results = default_initial_values(query).merge({ + time_started: Gitlab::Metrics::System.monotonic_time + }) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + @results = default_initial_values(query_or_multiplex) + end + + def result + complexity, depth, field_usages = + GraphQL::Analysis::AST.analyze_query(@subject, ALL_ANALYZERS, multiplex_analyzers: []) + + results[:depth] = depth + results[:complexity] = complexity + # This duration is not the execution time of the + # query but the execution time of the analyzer. + results[:duration_s] = duration(results[:time_started]) + results[:used_fields] = field_usages[:used_fields] + results[:used_deprecated_fields] = field_usages[:used_deprecated_fields] + + push_to_request_store(results) + + # This gl_analysis is included in the tracer log + query.context[:gl_analysis] = results.except!(:time_started, :query) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + private + + attr_reader :results + + def push_to_request_store(results) + query = @subject + + # TODO: This RequestStore management is used to handle setting request wide metadata + # to improve preexisting logging. We should handle this either with ApplicationContext + # or in a separate tracer. + # https://gitlab.com/gitlab-org/gitlab/-/issues/343802 + + RequestStore.store[:graphql_logs] ||= [] + RequestStore.store[:graphql_logs] << results.except(:time_started, :duration_s).merge({ + variables: process_variables(query.provided_variables), + operation_name: query.operation_name + }) + end + + def process_variables(variables) + filtered_variables = filter_sensitive_variables(variables) + filtered_variables.try(:to_s) || filtered_variables + end + + def filter_sensitive_variables(variables) + ActiveSupport::ParameterFilter + .new(::Rails.application.config.filter_parameters) + .filter(variables) + end + + def duration(time_started) + Gitlab::Metrics::System.monotonic_time - time_started + end + + def default_initial_values(query) + { + time_started: Gitlab::Metrics::System.monotonic_time, + duration_s: nil + } + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb b/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb new file mode 100644 index 00000000000..4e90e4c912f --- /dev/null +++ b/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# Recursive queries, with relatively low effort, can quickly spiral out of control exponentially +# and may not be picked up by depth and complexity alone. +module Gitlab + module Graphql + module QueryAnalyzers + module AST + class RecursionAnalyzer < GraphQL::Analysis::AST::Analyzer + IGNORED_FIELDS = %w(node edges nodes ofType).freeze + RECURSION_THRESHOLD = 2 + + def initialize(query) + super + + @node_visits = {} + @recurring_fields = {} + end + + def on_enter_field(node, _parent, visitor) + return if skip_node?(node, visitor) + + node_name = node.name + node_visits[node_name] ||= 0 + node_visits[node_name] += 1 + + times_encountered = @node_visits[node_name] + recurring_fields[node_name] = times_encountered if recursion_too_deep?(node_name, times_encountered) + end + + # Visitors are all defined on the AST::Analyzer base class + # We override them for custom analyzers. + def on_leave_field(node, _parent, visitor) + return if skip_node?(node, visitor) + + node_name = node.name + node_visits[node_name] ||= 0 + node_visits[node_name] -= 1 + end + + def result + @recurring_fields = @recurring_fields.select { |k, v| recursion_too_deep?(k, v) } + + if @recurring_fields.any? + GraphQL::AnalysisError.new(<<~MSG) + Recursive query - too many of fields '#{@recurring_fields}' detected + in single branch of the query") + MSG + end + end + + private + + attr_reader :node_visits, :recurring_fields + + def recursion_too_deep?(node_name, times_encountered) + return if IGNORED_FIELDS.include?(node_name) + + times_encountered > recursion_threshold + end + + def skip_node?(node, visitor) + # We don't want to count skipped fields or fields + # inside fragment definitions + return false if visitor.skipping? || visitor.visiting_fragment_definition? + + !node.is_a?(GraphQL::Language::Nodes::Field) || node.selections.empty? + end + + # separated into a method for use in allow_high_graphql_recursion + def recursion_threshold + RECURSION_THRESHOLD + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb deleted file mode 100644 index 207324e73bd..00000000000 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module QueryAnalyzers - class LoggerAnalyzer - COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value } - DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value } - FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] } - ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze - - def initial_value(query) - { - time_started: Gitlab::Metrics::System.monotonic_time, - query: query - } - end - - def call(memo, *) - memo - end - - def final_value(memo) - return if memo.nil? - - query = memo[:query] - complexity, depth, field_usages = GraphQL::Analysis.analyze_query(query, ALL_ANALYZERS) - - memo[:depth] = depth - memo[:complexity] = complexity - # This duration is not the execution time of the - # query but the execution time of the analyzer. - memo[:duration_s] = duration(memo[:time_started]) - memo[:used_fields] = field_usages.first - memo[:used_deprecated_fields] = field_usages.second - - push_to_request_store(memo) - - # This gl_analysis is included in the tracer log - query.context[:gl_analysis] = memo.except!(:time_started, :query) - rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) - end - - private - - def push_to_request_store(memo) - query = memo[:query] - - # TODO: This RequestStore management is used to handle setting request wide metadata - # to improve preexisting logging. We should handle this either with ApplicationContext - # or in a separate tracer. - # https://gitlab.com/gitlab-org/gitlab/-/issues/343802 - - RequestStore.store[:graphql_logs] ||= [] - RequestStore.store[:graphql_logs] << memo.except(:time_started, :duration_s, :query).merge({ - variables: process_variables(query.provided_variables), - operation_name: query.operation_name - }) - end - - def process_variables(variables) - filtered_variables = filter_sensitive_variables(variables) - - if filtered_variables.respond_to?(:to_s) - filtered_variables.to_s - else - filtered_variables - end - end - - def filter_sensitive_variables(variables) - ActiveSupport::ParameterFilter - .new(::Rails.application.config.filter_parameters) - .filter(variables) - end - - def duration(time_started) - Gitlab::Metrics::System.monotonic_time - time_started - end - end - end - end -end diff --git a/lib/gitlab/graphql/query_analyzers/recursion_analyzer.rb b/lib/gitlab/graphql/query_analyzers/recursion_analyzer.rb deleted file mode 100644 index 79a7104a2ff..00000000000 --- a/lib/gitlab/graphql/query_analyzers/recursion_analyzer.rb +++ /dev/null @@ -1,62 +0,0 @@ -# frozen_string_literal: true - -# Recursive queries, with relatively low effort, can quickly spiral out of control exponentially -# and may not be picked up by depth and complexity alone. -module Gitlab - module Graphql - module QueryAnalyzers - class RecursionAnalyzer - IGNORED_FIELDS = %w(node edges nodes ofType).freeze - RECURSION_THRESHOLD = 2 - - def initial_value(query) - { - recurring_fields: {} - } - end - - def call(memo, visit_type, irep_node) - return memo if skip_node?(irep_node) - - node_name = irep_node.ast_node.name - times_encountered = memo[node_name] || 0 - - if visit_type == :enter - times_encountered += 1 - memo[:recurring_fields][node_name] = times_encountered if recursion_too_deep?(node_name, times_encountered) - else - times_encountered -= 1 - end - - memo[node_name] = times_encountered - memo - end - - def final_value(memo) - recurring_fields = memo[:recurring_fields] - recurring_fields = recurring_fields.select { |k, v| recursion_too_deep?(k, v) } - if recurring_fields.any? - GraphQL::AnalysisError.new("Recursive query - too many of fields '#{recurring_fields}' detected in single branch of the query") - end - end - - private - - def recursion_too_deep?(node_name, times_encountered) - return if IGNORED_FIELDS.include?(node_name) - - times_encountered > recursion_threshold - end - - def skip_node?(irep_node) - ast_node = irep_node.ast_node - !ast_node.is_a?(GraphQL::Language::Nodes::Field) || ast_node.selections.empty? - end - - def recursion_threshold - RECURSION_THRESHOLD - end - end - end - end -end diff --git a/lib/gitlab/hash_digest/facade.rb b/lib/gitlab/hash_digest/facade.rb new file mode 100644 index 00000000000..d8efef02893 --- /dev/null +++ b/lib/gitlab/hash_digest/facade.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module HashDigest + # Used for rolling out to use OpenSSL::Digest::SHA256 + # for ActiveSupport::Digest + class Facade + class << self + def hexdigest(...) + hash_digest_class.hexdigest(...) + end + + def hash_digest_class + if use_sha256? + ::OpenSSL::Digest::SHA256 + else + ::Digest::MD5 # rubocop:disable Fips/MD5 + end + end + + def use_sha256? + return false unless Feature.feature_flags_available? + + Feature.enabled?(:active_support_hash_digest_sha256) + end + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 758a594036b..b71abe5c052 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -2,9 +2,6 @@ module Gitlab class Highlight - TIMEOUT_BACKGROUND = 30.seconds - TIMEOUT_FOREGROUND = 1.5.seconds - def self.highlight(blob_name, blob_content, language: nil, plain: false) new(blob_name, blob_content, language: language) .highlight(blob_content, continue: false, plain: plain) @@ -72,7 +69,7 @@ module Gitlab def highlight_rich(text, continue: true) tag = lexer.tag tokens = lexer.lex(text, continue: continue) - Timeout.timeout(timeout_time) { @formatter.format(tokens, **context, tag: tag).html_safe } + Gitlab::RenderTimeout.timeout { @formatter.format(tokens, **context, tag: tag).html_safe } rescue Timeout::Error => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) @@ -80,10 +77,6 @@ module Gitlab highlight_plain(text) end - def timeout_time - Gitlab::Runtime.sidekiq? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND - end - def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 06ddd65d075..2b4bdbd48bd 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -52,7 +52,7 @@ module Gitlab source: merge_request.source_project.try(:hook_attrs), target: merge_request.target_project.hook_attrs, last_commit: merge_request.diff_head_commit&.hook_attrs, - work_in_progress: merge_request.work_in_progress?, + work_in_progress: merge_request.draft?, total_time_spent: merge_request.total_time_spent, time_change: merge_request.time_change, human_total_time_spent: merge_request.human_total_time_spent, diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index c8239c9e308..8d9f86d3232 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,11 +43,11 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 43, + 'da_DK' => 41, 'de' => 14, 'en' => 100, 'eo' => 0, - 'es' => 39, + 'es' => 36, 'fil_PH' => 0, 'fr' => 10, 'gl_ES' => 0, @@ -55,15 +55,15 @@ module Gitlab 'it' => 1, 'ja' => 33, 'ko' => 12, - 'nb_NO' => 29, + 'nb_NO' => 27, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 50, - 'ro_RO' => 58, - 'ru' => 30, - 'tr_TR' => 13, - 'uk' => 47, - 'zh_CN' => 96, + 'pt_BR' => 54, + 'ro_RO' => 79, + 'ru' => 29, + 'tr_TR' => 12, + 'uk' => 44, + 'zh_CN' => 94, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 22a7a8dd7cd..6ad368a5d2f 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -56,7 +56,11 @@ module Gitlab end def copy_file_for_lfs_object(lfs_object) - copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) + file_path = lfs_object.file.path + + return unless File.exist?(file_path) + + copy_files(file_path, destination_path_for_object(lfs_object)) end def append_lfs_json_for_batch(lfs_objects_batch) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 1625c39595c..5a1787218f5 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -283,6 +283,7 @@ included_attributes: - :analytics_access_level - :security_and_compliance_access_level - :container_registry_access_level + - :package_registry_access_level prometheus_metrics: - :created_at - :updated_at @@ -684,6 +685,7 @@ included_attributes: - :operations_access_level - :security_and_compliance_access_level - :container_registry_access_level + - :package_registry_access_level - :allow_merge_on_skipped_pipeline - :auto_devops_deploy_strategy - :auto_devops_enabled diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index c9f5005cede..5d3a6b0c6e1 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -11,17 +11,17 @@ module Gitlab # We exclude `bare_repository` here as it has no import class associated IMPORT_TABLE = [ - ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), - ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), - ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), - ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), - ImportSource.new('google_code', 'Google Code', nil), - ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), - ImportSource.new('git', 'Repo by URL', nil), - ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), - ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), - ImportSource.new('manifest', 'Manifest file', nil), - ImportSource.new('phabricator', 'Phabricator', Gitlab::PhabricatorImport::Importer) + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), + ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), + ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', nil), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repository by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), + ImportSource.new('manifest', 'Manifest file', nil), + ImportSource.new('phabricator', 'Phabricator', Gitlab::PhabricatorImport::Importer) ].freeze class << self diff --git a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb index f3f8e774b4b..3fdb34d42b7 100644 --- a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb +++ b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb @@ -2,6 +2,8 @@ module Gitlab class InactiveProjectsDeletionWarningTracker + include Gitlab::Utils::StrongMemoize + attr_reader :project_id DELETION_TRACKING_REDIS_KEY = 'inactive_projects_deletion_warning_email_notified' @@ -38,10 +40,33 @@ module Gitlab end end + def notification_date + Gitlab::Redis::SharedState.with do |redis| + redis.hget(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + + def scheduled_deletion_date + if notification_date.present? + (notification_date.to_date + grace_period_after_notification).to_s + else + grace_period_after_notification.from_now.to_date.to_s + end + end + def reset Gitlab::Redis::SharedState.with do |redis| redis.hdel(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") end end + + private + + def grace_period_after_notification + strong_memoize(:grace_period_after_notification) do + (::Gitlab::CurrentSettings.inactive_projects_delete_after_months - + ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months).months + end + end end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 379c27caeb7..b8d8deb3418 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -31,6 +31,7 @@ module Gitlab instrument_thread_memory_allocations(payload) instrument_load_balancing(payload) instrument_pid(payload) + instrument_worker_id(payload) instrument_uploads(payload) instrument_rate_limiting_gates(payload) end @@ -106,6 +107,10 @@ module Gitlab payload[:pid] = Process.pid end + def instrument_worker_id(payload) + payload[:worker_id] = ::Prometheus::PidProvider.worker_id + end + def instrument_thread_memory_allocations(payload) counters = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations( ::Gitlab::RequestContext.instance.thread_memory_allocations) diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index fc5834613fd..4ddafbac4c6 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -63,6 +63,10 @@ module Gitlab # Gitea doesn't have a Release API yet # See https://github.com/go-gitea/gitea/issues/330 + # On re-enabling care should be taken to include releases `author_id` field and enable corresponding tests. + # See: + # 1) https://gitlab.com/gitlab-org/gitlab/-/issues/343448#note_985979730 + # 2) https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89694/diffs#dfc4a8141aa296465ea3c50b095a30292fb6ebc4_180_182 unless project.gitea_import? import_releases end diff --git a/lib/gitlab/mailgun/webhook_processors/base.rb b/lib/gitlab/mailgun/webhook_processors/base.rb new file mode 100644 index 00000000000..9402637a51d --- /dev/null +++ b/lib/gitlab/mailgun/webhook_processors/base.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Mailgun + module WebhookProcessors + class Base + def initialize(payload) + @payload = payload || {} + end + + def execute + end + + private + + attr_reader :payload + end + end + end +end diff --git a/lib/gitlab/mailgun/webhook_processors/failure_logger.rb b/lib/gitlab/mailgun/webhook_processors/failure_logger.rb new file mode 100644 index 00000000000..a7a85bd1672 --- /dev/null +++ b/lib/gitlab/mailgun/webhook_processors/failure_logger.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Mailgun + module WebhookProcessors + class FailureLogger < Base + def execute + log_failure if permanent_failure? || temporary_failure_over_threshold? + end + + def permanent_failure? + payload['event'] == 'failed' && payload['severity'] == 'permanent' + end + + def temporary_failure_over_threshold? + payload['event'] == 'failed' && payload['severity'] == 'temporary' && + Gitlab::ApplicationRateLimiter.throttled?(:temporary_email_failure, scope: payload['recipient']) + end + + private + + def log_failure + Gitlab::ErrorTracking::Logger.error( + event: 'email_delivery_failure', + mailgun_event_id: payload['id'], + recipient: payload['recipient'], + failure_type: payload['severity'], + failure_reason: payload['reason'], + failure_code: payload.dig('delivery-status', 'code'), + failure_message: payload.dig('delivery-status', 'message') + ) + end + end + end + end +end diff --git a/lib/gitlab/mailgun/webhook_processors/member_invites.rb b/lib/gitlab/mailgun/webhook_processors/member_invites.rb new file mode 100644 index 00000000000..f54c44381f0 --- /dev/null +++ b/lib/gitlab/mailgun/webhook_processors/member_invites.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Mailgun + module WebhookProcessors + class MemberInvites < Base + ProcessWebhookServiceError = Class.new(StandardError) + + def execute + return unless should_process? + + @member = Member.find_by_invite_token(invite_token) + update_member_and_log if member + rescue ProcessWebhookServiceError => e + Gitlab::ErrorTracking.track_exception(e) + end + + private + + attr_reader :member + + def should_process? + payload['event'] == 'failed' && payload['severity'] == 'permanent' && + payload['tags']&.include?(::Members::Mailgun::INVITE_EMAIL_TAG) + end + + def update_member_and_log + log_update_event if member.update(invite_email_success: false) + end + + def log_update_event + Gitlab::AppLogger.info( + message: "UPDATED MEMBER INVITE_EMAIL_SUCCESS: member_id: #{member.id}", + event: 'updated_member_invite_email_success' + ) + end + + def invite_token + # may want to validate schema in some way using ::JSONSchemer.schema(SCHEMA_PATH).valid?(message) if this + # gets more complex + payload.dig('user-variables', ::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY) || + raise(ProcessWebhookServiceError, "Expected to receive #{::Members::Mailgun::INVITE_EMAIL_TOKEN_KEY} " \ + "in user-variables: #{payload}") + end + end + end + end +end diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index 283502d90c1..09ba95666de 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -11,7 +11,7 @@ module Gitlab # this if the change to the renderer output is a new feature or a # minor bug fix. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330313 - CACHE_COMMONMARK_VERSION = 30 + CACHE_COMMONMARK_VERSION = 31 CACHE_COMMONMARK_VERSION_START = 10 BaseError = Class.new(StandardError) diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb new file mode 100644 index 00000000000..454c54569de --- /dev/null +++ b/lib/gitlab/memory/jemalloc.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'fiddle' + +module Gitlab + module Memory + module Jemalloc + extend self + + STATS_FORMATS = { + json: { options: 'J', extension: 'json' }, + text: { options: '', extension: 'txt' } + }.freeze + + STATS_DEFAULT_FORMAT = :json + + # Return jemalloc stats as a string. + def stats(format: STATS_DEFAULT_FORMAT) + verify_format!(format) + + with_malloc_stats_print do |stats_print| + StringIO.new.tap { |io| write_stats(stats_print, io, STATS_FORMATS[format]) }.string + end + end + + # Write jemalloc stats to the given directory. + def dump_stats(path:, format: STATS_DEFAULT_FORMAT) + verify_format!(format) + + with_malloc_stats_print do |stats_print| + format_settings = STATS_FORMATS[format] + File.open(File.join(path, file_name(format_settings[:extension])), 'wb') do |io| + write_stats(stats_print, io, format_settings) + end + end + end + + private + + def verify_format!(format) + raise "format must be one of #{STATS_FORMATS.keys}" unless STATS_FORMATS.key?(format) + end + + def with_malloc_stats_print + fiddle_func = malloc_stats_print + return unless fiddle_func + + yield fiddle_func + end + + def malloc_stats_print + method = Fiddle::Handle.sym("malloc_stats_print") + + Fiddle::Function.new( + method, + # C signature: + # void (write_cb_t *write_cb, void *cbopaque, const char *opts) + # arg1: callback function pointer (see below) + # arg2: pointer to cbopaque holding additional callback data; always NULL here + # arg3: options string, affects output format (text or JSON) + # + # Callback signature (write_cb_t): + # void (void *, const char *) + # arg1: pointer to cbopaque data (see above; unused) + # arg2: pointer to string buffer holding textual output + [Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP], + Fiddle::TYPE_VOID + ) + rescue Fiddle::DLError + # This means the Fiddle::Handle to jemalloc was not open (jemalloc wasn't loaded) + # or already closed. Eiher way, return nil. + end + + def write_stats(stats_print, io, format) + callback = Fiddle::Closure::BlockCaller.new( + Fiddle::TYPE_VOID, [Fiddle::TYPE_VOIDP, Fiddle::TYPE_VOIDP]) do |_, fragment| + io << fragment + end + + stats_print.call(callback, nil, format[:options]) + end + + def file_name(extension) + "jemalloc_stats.#{$$}.#{Time.current.to_i}.#{extension}" + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/alerts_inserter.rb b/lib/gitlab/metrics/dashboard/stages/alerts_inserter.rb deleted file mode 100644 index 38736158c3b..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/alerts_inserter.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -require 'set' - -module Gitlab - module Metrics - module Dashboard - module Stages - class AlertsInserter < BaseStage - include ::Gitlab::Utils::StrongMemoize - - def transform! - return if metrics_with_alerts.empty? - - for_metrics do |metric| - next unless metrics_with_alerts.include?(metric[:metric_id]) - - metric[:alert_path] = alert_path(metric[:metric_id], project, params[:environment]) - end - end - - private - - def metrics_with_alerts - strong_memoize(:metrics_with_alerts) do - alerts = ::Projects::Prometheus::AlertsFinder - .new(project: project, environment: params[:environment]) - .execute - - Set.new(alerts.map(&:prometheus_metric_id)) - end - end - - def alert_path(metric_id, project, environment) - ::Gitlab::Routing.url_helpers.project_prometheus_alert_path(project, metric_id, environment_id: environment.id, format: :json) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 965d85e20e5..86372973c82 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -72,7 +72,7 @@ module Gitlab { host: host.host, port: host.port, - class: load_balancer.configuration.primary_connection_specification_name, + class: load_balancer.configuration.connection_specification_name, db_config_name: Gitlab::Database.db_config_name(host.connection) } end diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb index fcd893b675f..2de19514354 100644 --- a/lib/gitlab/metrics/sli.rb +++ b/lib/gitlab/metrics/sli.rb @@ -68,10 +68,6 @@ module Gitlab prometheus.counter(counter_name('total'), "Total number of measurements for #{name}") end - def counter_name(suffix) - :"#{COUNTER_PREFIX}:#{name}_#{self.class.name.demodulize.underscore}:#{suffix}" - end - def prometheus Gitlab::Metrics end @@ -85,6 +81,10 @@ module Gitlab private + def counter_name(suffix) + :"#{COUNTER_PREFIX}:#{name}_apdex:#{suffix}" + end + def numerator_counter prometheus.counter(counter_name('success_total'), "Number of successful measurements for #{name}") end @@ -99,6 +99,10 @@ module Gitlab private + def counter_name(suffix) + :"#{COUNTER_PREFIX}:#{name}:#{suffix}" + end + def numerator_counter prometheus.counter(counter_name('error_total'), "Number of error measurements for #{name}") end diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index ef6e0db5673..f66dfe44054 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -54,7 +54,8 @@ module Gitlab end def match_content_type?(env) - env['CONTENT_TYPE'] == 'application/json' || + env['CONTENT_TYPE'].nil? || + env['CONTENT_TYPE'] == 'application/json' || env['CONTENT_TYPE'] == 'application/x-sentry-envelope' end diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index 693f1470d9d..9a850246221 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -92,6 +92,14 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Returns a relation that includes ID of the descendants_base set of objects + # and all their descendants IDs (recursively). + # rubocop: disable CodeReuse/ActiveRecord + def base_and_descendant_ids + read_only(base_and_descendant_ids_cte.apply_to(unscoped_model.select(objects_table[:id]))) + end + # rubocop: enable CodeReuse/ActiveRecord + # Returns a relation that includes the base objects, their ancestors, # and the descendants of the base objects. # @@ -214,6 +222,26 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord + def base_and_descendant_ids_cte + cte = SQL::RecursiveCTE.new(:base_and_descendants) + + base_query = descendants_base.except(:order).select(objects_table[:id]) + + cte << base_query + + # Recursively get all the descendants of the base set. + descendants_query = unscoped_model + .select(objects_table[:id]) + .from(from_tables(cte)) + .where(descendant_conditions(cte)) + .except(:order) + + cte << descendants_query + cte + end + # rubocop: enable CodeReuse/ActiveRecord + def objects_table model.arel_table end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb deleted file mode 100644 index ae5539c03b1..00000000000 --- a/lib/gitlab/pages_transfer.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -# To make a call happen in a new Sidekiq job, add `.async` before the call. For -# instance: -# -# PagesTransfer.new.async.move_namespace(...) -# -module Gitlab - class PagesTransfer < ProjectTransfer - METHODS = %w[move_namespace move_project rename_project rename_namespace].freeze - - class Async - METHODS.each do |meth| - define_method meth do |*args| - next unless Settings.pages.local_store.enabled - - PagesTransferWorker.perform_async(meth, args) - end - end - end - - METHODS.each do |meth| - define_method meth do |*args| - next unless Settings.pages.local_store.enabled - - super(*args) - end - end - - def async - @async ||= Async.new - end - - def root_dir - Gitlab.config.pages.path - end - end -end diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb index c5c73d50518..20d8f7be8fd 100644 --- a/lib/gitlab/patch/database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -1,77 +1,15 @@ # frozen_string_literal: true -# The purpose of this code is to transform legacy `database.yml` -# into a `database.yml` containing `main:` as a name of a first database -# -# This should be removed once all places using legacy `database.yml` -# are fixed. The likely moment to remove this check is the %14.0. -# -# This converts the following syntax: -# -# production: -# adapter: postgresql -# database: gitlabhq_production -# username: git -# password: "secure password" -# host: localhost -# -# Into: -# -# production: -# main: -# adapter: postgresql -# database: gitlabhq_production -# username: git -# password: "secure password" -# host: localhost -# - +# The purpose of this code is to set the migrations path +# for the Geo tracking database. module Gitlab module Patch module DatabaseConfig extend ActiveSupport::Concern - def load_database_yaml - return super unless Gitlab.ee? - - super.deep_merge(load_geo_database_yaml) - end - - # This method is taken from Rails to load a database YAML file without - # evaluating ERB. This allows us to create the rake tasks for the Geo - # tracking database without filling in the configuration values or - # loading the environment. To be removed when we start configure Geo - # tracking database in database.yml instead of custom database_geo.yml - # - # https://github.com/rails/rails/blob/v6.1.4/railties/lib/rails/application/configuration.rb#L255 - def load_geo_database_yaml - path = Rails.root.join("config/database_geo.yml") - return {} unless File.exist?(path) - - require "rails/application/dummy_erb_compiler" - - yaml = DummyERB.new(Pathname.new(path).read).result - config = YAML.load(yaml) || {} # rubocop:disable Security/YAMLLoad - - config.to_h do |env, configs| - # This check is taken from Rails where the transformation - # of a flat database.yml is done into `primary:` - # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 - if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } - configs = { "geo" => configs } - end - - [env, configs] - end - end - def database_configuration super.to_h do |env, configs| if Gitlab.ee? - if !configs.key?("geo") && File.exist?(Rails.root.join("config/database_geo.yml")) - configs["geo"] = Rails.application.config_for(:database_geo).stringify_keys - end - if configs.key?("geo") migrations_paths = Array(configs["geo"]["migrations_paths"]) migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty? diff --git a/lib/gitlab/project_stats_refresh_conflicts_logger.rb b/lib/gitlab/project_stats_refresh_conflicts_logger.rb new file mode 100644 index 00000000000..3e7eecce89c --- /dev/null +++ b/lib/gitlab/project_stats_refresh_conflicts_logger.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + class ProjectStatsRefreshConflictsLogger # rubocop:disable Gitlab/NamespacedClass + def self.warn_artifact_deletion_during_stats_refresh(project_id:, method:) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Deleted artifacts undergoing refresh', + method: method, + project_id: project_id + ) + + Gitlab::AppLogger.warn(payload) + end + + def self.warn_request_rejected_during_stats_refresh(project_id) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Rejected request due to project undergoing stats refresh', + project_id: project_id + ) + + Gitlab::AppLogger.warn(payload) + end + + def self.warn_skipped_artifact_deletion_during_stats_refresh(project_ids:, method:) + payload = Gitlab::ApplicationContext.current.merge( + message: 'Skipped deleting artifacts undergoing refresh', + method: method, + project_ids: project_ids + ) + + Gitlab::AppLogger.warn(payload) + end + end +end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index e7a12edf763..0ab6055408f 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -42,6 +42,7 @@ module Gitlab class << self # TODO: Review child inheritance of this table (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430928221) + # rubocop:disable Metrics/AbcSize def localized_templates_table [ ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'), @@ -53,6 +54,7 @@ module Gitlab ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), + ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'), ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'), ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), @@ -72,6 +74,7 @@ module Gitlab ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux') ].freeze end + # rubocop:enable Metrics/AbcSize def all localized_templates_table diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index efeb1e07d49..5bcbf7b5cca 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -2,14 +2,42 @@ module Gitlab module ProtocolAccess - def self.allowed?(protocol) - if protocol == 'web' - true - elsif Gitlab::CurrentSettings.enabled_git_access_protocol.blank? + class << self + def allowed?(protocol, project: nil) + # Web is always allowed + return true if protocol == "web" + + # System settings + return false unless instance_allowed?(protocol) + + # Group-level settings + return false unless namespace_allowed?(protocol, namespace: project&.root_namespace) + + # Default to allowing all protocols true - else + end + + private + + def instance_allowed?(protocol) + # If admin hasn't configured this setting, default to true + return true if Gitlab::CurrentSettings.enabled_git_access_protocol.blank? + protocol == Gitlab::CurrentSettings.enabled_git_access_protocol end + + def namespace_allowed?(protocol, namespace: nil) + # If the namespace parameter was nil, we default to true here + return true if namespace.nil? + + # Return immediately if all protocols are allowed + return true if namespace.enabled_git_access_protocol == "all" + + # If the setting is somehow nil, such as in an unsaved state, we default to allow + return true if namespace.enabled_git_access_protocol.blank? + + protocol == namespace.enabled_git_access_protocol + end end end end diff --git a/lib/gitlab/quick_actions/commit_actions.rb b/lib/gitlab/quick_actions/commit_actions.rb index 49f5ddf24eb..661e768ffc4 100644 --- a/lib/gitlab/quick_actions/commit_actions.rb +++ b/lib/gitlab/quick_actions/commit_actions.rb @@ -8,7 +8,7 @@ module Gitlab included do # Commit only quick actions definitions - desc _('Tag this commit.') + desc { _('Tag this commit.') } explanation do |tag_name, message| if message.present? _("Tags this commit to %{tag_name} with \"%{message}\".") % { tag_name: tag_name, message: message } diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 4bac0643a91..259d9e38d65 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -55,7 +55,7 @@ module Gitlab @updates[:state_event] = 'reopen' end - desc _('Change title') + desc { _('Change title') } explanation do |title_param| _('Changes the title to "%{title_param}".') % { title_param: title_param } end @@ -72,7 +72,7 @@ module Gitlab @updates[:title] = title_param end - desc _('Add label(s)') + desc { _('Add label(s)') } explanation do |labels_param| labels = find_label_references(labels_param) @@ -91,7 +91,7 @@ module Gitlab run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end - desc _('Remove all or specific label(s)') + desc { _('Remove all or specific label(s)') } explanation do |labels_param = nil| label_references = labels_param.present? ? find_label_references(labels_param) : [] if label_references.any? @@ -128,7 +128,7 @@ module Gitlab @execution_message[:unlabel] = remove_label_message(label_references) end - desc _('Replace all label(s)') + desc { _('Replace all label(s)') } explanation do |labels_param| labels = find_label_references(labels_param) "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? @@ -144,9 +144,9 @@ module Gitlab run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids) end - desc _('Add a to do') - explanation _('Adds a to do.') - execution_message _('Added a to do.') + desc { _('Add a to do') } + explanation { _('Adds a to do.') } + execution_message { _('Added a to do.') } types Issuable condition do quick_action_target.persisted? && @@ -156,9 +156,9 @@ module Gitlab @updates[:todo_event] = 'add' end - desc _('Mark to do as done') - explanation _('Marks to do as done.') - execution_message _('Marked to do as done.') + desc { _('Mark to do as done') } + explanation { _('Marks to do as done.') } + execution_message { _('Marked to do as done.') } types Issuable condition do quick_action_target.persisted? && @@ -168,7 +168,7 @@ module Gitlab @updates[:todo_event] = 'done' end - desc _('Subscribe') + desc { _('Subscribe') } explanation do _('Subscribes to this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } @@ -186,7 +186,7 @@ module Gitlab @updates[:subscription_event] = 'subscribe' end - desc _('Unsubscribe') + desc { _('Unsubscribe') } explanation do _('Unsubscribes from this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } @@ -204,7 +204,7 @@ module Gitlab @updates[:subscription_event] = 'unsubscribe' end - desc _('Toggle emoji award') + desc { _('Toggle emoji award') } explanation do |name| _("Toggles :%{name}: emoji award.") % { name: name } if name end @@ -226,22 +226,22 @@ module Gitlab end end - desc _("Append the comment with %{shrug}") % { shrug: SHRUG } + desc { _("Append the comment with %{shrug}") % { shrug: SHRUG } } params '<Comment>' types Issuable substitution :shrug do |comment| "#{comment} #{SHRUG}" end - desc _("Append the comment with %{tableflip}") % { tableflip: TABLEFLIP } + desc { _("Append the comment with %{tableflip}") % { tableflip: TABLEFLIP } } params '<Comment>' types Issuable substitution :tableflip do |comment| "#{comment} #{TABLEFLIP}" end - desc _('Set severity') - explanation _('Sets the severity') + desc { _('Set severity') } + explanation { _('Sets the severity') } params '1 / S1 / Critical' types Issue condition do diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 2f89774a257..189627506f3 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -8,7 +8,7 @@ module Gitlab included do # Issue only quick actions definition - desc _('Set due date') + desc { _('Set due date') } explanation do |due_date| _("Sets the due date to %{due_date}.") % { due_date: due_date.strftime('%b %-d, %Y') } if due_date end @@ -32,9 +32,9 @@ module Gitlab end end - desc _('Remove due date') - explanation _('Removes the due date.') - execution_message _('Removed the due date.') + desc { _('Remove due date') } + explanation { _('Removes the due date.') } + execution_message { _('Removed the due date.') } types Issue condition do quick_action_target.persisted? && @@ -46,7 +46,7 @@ module Gitlab @updates[:due_date] = nil end - desc _('Move issue from one column of the board to another') + desc { _('Move issue from one column of the board to another') } explanation do |target_list_name| label = find_label_references(target_list_name).first _("Moves issue to %{label} column in the board.") % { label: label } if label @@ -78,7 +78,7 @@ module Gitlab @execution_message[:board_move] = message end - desc _('Mark this issue as a duplicate of another issue') + desc { _('Mark this issue as a duplicate of another issue') } explanation do |duplicate_reference| _("Marks this issue as a duplicate of %{duplicate_reference}.") % { duplicate_reference: duplicate_reference } end @@ -102,7 +102,7 @@ module Gitlab @execution_message[:duplicate] = message end - desc _('Clone this issue') + desc { _('Clone this issue') } explanation do |project = quick_action_target.project.full_path| _("Clones this issue, without comments, to %{project}.") % { project: project } end @@ -137,7 +137,7 @@ module Gitlab @execution_message[:clone] = message end - desc _('Move this issue to another project.') + desc { _('Move this issue to another project.') } explanation do |path_to_project| _("Moves this issue to %{path_to_project}.") % { path_to_project: path_to_project } end @@ -161,7 +161,7 @@ module Gitlab @execution_message[:move] = message end - desc _('Make issue confidential') + desc { _('Make issue confidential') } explanation do _('Makes this issue confidential.') end @@ -178,7 +178,7 @@ module Gitlab @updates[:confidential] = true end - desc _('Create a merge request') + desc { _('Create a merge request') } explanation do |branch_name = nil| if branch_name _("Creates branch '%{branch_name}' and a merge request to resolve this issue.") % { branch_name: branch_name } @@ -205,8 +205,8 @@ module Gitlab } end - desc _('Add Zoom meeting') - explanation _('Adds a Zoom meeting.') + desc { _('Add Zoom meeting') } + explanation { _('Adds a Zoom meeting.') } params '<Zoom URL>' types Issue condition do @@ -222,9 +222,9 @@ module Gitlab @updates.merge!(result.payload) if result.payload end - desc _('Remove Zoom meeting') - explanation _('Remove Zoom meeting.') - execution_message _('Zoom meeting removed') + desc { _('Remove Zoom meeting') } + explanation { _('Remove Zoom meeting.') } + execution_message { _('Zoom meeting removed') } types Issue condition do @zoom_service = zoom_link_service @@ -235,8 +235,8 @@ module Gitlab @execution_message[:remove_zoom] = result.message end - desc _('Add email participant(s)') - explanation _('Adds email participant(s).') + desc { _('Add email participant(s)') } + explanation { _('Adds email participant(s).') } params 'email1@example.com email2@example.com (up to 6 emails)' types Issue condition do @@ -264,8 +264,8 @@ module Gitlab end end - desc _('Promote issue to incident') - explanation _('Promotes issue to incident') + desc { _('Promote issue to incident') } + explanation { _('Promotes issue to incident') } types Issue condition do quick_action_target.persisted? && @@ -285,8 +285,8 @@ module Gitlab end end - desc _('Add customer relation contacts') - explanation _('Add customer relation contact(s).') + desc { _('Add customer relation contacts') } + explanation { _('Add customer relation contact(s).') } params '[contact:contact@example.com] [contact:person@example.org]' types Issue condition do @@ -300,13 +300,13 @@ module Gitlab @updates[:add_contacts] = contact_emails.split(' ') end - desc _('Remove customer relation contacts') - explanation _('Remove customer relation contact(s).') + desc { _('Remove customer relation contacts') } + explanation { _('Remove customer relation contact(s).') } params '[contact:contact@example.com] [contact:person@example.org]' types Issue condition do current_user.can?(:set_issue_crm_contacts, quick_action_target) && - CustomerRelations::Contact.exists_for_group?(quick_action_target.project.root_ancestor) + quick_action_target.customer_relations_contacts.exists? end execution_message do _('One or more contacts were successfully removed.') diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 4a75fa0a571..a0faf8dd460 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -8,7 +8,7 @@ module Gitlab included do # Issue, MergeRequest: quick actions definitions - desc _('Assign') + desc { _('Assign') } explanation do |users| _('Assigns %{assignee_users_sentence}.') % { assignee_users_sentence: assignee_users_sentence(users) } end @@ -81,7 +81,7 @@ module Gitlab end end - desc _('Set milestone') + desc { _('Set milestone') } explanation do |milestone| _("Sets the milestone to %{milestone_reference}.") % { milestone_reference: milestone.to_reference } if milestone end @@ -103,7 +103,7 @@ module Gitlab @updates[:milestone_id] = milestone.id if milestone end - desc _('Remove milestone') + desc { _('Remove milestone') } explanation do _("Removes %{milestone_reference} milestone.") % { milestone_reference: quick_action_target.milestone.to_reference(format: :name) } end @@ -121,7 +121,7 @@ module Gitlab @updates[:milestone_id] = nil end - desc _('Copy labels and milestone from other issue or merge request in this project') + desc { _('Copy labels and milestone from other issue or merge request in this project') } explanation do |source_issuable| _("Copy labels and milestone from %{source_issuable_reference}.") % { source_issuable_reference: source_issuable.to_reference } end @@ -143,7 +143,7 @@ module Gitlab end end - desc _('Set time estimate') + desc { _('Set time estimate') } explanation do |time_estimate| formatted_time_estimate = format_time_estimate(time_estimate) _("Sets time estimate to %{time_estimate}.") % { time_estimate: formatted_time_estimate } if formatted_time_estimate @@ -167,7 +167,7 @@ module Gitlab end end - desc _('Add or subtract spent time') + desc { _('Add or subtract spent time') } explanation do |time_spent, time_spent_date| spend_time_message(time_spent, time_spent_date, false) end @@ -194,9 +194,9 @@ module Gitlab end end - desc _('Remove time estimate') - explanation _('Removes time estimate.') - execution_message _('Removed time estimate.') + desc { _('Remove time estimate') } + explanation { _('Removes time estimate.') } + execution_message { _('Removed time estimate.') } types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -206,9 +206,9 @@ module Gitlab @updates[:time_estimate] = 0 end - desc _('Remove spent time') - explanation _('Removes spent time.') - execution_message _('Removed spent time.') + desc { _('Remove spent time') } + explanation { _('Removes spent time.') } + execution_message { _('Removed spent time.') } condition do quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) @@ -218,9 +218,9 @@ module Gitlab @updates[:spend_time] = { duration: :reset, user_id: current_user.id } end - desc _("Lock the discussion") - explanation _("Locks the discussion.") - execution_message _("Locked the discussion.") + desc { _("Lock the discussion") } + explanation { _("Locks the discussion.") } + execution_message { _("Locked the discussion.") } types Issue, MergeRequest condition do quick_action_target.persisted? && @@ -231,9 +231,9 @@ module Gitlab @updates[:discussion_locked] = true end - desc _("Unlock the discussion") - explanation _("Unlocks the discussion.") - execution_message _("Unlocked the discussion.") + desc { _("Unlock the discussion") } + explanation { _("Unlocks the discussion.") } + execution_message { _("Unlocked the discussion.") } types Issue, MergeRequest condition do quick_action_target.persisted? && diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index abf55f56c73..167e7ad67a9 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -88,19 +88,19 @@ module Gitlab @execution_message[:rebase] = _('Scheduled a rebase of branch %{branch}.') % { branch: branch } end - desc 'Toggle the Draft status' + desc { _('Toggle the Draft status') } explanation do noun = quick_action_target.to_ability_name.humanize(capitalize: false) - if quick_action_target.work_in_progress? - _("Unmarks this %{noun} as a draft.") + if quick_action_target.draft? + _("Marks this %{noun} as ready.") else _("Marks this %{noun} as a draft.") end % { noun: noun } end execution_message do noun = quick_action_target.to_ability_name.humanize(capitalize: false) - if quick_action_target.work_in_progress? - _("Unmarked this %{noun} as a draft.") + if quick_action_target.draft? + _("Marked this %{noun} as ready.") else _("Marked this %{noun} as a draft.") end % { noun: noun } @@ -108,15 +108,45 @@ module Gitlab types MergeRequest condition do - quick_action_target.respond_to?(:work_in_progress?) && - # Allow it to mark as WIP on MR creation page _or_ through MR notes. + quick_action_target.respond_to?(:draft?) && + # Allow it to mark as draft on MR creation page or through MR notes + # (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end command :draft do - @updates[:wip_event] = quick_action_target.work_in_progress? ? 'unwip' : 'wip' + @updates[:wip_event] = quick_action_target.draft? ? 'ready' : 'draft' end - desc _('Set target branch') + desc { _('Set the Ready status') } + explanation do + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if quick_action_target.draft? + _("Marks this %{noun} as ready.") + else + _("No change to this %{noun}'s draft status.") + end % { noun: noun } + end + execution_message do + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if quick_action_target.draft? + _("Marked this %{noun} as ready.") + else + _("No change to this %{noun}'s draft status.") + end % { noun: noun } + end + + types MergeRequest + condition do + # Allow it to mark as draft on MR creation page or through MR notes + # + quick_action_target.respond_to?(:draft?) && + (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) + end + command :ready do + @updates[:wip_event] = 'ready' if quick_action_target.draft? + end + + desc { _('Set target branch') } explanation do |branch_name| _('Sets target branch to %{branch_name}.') % { branch_name: branch_name } end @@ -137,8 +167,8 @@ module Gitlab @updates[:target_branch] = branch_name if project.repository.branch_exists?(branch_name) end - desc _('Submit a review') - explanation _('Submit the current review.') + desc { _('Submit a review') } + explanation { _('Submit the current review.') } types MergeRequest condition do quick_action_target.persisted? @@ -154,8 +184,8 @@ module Gitlab end end - desc _('Approve a merge request') - explanation _('Approve the current merge request.') + desc { _('Approve a merge request') } + explanation { _('Approve the current merge request.') } types MergeRequest condition do quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) @@ -168,8 +198,8 @@ module Gitlab @execution_message[:approve] = _('Approved the current merge request.') end - desc _('Unapprove a merge request') - explanation _('Unapprove the current merge request.') + desc { _('Unapprove a merge request') } + explanation { _('Unapprove the current merge request.') } types MergeRequest condition do quick_action_target.persisted? && quick_action_target.can_be_unapproved_by?(current_user) diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb index 1de23523f01..4c8035f192e 100644 --- a/lib/gitlab/quick_actions/relate_actions.rb +++ b/lib/gitlab/quick_actions/relate_actions.rb @@ -7,7 +7,7 @@ module Gitlab include ::Gitlab::QuickActions::Dsl included do - desc _('Mark this issue as related to another issue') + desc { _('Mark this issue as related to another issue') } explanation do |related_reference| _('Marks this issue as related to %{issue_ref}.') % { issue_ref: related_reference } end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index b2664f87306..f5fb6b5af3d 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -12,9 +12,9 @@ module Gitlab rack_attack::Request.include(Gitlab::RackAttack::Request) # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response - rack_attack.throttled_response = lambda do |env| + rack_attack.throttled_responder = lambda do |request| throttled_headers = Gitlab::RackAttack.throttled_response_headers( - env['rack.attack.matched'], env['rack.attack.match_data'] + request.env['rack.attack.matched'], request.env['rack.attack.match_data'] ) [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] end diff --git a/lib/gitlab/redis/duplicate_jobs.rb b/lib/gitlab/redis/duplicate_jobs.rb new file mode 100644 index 00000000000..beb3ba1abee --- /dev/null +++ b/lib/gitlab/redis/duplicate_jobs.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + # Pseudo-store to transition `Gitlab::SidekiqMiddleware::DuplicateJobs` from + # using `Sidekiq.redis` to using the `SharedState` redis store. + class DuplicateJobs < ::Gitlab::Redis::Wrapper + class << self + def store_name + 'SharedState' + end + + private + + def redis + primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) + + # `Sidekiq.redis` is a namespaced redis connection. This means keys are actually being stored under + # "resque:gitlab:resque:gitlab:duplicate:". For backwards compatibility, we make the secondary store + # namespaced in the same way, but omit it from the primary so keys have proper format there. + secondary_store = ::Redis::Namespace.new( + Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE, redis: ::Redis.new(Gitlab::Redis::Queues.params) + ) + + MultiStore.new(primary_store, secondary_store, name.demodulize) + end + end + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb new file mode 100644 index 00000000000..24c540eea47 --- /dev/null +++ b/lib/gitlab/redis/multi_store.rb @@ -0,0 +1,300 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class MultiStore + include Gitlab::Utils::StrongMemoize + + class ReadFromPrimaryError < StandardError + def message + 'Value not found on the redis primary store. Read from the redis secondary store successful.' + end + end + class PipelinedDiffError < StandardError + def message + 'Pipelined command executed on both stores successfully but results differ between them.' + end + end + class MethodMissingError < StandardError + def message + 'Method missing. Falling back to execute method on the redis secondary store.' + end + end + + attr_reader :primary_store, :secondary_store, :instance_name + + FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' + FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' + FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' + + SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i(info).freeze + + READ_COMMANDS = %i( + get + mget + smembers + scard + ).freeze + + WRITE_COMMANDS = %i( + set + setnx + setex + sadd + srem + del + flushdb + rpush + ).freeze + + PIPELINED_COMMANDS = %i( + pipelined + multi + ).freeze + + # To transition between two Redis store, `primary_store` should be the target store, + # and `secondary_store` should be the current store. Transition is controlled with feature flags: + # + # - At the default state, all read and write operations are executed in the secondary instance. + # - Turning use_primary_and_secondary_stores_for_<instance_name> on: The store writes to both instances. + # The read commands are executed in primary, but fallback to secondary. + # Other commands are executed in the the default instance (Secondary). + # - Turning use_primary_store_as_default_for_<instance_name> on: The behavior is the same as above, + # but other commands are executed in the primary now. + # - Turning use_primary_and_secondary_stores_for_<instance_name> off: commands are executed in the primary store. + def initialize(primary_store, secondary_store, instance_name) + @primary_store = primary_store + @secondary_store = secondary_store + @instance_name = instance_name + + validate_stores! + end + + # rubocop:disable GitlabSecurity/PublicSend + READ_COMMANDS.each do |name| + define_method(name) do |*args, &block| + if use_primary_and_secondary_stores? + read_command(name, *args, &block) + else + default_store.send(name, *args, &block) + end + end + end + + WRITE_COMMANDS.each do |name| + define_method(name) do |*args, **kwargs, &block| + if use_primary_and_secondary_stores? + write_command(name, *args, **kwargs, &block) + else + default_store.send(name, *args, **kwargs, &block) + end + end + end + + PIPELINED_COMMANDS.each do |name| + define_method(name) do |*args, **kwargs, &block| + if use_primary_and_secondary_stores? + pipelined_both(name, *args, **kwargs, &block) + else + default_store.send(name, *args, **kwargs, &block) + end + end + end + + def method_missing(...) + return @instance.send(...) if @instance + + log_method_missing(...) + + default_store.send(...) + end + # rubocop:enable GitlabSecurity/PublicSend + + def respond_to_missing?(command_name, include_private = false) + true + end + + # This is needed because of Redis::Rack::Connection is requiring Redis::Store + # https://github.com/redis-store/redis-rack/blob/a833086ba494083b6a384a1a4e58b36573a9165d/lib/redis/rack/connection.rb#L15 + # Done similarly in https://github.com/lsegal/yard/blob/main/lib/yard/templates/template.rb#L122 + def is_a?(klass) + return true if klass == default_store.class + + super(klass) + end + alias_method :kind_of?, :is_a? + + def to_s + use_primary_and_secondary_stores? ? primary_store.to_s : default_store.to_s + end + + def use_primary_and_secondary_stores? + feature_enabled?("use_primary_and_secondary_stores_for") + end + + def use_primary_store_as_default? + feature_enabled?("use_primary_store_as_default_for") + end + + def increment_pipelined_command_error_count(command_name) + @pipelined_command_error ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_pipelined_diff_error_total, + 'Redis MultiStore pipelined command diff between stores') + @pipelined_command_error.increment(command: command_name, instance_name: instance_name) + end + + def increment_read_fallback_count(command_name) + @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total, + 'Client side Redis MultiStore reading fallback') + @read_fallback_counter.increment(command: command_name, instance_name: instance_name) + end + + def increment_method_missing_count(command_name) + @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, + 'Client side Redis MultiStore method missing') + @method_missing_counter.increment(command: command_name, instance_name: instance_name) + end + + def log_error(exception, command_name, extra = {}) + Gitlab::ErrorTracking.log_exception( + exception, + extra.merge(command_name: command_name, instance_name: instance_name)) + end + + private + + # @return [Boolean] + def feature_enabled?(prefix) + feature_table_exists? && + Feature.enabled?("#{prefix}_#{instance_name.underscore}") && + !same_redis_store? + end + + # @return [Boolean] + def feature_table_exists? + Feature::FlipperFeature.table_exists? + rescue StandardError + false + end + + def default_store + use_primary_store_as_default? ? primary_store : secondary_store + end + + def log_method_missing(command_name, *_args) + return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) + + log_error(MethodMissingError.new, command_name) + increment_method_missing_count(command_name) + end + + def read_command(command_name, *args, &block) + if @instance + send_command(@instance, command_name, *args, &block) + else + read_one_with_fallback(command_name, *args, &block) + end + end + + def write_command(command_name, *args, **kwargs, &block) + if @instance + send_command(@instance, command_name, *args, **kwargs, &block) + else + write_both(command_name, *args, **kwargs, &block) + end + end + + def read_one_with_fallback(command_name, *args, &block) + begin + value = send_command(primary_store, command_name, *args, &block) + rescue StandardError => e + log_error(e, command_name, + multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) + end + + value || fallback_read(command_name, *args, &block) + end + + def fallback_read(command_name, *args, &block) + value = send_command(secondary_store, command_name, *args, &block) + + if value + log_error(ReadFromPrimaryError.new, command_name) + increment_read_fallback_count(command_name) + end + + value + end + + def write_both(command_name, *args, **kwargs, &block) + begin + send_command(primary_store, command_name, *args, **kwargs, &block) + rescue StandardError => e + log_error(e, command_name, + multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE) + end + + send_command(secondary_store, command_name, *args, **kwargs, &block) + end + + # Run the entire pipeline on both stores. We assume that `&block` is idempotent. + def pipelined_both(command_name, *args, **kwargs, &block) + begin + result_primary = send_command(primary_store, command_name, *args, **kwargs, &block) + rescue StandardError => e + log_error(e, command_name, multi_store_error_message: FAILED_TO_RUN_PIPELINE) + end + + result_secondary = send_command(secondary_store, command_name, *args, **kwargs, &block) + + # Pipelined commands return an array with all results. If they differ, + # log an error + if result_primary != result_secondary + log_error(PipelinedDiffError.new, command_name) + increment_pipelined_command_error_count(command_name) + end + + result_secondary + end + + def same_redis_store? + strong_memoize(:same_redis_store) do + # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>" + primary_store.inspect == secondary_store.inspect + end + end + + # rubocop:disable GitlabSecurity/PublicSend + def send_command(redis_instance, command_name, *args, **kwargs, &block) + if block_given? + # Make sure that block is wrapped and executed only on the redis instance that is executing the block + redis_instance.send(command_name, *args, **kwargs) do |*params| + with_instance(redis_instance, *params, &block) + end + else + redis_instance.send(command_name, *args, **kwargs) + end + end + # rubocop:enable GitlabSecurity/PublicSend + + def with_instance(instance, *params) + @instance = instance + + yield(*params) + ensure + @instance = nil + end + + def redis_store?(store) + store.is_a?(::Redis) || store.is_a?(::Redis::Namespace) + end + + def validate_stores! + raise ArgumentError, 'primary_store is required' unless primary_store + raise ArgumentError, 'secondary_store is required' unless secondary_store + raise ArgumentError, 'instance_name is required' unless instance_name + raise ArgumentError, 'invalid primary_store' unless redis_store?(primary_store) + raise ArgumentError, 'invalid secondary_store' unless redis_store?(secondary_store) + end + end + end +end diff --git a/lib/gitlab/redis/sidekiq_status.rb b/lib/gitlab/redis/sidekiq_status.rb new file mode 100644 index 00000000000..d4362c7cad8 --- /dev/null +++ b/lib/gitlab/redis/sidekiq_status.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + # Pseudo-store to transition `Gitlab::SidekiqStatus` from + # using `Sidekiq.redis` to using the `SharedState` redis store. + class SidekiqStatus < ::Gitlab::Redis::Wrapper + class << self + def store_name + 'SharedState' + end + + private + + def redis + primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) + secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) + + MultiStore.new(primary_store, secondary_store, name.demodulize) + end + end + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 205106afddb..b0f4194b7a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,7 +5,7 @@ module Gitlab module Packages CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze - + PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze def conan_package_reference_regex @@ -119,9 +119,9 @@ module Gitlab # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205 @debian_version_regex ||= %r{ \A(?: - (?:([0-9]{1,9}):)? (?# epoch) - ([0-9][0-9a-z\.+~-]*) (?# version) - (?:(-[0-0a-z\.+~]+))? (?# revision) + (?:([0-9]{1,9}):)? (?# epoch) + ([0-9][0-9a-z\.+~]*-?){1,15} (?# version-revision) + (?<!-) )\z}xi.freeze end @@ -481,6 +481,11 @@ module Gitlab "can contain only lowercase letters, digits, '_' and '-'. " \ "Must start with a letter, and cannot end with '-' or '_'" end + + # One or more `part`s, separated by separator + def sep_by_1(separator, part) + %r(#{part} (#{separator} #{part})*)x + end end end diff --git a/lib/gitlab/render_timeout.rb b/lib/gitlab/render_timeout.rb new file mode 100644 index 00000000000..b3c2a5b4c2a --- /dev/null +++ b/lib/gitlab/render_timeout.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module RenderTimeout + BACKGROUND = 30.seconds + FOREGROUND = 1.5.seconds + + def self.timeout(background: BACKGROUND, foreground: FOREGROUND, &block) + period = Gitlab::Runtime.sidekiq? ? background : foreground + + Timeout.timeout(period, &block) + end + end +end diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb index 52da10eff3e..14f07140825 100644 --- a/lib/gitlab/service_desk_email.rb +++ b/lib/gitlab/service_desk_email.rb @@ -23,6 +23,12 @@ module Gitlab config.address.sub(Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER, key) end + + def key_from_fallback_message_id(mail_id) + message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ + + mail_id[message_id_regexp, 1] + end end end end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index cfe91b9a266..de08de6632b 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -11,7 +11,11 @@ module Gitlab def parse_job(job) # Error information from the previous try is in the payload for # displaying in the Sidekiq UI, but is very confusing in logs! - job = job.except('error_backtrace', 'error_class', 'error_message') + job = job.except( + 'error_backtrace', 'error_class', 'error_message', + 'exception.backtrace', 'exception.class', 'exception.message', 'exception.sql' + ) + job['class'] = job.delete('wrapped') if job['wrapped'].present? job['job_size_bytes'] = Sidekiq.dump_json(job['args']).bytesize diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index a9bfcce2e0a..6eb39981ef4 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -79,9 +79,14 @@ module Gitlab if job_exception payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" payload['job_status'] = 'fail' - payload['error_message'] = job_exception.message - payload['error_class'] = job_exception.class.name - add_exception_backtrace!(job_exception, payload) + + Gitlab::ExceptionLogFormatter.format!(job_exception, payload) + + # Deprecated fields for compatibility + # See https://gitlab.com/gitlab-org/gitlab/-/issues/364241 + payload['error_class'] = payload['exception.class'] + payload['error_message'] = payload['exception.message'] + payload['error_backtrace'] = payload['exception.backtrace'] else payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' @@ -98,12 +103,6 @@ module Gitlab payload['completed_at'] = Time.now.utc.to_f end - def add_exception_backtrace!(job_exception, payload) - return if job_exception.backtrace.blank? - - payload['error_backtrace'] = Rails.backtrace_cleaner.clean(job_exception.backtrace) - end - def elapsed(t0) t1 = get_time { duration: t1[:now] - t0[:now] } diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 601c8d1c3cf..7533770e254 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -63,7 +63,7 @@ module Gitlab read_jid = nil read_wal_locations = {} - Sidekiq.redis do |redis| + with_redis do |redis| redis.multi do |multi| multi.set(idempotency_key, jid, ex: expiry, nx: true) read_wal_locations = check_existing_wal_locations!(multi, expiry) @@ -81,7 +81,7 @@ module Gitlab def update_latest_wal_location! return unless job_wal_locations.present? - Sidekiq.redis do |redis| + with_redis do |redis| redis.multi do |multi| job_wal_locations.each do |connection_name, location| multi.eval( @@ -100,20 +100,19 @@ module Gitlab strong_memoize(:latest_wal_locations) do read_wal_locations = {} - Sidekiq.redis do |redis| + with_redis do |redis| redis.multi do |multi| job_wal_locations.keys.each do |connection_name| read_wal_locations[connection_name] = multi.lindex(wal_location_key(connection_name), 0) end end end - read_wal_locations.transform_values(&:value).compact end end def delete! - Sidekiq.redis do |redis| + with_redis do |redis| redis.multi do |multi| multi.del(idempotency_key, deduplicated_flag_key) delete_wal_locations!(multi) @@ -140,7 +139,7 @@ module Gitlab def set_deduplicated_flag!(expiry = duplicate_key_ttl) return unless reschedulable? - Sidekiq.redis do |redis| + with_redis do |redis| redis.set(deduplicated_flag_key, DEDUPLICATED_FLAG_VALUE, ex: expiry, nx: true) end end @@ -148,7 +147,7 @@ module Gitlab def should_reschedule? return false unless reschedulable? - Sidekiq.redis do |redis| + with_redis do |redis| redis.get(deduplicated_flag_key).present? end end @@ -272,6 +271,18 @@ module Gitlab def reschedulable? !scheduled? && options[:if_deduplicated] == :reschedule_once end + + def with_redis + if Feature.enabled?(:use_primary_and_secondary_stores_for_duplicate_jobs) || + Feature.enabled?(:use_primary_store_as_default_for_duplicate_jobs) + # TODO: Swap for Gitlab::Redis::SharedState after store transition + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 + Gitlab::Redis::DuplicateJobs.with { |redis| yield redis } + else + # Keep the old behavior intact if neither feature flag is turned on + Sidekiq.redis { |redis| yield redis } + end + end end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb index 8c7e15364f8..347f4e61d19 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb @@ -10,6 +10,8 @@ module Gitlab class UntilExecuted < DeduplicatesWhenScheduling override :perform def perform(job) + job_deleted = false + super yield @@ -17,7 +19,10 @@ module Gitlab should_reschedule = duplicate_job.should_reschedule? # Deleting before rescheduling to make sure we don't deduplicate again. duplicate_job.delete! + job_deleted = true duplicate_job.reschedule if should_reschedule + ensure + duplicate_job.delete! unless job_deleted end end end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/client.rb b/lib/gitlab/sidekiq_middleware/worker_context/client.rb index 7d3925e9dec..d9797e9c7c7 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/client.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/client.rb @@ -19,14 +19,21 @@ module Gitlab # This should be inside the context for the arguments so # that we don't override the feature category on the worker # with the one from the caller. - # + + root_caller_id = Gitlab::ApplicationContext.current_context_attribute(:root_caller_id) || + Gitlab::ApplicationContext.current_context_attribute(:caller_id) + + context = { + root_caller_id: root_caller_id + } + # We do not want to set anything explicitly in the context # when the feature category is 'not_owned'. - if worker_class.feature_category_not_owned? - yield - else - Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s, &block) + unless worker_class.feature_category_not_owned? + context[:feature_category] = worker_class.get_feature_category.to_s end + + Gitlab::ApplicationContext.with_context(**context, &block) end end end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/server.rb b/lib/gitlab/sidekiq_middleware/worker_context/server.rb index d026f4918c6..7c2dd80c17a 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/server.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/server.rb @@ -12,7 +12,9 @@ module Gitlab # This is not a worker we know about, perhaps from a gem return yield unless worker_class.respond_to?(:get_worker_context) - Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s) do + feature_category = worker_class.get_feature_category.to_s + + Gitlab::ApplicationContext.with_context(feature_category: feature_category) do # Use the context defined on the class level as the more specific context wrap_in_optional_context(worker_class.get_worker_context, &block) end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 66417b3697e..9d08d236720 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -36,7 +36,7 @@ module Gitlab def self.set(jid, expire = DEFAULT_EXPIRATION) return unless expire - Sidekiq.redis do |redis| + with_redis do |redis| redis.set(key_for(jid), 1, ex: expire) end end @@ -45,7 +45,7 @@ module Gitlab # # jid - The Sidekiq job ID to remove. def self.unset(jid) - Sidekiq.redis do |redis| + with_redis do |redis| redis.del(key_for(jid)) end end @@ -94,8 +94,7 @@ module Gitlab keys = job_ids.map { |jid| key_for(jid) } - Sidekiq - .redis { |redis| redis.mget(*keys) } + with_redis { |redis| redis.mget(*keys) } .map { |result| !result.nil? } end @@ -118,5 +117,18 @@ module Gitlab def self.key_for(jid) STATUS_KEY % jid end + + def self.with_redis + if Feature.enabled?(:use_primary_and_secondary_stores_for_sidekiq_status) || + Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) + # TODO: Swap for Gitlab::Redis::SharedState after store transition + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 + Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } + else + # Keep the old behavior intact if neither feature flag is turned on + Sidekiq.redis { |redis| yield redis } + end + end + private_class_method :with_redis end end diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb index 8f37602aeaa..67f61e0db40 100644 --- a/lib/gitlab/sql/cte.rb +++ b/lib/gitlab/sql/cte.rb @@ -33,7 +33,7 @@ module Gitlab # Returns the Arel relation for this CTE. def to_arel - sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})") + sql = Arel::Nodes::SqlLiteral.new("(#{query_as_sql})") Gitlab::Database::AsWithMaterialized.new(table, sql, materialized: @materialized) end @@ -54,6 +54,12 @@ module Gitlab .with(to_arel) .from(alias_to(relation.model.arel_table)) end + + private + + def query_as_sql + query.is_a?(String) ? query : query.to_sql + end end end end diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb new file mode 100644 index 00000000000..1a236e1a70c --- /dev/null +++ b/lib/gitlab/ssh/signature.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Signature verification with ed25519 keys +# requires this gem to be loaded. +require 'ed25519' + +module Gitlab + module Ssh + class Signature + include Gitlab::Utils::StrongMemoize + + def initialize(signature_text, signed_text, committer_email) + @signature_text = signature_text + @signed_text = signed_text + @committer_email = committer_email + end + + def verification_status + strong_memoize(:verification_status) do + next :unverified unless all_attributes_present? + next :unverified unless valid_signature_blob? && committer + next :unknown_key unless signed_by_key + next :other_user unless signed_by_key.user == committer + + :verified + end + end + + private + + def all_attributes_present? + # Signing an empty string is valid, but signature_text and committer_email + # must be non-empty. + @signed_text && @signature_text.present? && @committer_email.present? + end + + # Verifies the signature using the public key embedded in the blob. + # This proves that the signed_text was signed by the private key + # of the public key identified by `key_fingerprint`. Afterwards, we + # still need to check that the key belongs to the committer. + def valid_signature_blob? + return false unless signature + + signature.verify(@signed_text) + end + + def committer + # Lookup by email because users can push verified commits that were made + # by someone else. For example: Doing a rebase. + strong_memoize(:committer) { User.find_by_any_email(@committer_email, confirmed: true) } + end + + def signature + strong_memoize(:signature) do + ::SSHData::Signature.parse_pem(@signature_text) + rescue SSHData::DecodeError + nil + end + end + + def key_fingerprint + strong_memoize(:key_fingerprint) { signature&.public_key&.fingerprint } + end + + def signed_by_key + strong_memoize(:signed_by_key) do + next unless key_fingerprint + + Key.find_by_fingerprint_sha256(key_fingerprint) + end + end + end + end +end diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 78682a89655..e9c8e816f18 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -15,6 +15,29 @@ module Gitlab Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) ].freeze + BANNED_SSH_KEY_FINGERPRINTS = [ + # https://github.com/rapid7/ssh-badkeys/tree/master/authorized + # banned ssh rsa keys + "SHA256:Z+q4XhSwWY7q0BIDVPR1v/S306FjGBsid7tLq/8kIxM", + "SHA256:uy5wXyEgbRCGsk23+J6f85om7G55Cu3UIPwC7oMZhNQ", + "SHA256:9prMbqhS4QteoFQ1ZRJDqSBLWoHXPyKB0iWR05Ghro4", + "SHA256:1M4RzhMyWuFS/86uPY/ce2prh/dVTHW7iD2RhpquOZA", + + # banned ssh dsa keys + "SHA256:/JLp6z6uGE3BPcs70RQob6QOdEWQ6nDC0xY7ejPOCc0", + "SHA256:whDP3xjKBEettbDuecxtGsfWBST+78gb6McdB9P7jCU", + "SHA256:MEc4HfsOlMqJ3/9QMTmrKn5Xj/yfnMITMW8EwfUfTww", + "SHA256:aPoYT2nPIfhqv6BIlbCCpbDjirBxaDFOtPfZ2K20uWw", + "SHA256:VtjqZ5fiaeoZ3mXOYi49Lk9aO31iT4pahKFP9JPiQPc", + + # other banned ssh keys + # https://github.com/BenBE/kompromat/commit/c8d9a05ea155a1ed609c617d4516f0ac978e8559 + "SHA256:Z+q4XhSwWY7q0BIDVPR1v/S306FjGBsid7tLq/8kIxM", + + # https://www.ctrlu.net/vuln/0006.html + "SHA256:2ewGtK7Dc8XpnfNKShczdc8HSgoEGpoX+MiJkfH2p5I" + ].to_set.freeze + def self.technologies if Gitlab::FIPS.enabled? Gitlab::FIPS::SSH_KEY_TECHNOLOGIES @@ -115,6 +138,10 @@ module Gitlab end end + def banned? + BANNED_SSH_KEY_FINGERPRINTS.include?(fingerprint_sha256) + end + private def technology diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb deleted file mode 100644 index 4180f6ccf00..00000000000 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - # - # Base GitLab Static Site Editor Configuration facade - # - class FileConfig - ConfigError = Class.new(StandardError) - - def initialize(yaml) - content_hash = content_hash(yaml) - @global = Entry::Global.new(content_hash) - @global.compose! - rescue Gitlab::Config::Loader::FormatError => e - raise FileConfig::ConfigError, e.message - end - - def valid? - @global.valid? - end - - def errors - @global.errors - end - - def to_hash_with_defaults - # NOTE: The current approach of simply mapping all the descendents' keys and values ('config') - # into a flat hash may need to be enhanced as we add more complex, non-scalar entries. - @global.descendants.to_h { |descendant| [descendant.key, descendant.config] } - end - - private - - def content_hash(yaml) - Gitlab::Config::Loader::Yaml.new(yaml).load! - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/global.rb b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb deleted file mode 100644 index c295ccf1d11..00000000000 --- a/lib/gitlab/static_site_editor/config/file_config/entry/global.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class FileConfig - module Entry - ## - # This class represents a global entry - root Entry for entire - # GitLab StaticSiteEditor Configuration file. - # - class Global < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[ - image_upload_path - mounts - static_site_generator - ].freeze - - attributes ALLOWED_KEYS - - validations do - validates :config, allowed_keys: ALLOWED_KEYS - end - - entry :image_upload_path, Entry::ImageUploadPath, - description: 'Configuration of the Static Site Editor image upload path.' - entry :mounts, Entry::Mounts, - description: 'Configuration of the Static Site Editor mounts.' - entry :static_site_generator, Entry::StaticSiteGenerator, - description: 'Configuration of the Static Site Editor static site generator.' - end - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb deleted file mode 100644 index 6a2b9e10d33..00000000000 --- a/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class FileConfig - module Entry - ## - # Entry that represents the path to which images will be uploaded - # - class ImageUploadPath < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - - validations do - validates :config, type: String - end - - def self.default - 'source/images' - end - end - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb deleted file mode 100644 index b10956e17a5..00000000000 --- a/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class FileConfig - module Entry - ## - # Entry that represents the mappings of mounted source directories to target paths - # - class Mount < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[source target].freeze - - attributes ALLOWED_KEYS - - validations do - validates :config, allowed_keys: ALLOWED_KEYS - - validates :source, type: String, presence: true - validates :target, type: String, presence: true, allow_blank: true - end - - def self.default - # NOTE: This is the default for middleman projects. Ideally, this would be determined - # based on the defaults for whatever `static_site_generator` is configured. - { - source: 'source', - target: '' - } - end - end - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb deleted file mode 100644 index 10bd377e419..00000000000 --- a/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class FileConfig - module Entry - ## - # Entry that represents the mappings of mounted source directories to target paths - # - class Mounts < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - - entry :mount, Entry::Mount, description: 'Configuration of a Static Site Editor mount.' - - validations do - validates :config, type: Array, presence: true - end - - def skip_config_hash_validation? - true - end - - def self.default - [Entry::Mount.default] - end - end - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb deleted file mode 100644 index 593c0951f93..00000000000 --- a/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class FileConfig - module Entry - ## - # Entry that represents the static site generator tool/framework. - # - class StaticSiteGenerator < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - - validations do - validates :config, type: String, inclusion: { in: %w[middleman], message: "should be 'middleman'" } - end - - def self.default - 'middleman' - end - end - end - end - end - end -end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb deleted file mode 100644 index 1555c3469a5..00000000000 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module StaticSiteEditor - module Config - class GeneratedConfig - def initialize(repository, ref, path, return_url) - @repository = repository - @ref = ref - @path = path - @return_url = return_url - end - - def data - merge_requests_illustration_path = ActionController::Base.helpers.image_path('illustrations/merge_requests.svg') - { - branch: ref, - path: path, - commit_id: commit_id, - project_id: project.id, - project: project.path, - namespace: project.namespace.full_path, - return_url: sanitize_url(return_url), - is_supported_content: supported_content?, - base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path), - merge_requests_illustration_path: merge_requests_illustration_path - } - end - - private - - attr_reader :repository, :ref, :path, :return_url - - delegate :project, to: :repository - - def supported_extensions - %w[.md .md.erb].freeze - end - - def commit_id - repository.commit(ref)&.id - end - - def supported_content? - branch_supported? && extension_supported? && file_exists? - end - - def branch_supported? - ref.in?(%w[master main]) - end - - def extension_supported? - supported_extensions.any? { |ext| path.end_with?(ext) } - end - - def file_exists? - commit_id.present? && !repository.blob_at(commit_id, path).nil? - end - - def full_path - "#{ref}/#{path}" - end - - def sanitize_url(url) - url if Gitlab::UrlSanitizer.valid_web?(url) - end - end - end - end -end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 228da9ee370..ec2ec0d801f 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -23,8 +23,8 @@ module Gitlab Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', 'theme_light_green', '#156b39'), Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', 'theme_red', '#691a16'), Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', 'theme_light_red', '#a62e21'), - Theme.new(2, s_('NavigationTheme|Dark'), 'ui-dark', 'theme_dark', '#303030'), - Theme.new(3, s_('NavigationTheme|Light'), 'ui-light', 'theme_light', '#666'), + Theme.new(2, s_('NavigationTheme|Gray'), 'ui-gray', 'theme_gray', '#303030'), + Theme.new(3, s_('NavigationTheme|Light Gray'), 'ui-light-gray', 'theme_light_gray', '#666'), Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', nil, '#303030') ] end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 542dc476526..05ddc7e26cc 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -7,6 +7,12 @@ module Gitlab GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) + if Feature.enabled?(:standard_context_type_check) + check_argument_type(:namespace, namespace, [Namespace]) + check_argument_type(:project, project, [Project, Integer]) + check_argument_type(:user, user, [User, DeployToken]) + end + @namespace = namespace @plan = namespace&.actual_plan_name @project = project @@ -54,6 +60,14 @@ module Gitlab def project_id project.is_a?(Integer) ? project : project&.id end + + def check_argument_type(argument_name, argument_value, allowed_classes) + return if argument_value.nil? || allowed_classes.any? { |allowed_class| argument_value.is_a?(allowed_class) } + + exception = "Invalid argument type passed for #{argument_name}." \ + " Should be one of #{allowed_classes.map(&:to_s)}" + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(ArgumentError.new(exception)) + end end end end diff --git a/lib/gitlab/updated_notes_paginator.rb b/lib/gitlab/updated_notes_paginator.rb deleted file mode 100644 index d5c01bde6b3..00000000000 --- a/lib/gitlab/updated_notes_paginator.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - # UpdatedNotesPaginator implements a rudimentary form of keyset pagination on - # top of a notes relation that has been initialized with a `last_fetched_at` - # value. This class will attempt to limit the number of notes returned, and - # specify a new value for `last_fetched_at` that will pick up where the last - # page of notes left off. - class UpdatedNotesPaginator - LIMIT = 50 - MICROSECOND = 1_000_000 - - attr_reader :next_fetched_at, :notes - - def initialize(relation, last_fetched_at:) - @last_fetched_at = last_fetched_at - @now = Time.current - - notes, more = fetch_page(relation) - if more - init_middle_page(notes) - else - init_final_page(notes) - end - end - - def metadata - { last_fetched_at: next_fetched_at_microseconds, more: more } - end - - private - - attr_reader :last_fetched_at, :more, :now - - def next_fetched_at_microseconds - (next_fetched_at.to_i * MICROSECOND) + next_fetched_at.usec - end - - def fetch_page(relation) - relation = relation.order_updated_asc.with_order_id_asc - notes = relation.limit(LIMIT + 1).to_a - - return [notes, false] unless notes.size > LIMIT - - marker = notes.pop # Remove the marker note - - # Although very unlikely, it is possible that more notes with the same - # updated_at may exist, e.g., if created in bulk. Add them all to the page - # if this is detected, so pagination won't get stuck indefinitely - if notes.last.updated_at == marker.updated_at - notes += relation - .with_updated_at(marker.updated_at) - .id_not_in(notes.map(&:id)) - .to_a - end - - [notes, true] - end - - def init_middle_page(notes) - @more = true - - # The fetch overlap can be ignored if we're in an intermediate page. - @next_fetched_at = notes.last.updated_at + NotesFinder::FETCH_OVERLAP - @notes = notes - end - - def init_final_page(notes) - @more = false - @next_fetched_at = now - @notes = notes - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb new file mode 100644 index 00000000000..109d2245635 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountImportedProjectsTotalMetric < DatabaseMetric + # Relation and operation are not used, but are included to satisfy expectations + # of other metric generation logic. + relation { Project } + operation :count + + IMPORT_TYPES = %w(gitlab_project gitlab github bitbucket bitbucket_server gitea git manifest + gitlab_migration).freeze + + def value + count(project_relation) + count(entity_relation) + end + + def to_sql + project_relation_sql = Gitlab::Usage::Metrics::Query.for(:count, project_relation) + entity_relation_sql = Gitlab::Usage::Metrics::Query.for(:count, entity_relation) + + "SELECT (#{project_relation_sql}) + (#{entity_relation_sql})" + end + + private + + def project_relation + Project.imported_from(IMPORT_TYPES).where(time_constraints) + end + + def entity_relation + BulkImports::Entity.where(source_type: :project_entity).where(time_constraints) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb index 34247f4f6dd..07dfba70d92 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_issues_metric.rb @@ -11,6 +11,8 @@ module Gitlab finish { Issue.maximum(:id) } relation { Issue } + + cache_start_and_finish_as :issue end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index a000b4509c6..3b09100f3ff 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -18,7 +18,7 @@ module Gitlab UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass class << self - IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count).freeze + IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count sum average).freeze private_constant :IMPLEMENTED_OPERATIONS diff --git a/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb b/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb new file mode 100644 index 00000000000..e430bc8eb71 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IssuesCreatedFromAlertsMetric < NumbersMetric + ISSUES_FROM_ALERTS_METRICS = [ + IssuesWithAlertManagementAlertsMetric, + IssuesWithPrometheusAlertEvents, + IssuesWithSelfManagedPrometheusAlertEvents + ].freeze + + operation :add + + data do |time_frame| + ISSUES_FROM_ALERTS_METRICS.map { |metric| metric.new(time_frame: time_frame).value } + end + + # overwriting instrumentation to generate the appropriate sql query + def instrumentation + 'SELECT ' + ISSUES_FROM_ALERTS_METRICS.map do |metric| + "(#{metric.new(time_frame: time_frame).instrumentation})" + end.join(' + ') + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/issues_with_alert_management_alerts_metric.rb b/lib/gitlab/usage/metrics/instrumentations/issues_with_alert_management_alerts_metric.rb new file mode 100644 index 00000000000..62b91e50e07 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/issues_with_alert_management_alerts_metric.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IssuesWithAlertManagementAlertsMetric < DatabaseMetric + # this metric is used in IssuesCreatedFromAlertsMetric + # do not report metric directly in service ping + available? { false } + + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + relation { Issue.with_alert_management_alerts } + + cache_start_and_finish_as :issue + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/issues_with_prometheus_alert_events.rb b/lib/gitlab/usage/metrics/instrumentations/issues_with_prometheus_alert_events.rb new file mode 100644 index 00000000000..2befec65ac2 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/issues_with_prometheus_alert_events.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IssuesWithPrometheusAlertEvents < DatabaseMetric + # this metric is used in IssuesCreatedFromAlertsMetric + # do not report metric directly in service ping + available? { false } + + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + relation { Issue.with_prometheus_alert_events } + + cache_start_and_finish_as :issue + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/issues_with_self_managed_prometheus_alert_events.rb b/lib/gitlab/usage/metrics/instrumentations/issues_with_self_managed_prometheus_alert_events.rb new file mode 100644 index 00000000000..fdbaa65bc68 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/issues_with_self_managed_prometheus_alert_events.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IssuesWithSelfManagedPrometheusAlertEvents < DatabaseMetric + # this metric is used in IssuesCreatedFromAlertsMetric + # do not report metric directly in service ping + available? { false } + + operation :count + + start { Issue.minimum(:id) } + finish { Issue.maximum(:id) } + + relation { Issue.with_self_managed_prometheus_alert_events } + + cache_start_and_finish_as :issue + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/jira_imports_total_imported_issues_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/jira_imports_total_imported_issues_count_metric.rb new file mode 100644 index 00000000000..6ca57864b8a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/jira_imports_total_imported_issues_count_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class JiraImportsTotalImportedIssuesCountMetric < DatabaseMetric + operation :sum, column: :imported_issues_count + + relation { JiraImportState.finished } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb b/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb new file mode 100644 index 00000000000..8504ee368fc --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class NumbersMetric < BaseMetric + # Usage Example + # + # class BoardsCountMetric < NumbersMetric + # operation :add + # + # data do |time_frame| + # [ + # CountIssuesMetric.new(time_frame: time_frame).value, + # CountBoardsMetric.new(time_frame: time_frame).value, + # ] + # end + # end + + UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass + + class << self + IMPLEMENTED_OPERATIONS = %i(add).freeze + + private_constant :IMPLEMENTED_OPERATIONS + + def data(&block) + return @metric_data&.call unless block_given? + + @metric_data = block + end + + def operation(symbol) + raise UnimplementedOperationError unless symbol.in?(IMPLEMENTED_OPERATIONS) + + @metric_operation = symbol + end + + attr_reader :metric_operation, :metric_data + end + + def value + method(self.class.metric_operation).call(*data) + end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) + end + + private + + def data + self.class.metric_data.call(time_frame) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb new file mode 100644 index 00000000000..9da30db05dd --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class UniqueActiveUsersMetric < DatabaseMetric + operation :count + relation { ::User.active } + + metric_options do + { + batch_size: 10_000 + } + end + + def time_constraints + case time_frame + when '28d' + monthly_time_range_db_params(column: :last_activity_on) + else + super + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb index 0728af9e2ca..238a7a51a20 100644 --- a/lib/gitlab/usage/metrics/name_suggestion.rb +++ b/lib/gitlab/usage/metrics/name_suggestion.rb @@ -19,6 +19,8 @@ module Gitlab name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') when :sum name_suggestion(column: column, relation: relation, prefix: 'sum') + when :average + name_suggestion(column: column, relation: relation, prefix: 'average') when :redis REDIS_EVENT_METRIC_NAME when :alt diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb index 851aa7a50e8..e071b422c16 100644 --- a/lib/gitlab/usage/metrics/query.rb +++ b/lib/gitlab/usage/metrics/query.rb @@ -13,6 +13,8 @@ module Gitlab distinct_count(relation, column) when :sum sum(relation, column) + when :average + average(relation, column) when :estimate_batch_distinct_count estimate_batch_distinct_count(relation, column) when :histogram @@ -25,19 +27,23 @@ module Gitlab private def count(relation, column = nil) - raw_sql(relation, column) + raw_count_sql(relation, column) end def distinct_count(relation, column = nil) - raw_sql(relation, column, true) + raw_count_sql(relation, column, true) end def sum(relation, column) - relation.select(relation.all.table[column].sum).to_sql + raw_sum_sql(relation, column) + end + + def average(relation, column) + raw_average_sql(relation, column) end def estimate_batch_distinct_count(relation, column = nil) - raw_sql(relation, column, true) + raw_count_sql(relation, column, true) end # rubocop: disable CodeReuse/ActiveRecord @@ -62,15 +68,31 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord - def raw_sql(relation, column, distinct = false) + def raw_count_sql(relation, column, distinct = false) column ||= relation.primary_key - node = node_to_count(relation, column) + node = node_to_operate(relation, column) relation.unscope(:order).select(node.count(distinct)).to_sql end # rubocop: enable CodeReuse/ActiveRecord - def node_to_count(relation, column) + # rubocop: disable CodeReuse/ActiveRecord + def raw_sum_sql(relation, column) + node = node_to_operate(relation, column) + + relation.unscope(:order).select(node.sum).to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord + def raw_average_sql(relation, column) + node = node_to_operate(relation, column) + + relation.unscope(:order).select(node.average).to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + def node_to_operate(relation, column) if join_relation?(relation) && joined_column?(column) table_name, column_name = column.split(".") Arel::Table.new(table_name)[column_name] diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 7a17288e5e5..604fa364aa2 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -18,7 +18,6 @@ module Gitlab class UsageData - DEPRECATED_VALUE = -1000 MAX_GENERATION_TIME_FOR_SAAS = 40.hours CE_MEMOIZED_VALUES = %i( @@ -28,7 +27,6 @@ module Gitlab project_maximum_id user_minimum_id user_maximum_id - unique_visit_service deployment_minimum_id deployment_maximum_id auth_providers @@ -68,13 +66,17 @@ module Gitlab # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) + issues_created_manually_from_alerts = if Gitlab.com? + FALLBACK + else + count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) + end { counts: { assignee_lists: count(List.assignee), ci_builds: count(::Ci::Build), - ci_internal_pipelines: count(::Ci::Pipeline.internal), + ci_internal_pipelines: Gitlab.com? ? FALLBACK : count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), ci_pipeline_config_auto_devops: count(::Ci::Pipeline.auto_devops_source), ci_pipeline_config_repository: count(::Ci::Pipeline.repository_source), @@ -156,7 +158,7 @@ module Gitlab notes: count(Note) }.merge( runners_usage, - services_usage, + integrations_usage, usage_counters, user_preferences_usage, container_expiration_policies_usage, @@ -193,8 +195,7 @@ module Gitlab packages: count(::Packages::Package.where(monthly_time_range_db_params)), personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)), project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id), - promoted_issues: DEPRECATED_VALUE + projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id) }.tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -367,7 +368,7 @@ module Gitlab results end - def services_usage + def integrations_usage # rubocop: disable UsageData/LargeTable: Integration.available_integration_names(include_dev: false).each_with_object({}) do |name, response| type = Integration.integration_name_to_type(name) @@ -409,7 +410,7 @@ module Gitlab { jira_imports_total_imported_count: count(finished_jira_imports), jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id), - jira_imports_total_imported_issues_count: sum(JiraImportState.finished, :imported_issues_count) + jira_imports_total_imported_issues_count: add_metric('JiraImportsTotalImportedIssuesCountMetric') } # rubocop: enable UsageData/LargeTable end @@ -645,38 +646,6 @@ module Gitlab } end - def analytics_unique_visits_data - results = ::Gitlab::Analytics::UniqueVisits.analytics_events.each_with_object({}) do |target, hash| - hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } - end - results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) } - results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, **monthly_time_range) } - - { analytics_unique_visits: results } - end - - def compliance_unique_visits_data - results = ::Gitlab::Analytics::UniqueVisits.compliance_events.each_with_object({}) do |target, hash| - hash[target] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target) } - end - results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) } - results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, **monthly_time_range) } - - { compliance_unique_visits: results } - end - - def search_unique_visits_data - events = ::Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('search') - results = events.each_with_object({}) do |event, hash| - hash[event] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event, **weekly_time_range) } - end - - results['search_unique_visits_for_any_target_weekly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **weekly_time_range) } - results['search_unique_visits_for_any_target_monthly'] = redis_usage_data { ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: events, **monthly_time_range) } - - { search_unique_visits: results } - end - def action_monthly_active_users(time_period) date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } @@ -724,9 +693,6 @@ module Gitlab .merge(topology_usage_data) .merge(usage_activity_by_stage) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) - .merge(analytics_unique_visits_data) - .merge(compliance_unique_visits_data) - .merge(search_unique_visits_data) .merge(redis_hll_counters) .deep_merge(aggregated_metrics_data) end @@ -769,7 +735,6 @@ module Gitlab action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, - action_monthly_active_users_sse_edit: redis_usage_data { counter.count_sse_edit_actions(**date_range) }, action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } } end @@ -810,9 +775,8 @@ module Gitlab # rubocop: enable UsageData/LargeTable: 0.upto(series_amount - 1).map do |series| - # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. - sent_count = sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails - clicked_count = clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails + sent_count = sent_in_product_marketing_email_count(sent_emails, track, series) + clicked_count = clicked_in_product_marketing_email_count(clicked_emails, track, series) result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count unless track == 'experience' @@ -821,18 +785,20 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def unique_visit_service - strong_memoize(:unique_visit_service) do - ::Gitlab::Analytics::UniqueVisits.new - end + def sent_in_product_marketing_email_count(sent_emails, track, series) + # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. + sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails + end + + def clicked_in_product_marketing_email_count(clicked_emails, track, series) + # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. + clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails end def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. - add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)), - count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)), - count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)) + add_metric('IssuesCreatedFromAlertsMetric') end def clear_memoized @@ -879,7 +845,7 @@ module Gitlab gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :project_entity }) } - counters[:total] = add(*counters.values) + counters[:total] = add_metric('CountImportedProjectsTotalMetric', time_frame: time_frame) counters end diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index cdcad8fdc7b..2a3dcf267c6 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -15,7 +15,6 @@ module Gitlab MergeRequestCounter, DesignsCounter, KubernetesAgentCounter, - StaticSiteEditorCounter, DiffsCounter, ServiceUsageDataCounter ].freeze diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index f97ebdccecf..8feb24e49ac 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -6,7 +6,6 @@ module Gitlab EDIT_BY_SNIPPET_EDITOR = 'g_edit_by_snippet_ide' EDIT_BY_SFE = 'g_edit_by_sfe' EDIT_BY_WEB_IDE = 'g_edit_by_web_ide' - EDIT_BY_SSE = 'g_edit_by_sse' EDIT_CATEGORY = 'ide_edit' EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' @@ -40,14 +39,6 @@ module Gitlab count_unique(events, date_from, date_to) end - def track_sse_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_SSE, author, time) - end - - def count_sse_edit_actions(date_from:, date_to:) - count_unique(EDIT_BY_SSE, date_from, date_to) - end - def track_live_preview_edit_action(author:, time: Time.zone.now) track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time) end diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index e3bb3f6fef3..267b7fe673d 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -3,6 +3,10 @@ redis_slot: code_review category: code_review aggregation: weekly +- name: i_code_review_mr_with_invalid_approvers + redis_slot: code_review + category: code_review + aggregation: weekly - name: i_code_review_user_single_file_diffs redis_slot: code_review category: code_review diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 448ed4c66e1..0dcbaf59c9c 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -30,11 +30,6 @@ redis_slot: edit expiry: 29 aggregation: daily -- name: g_edit_by_sse - category: ide_edit - redis_slot: edit - expiry: 29 - aggregation: daily - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit @@ -134,6 +129,19 @@ redis_slot: incident_management category: incident_management aggregation: weekly +# Incident management timeline events +- name: incident_management_timeline_event_created + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_timeline_event_edited + redis_slot: incident_management + category: incident_management + aggregation: weekly +- name: incident_management_timeline_event_deleted + redis_slot: incident_management + category: incident_management + aggregation: weekly # Incident management alerts - name: incident_management_alert_create_incident redis_slot: incident_management @@ -144,7 +152,6 @@ redis_slot: incident_management category: incident_management_oncall aggregation: weekly - feature_flag: usage_data_i_incident_management_oncall_notification_sent # Testing category - name: i_testing_test_case_parsed category: testing diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 4ba7ea2d407..f980503b4bf 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -135,6 +135,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_ready + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_reassign category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 0fadd68aeab..9c0f8fe9a80 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -250,3 +250,7 @@ module Gitlab end end end + +# rubocop:disable Layout/LineLength +Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.prepend_mod_with('Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb deleted file mode 100644 index 3c5989d1e11..00000000000 --- a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - class StaticSiteEditorCounter < BaseCounter - KNOWN_EVENTS = %w[views commits merge_requests].freeze - PREFIX = 'static_site_editor' - - class << self - def increment_views_count - count(:views) - end - end - end - end -end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index b2d74b1f0dd..fef5cd680cb 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -85,6 +85,16 @@ module Gitlab failures: [] } end + + # rubocop: disable CodeReuse/ActiveRecord + def sent_in_product_marketing_email_count(sent_emails, track, series) + count(Users::InProductMarketingEmail.where(track: track, series: series)) + end + + def clicked_in_product_marketing_email_count(clicked_emails, track, series) + count(Users::InProductMarketingEmail.where(track: track, series: series).where.not(cta_clicked_at: nil)) + end + # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 633f4683b6b..4d1b234ae54 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -104,6 +104,15 @@ module Gitlab end end + def average(relation, column, batch_size: nil, start: nil, finish: nil) + with_duration do + Gitlab::Database::BatchCount.batch_average(relation, column, batch_size: batch_size, start: start, finish: finish) + rescue ActiveRecord::StatementInvalid => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + FALLBACK + end + end + # We don't support batching with histograms. # Please avoid using this method on large tables. # See https://gitlab.com/gitlab-org/gitlab/-/issues/323949. diff --git a/lib/gitlab/web_hooks/rate_limiter.rb b/lib/gitlab/web_hooks/rate_limiter.rb new file mode 100644 index 00000000000..73d59f6f786 --- /dev/null +++ b/lib/gitlab/web_hooks/rate_limiter.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module WebHooks + class RateLimiter + include Gitlab::Utils::StrongMemoize + + LIMIT_NAME = :web_hook_calls + NO_LIMIT = 0 + # SystemHooks (instance admin hooks) and ServiceHooks (integration hooks) + # are not rate-limited. + EXCLUDED_HOOK_TYPES = %w(SystemHook ServiceHook).freeze + + def initialize(hook) + @hook = hook + @parent = hook.parent + end + + # Increments the rate-limit counter. + # Returns true if the hook should be rate-limited. + def rate_limit! + return false if no_limit? + + ::Gitlab::ApplicationRateLimiter.throttled?( + limit_name, + scope: [root_namespace], + threshold: limit + ) + end + + # Returns true if the hook is currently over its rate-limit. + # It does not increment the rate-limit counter. + def rate_limited? + return false if no_limit? + + Gitlab::ApplicationRateLimiter.peek( + limit_name, + scope: [root_namespace], + threshold: limit + ) + end + + def limit + strong_memoize(:limit) do + next NO_LIMIT if hook.class.name.in?(EXCLUDED_HOOK_TYPES) + + root_namespace.actual_limits.limit_for(limit_name) || NO_LIMIT + end + end + + private + + attr_reader :hook, :parent + + def no_limit? + limit == NO_LIMIT + end + + def root_namespace + @root_namespace ||= parent.root_ancestor + end + + def limit_name + LIMIT_NAME + end + end + end +end + +Gitlab::WebHooks::RateLimiter.prepend_mod diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 3dd4e5e27d4..a8b51a95e59 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -205,7 +205,10 @@ module ObjectStorage end def requires_multipart_upload? - config.aws? && !has_length + return false unless config.aws? + return false if use_workhorse_s3_client? && Feature.enabled?(:s3_omit_multipart_urls) + + !has_length end def upload_id diff --git a/lib/security/ci_configuration/sast_build_action.rb b/lib/security/ci_configuration/sast_build_action.rb index 63f16a1bebe..73298bcd070 100644 --- a/lib/security/ci_configuration/sast_build_action.rb +++ b/lib/security/ci_configuration/sast_build_action.rb @@ -13,16 +13,16 @@ module Security private def variables(params) - collect_values(params, 'value') + collect_values(params, :value) end def default_sast_values(params) - collect_values(params, 'defaultValue') + collect_values(params, :default_value) end def collect_values(config, key) - global_variables = config['global']&.to_h { |k| [k['field'], k[key]] } || {} - pipeline_variables = config['pipeline']&.to_h { |k| [k['field'], k[key]] } || {} + global_variables = config[:global]&.to_h { |k| [k[:field], k[key]] } || {} + pipeline_variables = config[:pipeline]&.to_h { |k| [k[:field], k[key]] } || {} analyzer_variables = collect_analyzer_values(config, key) @@ -31,10 +31,10 @@ module Security def collect_analyzer_values(config, key) analyzer_variables = analyzer_variables_for(config, key) - analyzer_variables['SAST_EXCLUDED_ANALYZERS'] = if key == 'value' - config['analyzers'] - &.reject {|a| a['enabled'] } - &.collect {|a| a['name'] } + analyzer_variables['SAST_EXCLUDED_ANALYZERS'] = if key == :value + config[:analyzers] + &.reject {|a| a[:enabled] } + &.collect {|a| a[:name] } &.sort &.join(', ') else @@ -45,10 +45,10 @@ module Security end def analyzer_variables_for(config, key) - config['analyzers'] - &.select {|a| a['enabled'] && a['variables'] } - &.flat_map {|a| a['variables'] } - &.collect {|v| [v['field'], v[key]] }.to_h + config[:analyzers] + &.select {|a| a[:enabled] && a[:variables] } + &.flat_map {|a| a[:variables] } + &.collect {|v| [v[:field], v[key]] }.to_h end def update_existing_content! diff --git a/lib/service_ping/build_payload.rb b/lib/service_ping/build_payload.rb index 4d3b32a1fc0..3553b624ae0 100644 --- a/lib/service_ping/build_payload.rb +++ b/lib/service_ping/build_payload.rb @@ -3,8 +3,6 @@ module ServicePing class BuildPayload def execute - return {} unless ServicePingSettings.product_intelligence_enabled? - filtered_usage_data end diff --git a/lib/service_ping/permit_data_categories.rb b/lib/service_ping/permit_data_categories.rb index 51eec0808cb..cf69f7503b7 100644 --- a/lib/service_ping/permit_data_categories.rb +++ b/lib/service_ping/permit_data_categories.rb @@ -14,8 +14,6 @@ module ServicePing ].to_set.freeze def execute - return [] unless ServicePingSettings.product_intelligence_enabled? - CATEGORIES end end diff --git a/lib/sidebars/groups/menus/customer_relations_menu.rb b/lib/sidebars/groups/menus/customer_relations_menu.rb index 0aaa6ec45f1..e0772cfe403 100644 --- a/lib/sidebars/groups/menus/customer_relations_menu.rb +++ b/lib/sidebars/groups/menus/customer_relations_menu.rb @@ -35,7 +35,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Contacts'), link: group_crm_contacts_path(context.group), - active_routes: { path: 'groups/crm#contacts' }, + active_routes: { controller: 'groups/crm/contacts' }, item_id: :crm_contacts ) end @@ -44,7 +44,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Organizations'), link: group_crm_organizations_path(context.group), - active_routes: { path: 'groups/crm#organizations' }, + active_routes: { controller: 'groups/crm/organizations' }, item_id: :crm_organizations ) end diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 2b5b3cdbb22..85931e63ebc 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -54,7 +54,7 @@ module Sidebars ::Sidebars::MenuItem.new( title: _('Integrations'), link: project_settings_integrations_path(context.project), - active_routes: { path: %w[integrations#show services#edit] }, + active_routes: { path: %w[integrations#index integrations#edit] }, item_id: :integrations ) end @@ -104,15 +104,14 @@ module Sidebars end def packages_and_registries_menu_item - if !Gitlab.config.registry.enabled || - !can?(context.current_user, :destroy_container_image, context.project) + unless can?(context.current_user, :view_package_registry_project_settings, context.project) return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) end ::Sidebars::MenuItem.new( title: _('Packages & Registries'), link: project_settings_packages_and_registries_path(context.project), - active_routes: { path: 'packages_and_registries#index' }, + active_routes: { path: 'packages_and_registries#show' }, item_id: :packages_and_registries ) end diff --git a/lib/support/systemd/gitlab-sidekiq.service b/lib/support/systemd/gitlab-sidekiq.service index 7d09944c862..cab741010ed 100644 --- a/lib/support/systemd/gitlab-sidekiq.service +++ b/lib/support/systemd/gitlab-sidekiq.service @@ -11,7 +11,6 @@ User=git WorkingDirectory=/home/git/gitlab Environment=RAILS_ENV=production ExecStart=/usr/local/bin/bundle exec sidekiq --config /home/git/gitlab/config/sidekiq_queues.yml --environment production -ExecStop=/usr/local/bin/bundle exec sidekiqctl stop /run/gitlab/sidekiq.pid PIDFile=/home/git/gitlab/tmp/pids/sidekiq.pid Restart=on-failure RestartSec=1 diff --git a/lib/tasks/contracts.rake b/lib/tasks/contracts.rake new file mode 100644 index 00000000000..6bb7f30ad57 --- /dev/null +++ b/lib/tasks/contracts.rake @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +require 'pact/tasks/verification_task' + +contracts = File.expand_path('../../spec/contracts', __dir__) +provider = File.expand_path('provider', contracts) + +# rubocop:disable Rails/RakeEnvironment +namespace :contracts do + namespace :mr do + Pact::VerificationTask.new(:diffs_batch) do |pact| + pact.uri( + "#{contracts}/contracts/project/merge_request/show/mergerequest#show-merge_request_diffs_batch_endpoint.json", + pact_helper: "#{provider}/pact_helpers/project/merge_request/diffs_batch_helper.rb" + ) + end + + Pact::VerificationTask.new(:diffs_metadata) do |pact| + pact.uri( + "#{contracts}/contracts/project/merge_request/show/" \ + "mergerequest#show-merge_request_diffs_metadata_endpoint.json", + pact_helper: "#{provider}/pact_helpers/project/merge_request/diffs_metadata_helper.rb" + ) + end + + Pact::VerificationTask.new(:discussions) do |pact| + pact.uri( + "#{contracts}/contracts/project/merge_request/show/mergerequest#show-merge_request_discussions_endpoint.json", + pact_helper: "#{provider}/pact_helpers/project/merge_request/discussions_helper.rb" + ) + end + + desc 'Run all merge request contract tests' + task 'test:merge_request', :contract_mr do |_t, arg| + raise(ArgumentError, 'Merge request contract tests require contract_mr to be set') unless arg[:contract_mr] + + ENV['CONTRACT_MR'] = arg[:contract_mr] + errors = %w[metadata discussions diffs].each_with_object([]) do |task, err| + Rake::Task["contracts:mr:pact:verify:#{task}"].execute + rescue StandardError, SystemExit + err << "contracts:mr:pact:verify:#{task}" + end + + raise StandardError, "Errors in tasks #{errors.join(', ')}" unless errors.empty? + end + end +end +# rubocop:enable Rails/RakeEnvironment diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 068dc463d16..a446a17dfc3 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -131,14 +131,6 @@ namespace :gitlab do end end - desc 'GitLab | DB | Sets up EE specific database functionality' - - if Gitlab.ee? - task setup_ee: %w[db:drop:geo db:create:geo db:schema:load:geo db:migrate:geo] - else - task :setup_ee - end - desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump' task :clean_structure_sql do |task_name| ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| @@ -356,7 +348,13 @@ namespace :gitlab do Rake::Task['db:drop'].invoke Rake::Task['db:create'].invoke ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| - ActiveRecord::Base.establish_connection(db_config.configuration_hash.merge(username: username)) # rubocop: disable Database/EstablishConnection + config = ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config.env_name, + db_config.name, + db_config.configuration_hash.merge(username: username) + ) + + ActiveRecord::Base.establish_connection(config) # rubocop: disable Database/EstablishConnection Gitlab::Database.check_for_non_superuser Rake::Task['db:migrate'].invoke end diff --git a/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake b/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake new file mode 100644 index 00000000000..4d78acb3011 --- /dev/null +++ b/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + namespace :decomposition do + namespace :rollback do + SEQUENCE_NAME_MATCHER = /nextval\('([a-z_]+)'::regclass\)/.freeze + + desc 'Bump all the CI tables sequences on the Main Database' + task :bump_ci_sequences, [:increase_by] => :environment do |_t, args| + increase_by = args.increase_by.to_i + if increase_by < 1 + puts 'Please specify a positive integer `increase_by` value'.color(:red) + puts 'Example: rake gitlab:db:decomposition:rollback:bump_ci_sequences[100000]'.color(:green) + exit 1 + end + + sequences_by_gitlab_schema(ApplicationRecord, :gitlab_ci).each do |sequence_name| + increment_sequence_by(ApplicationRecord.connection, sequence_name, increase_by) + end + end + end + end + end +end + +# base_model is to choose which connection to use to query the tables +# gitlab_schema, can be 'gitlab_main', 'gitlab_ci', 'gitlab_shared' +def sequences_by_gitlab_schema(base_model, gitlab_schema) + tables = Gitlab::Database::GitlabSchema.tables_to_schema.select do |_table_name, schema_name| + schema_name == gitlab_schema + end.keys + + models = tables.map do |table| + model = Class.new(base_model) + model.table_name = table + model + end + + sequences = [] + models.each do |model| + model.columns.each do |column| + match_result = column.default_function&.match(SEQUENCE_NAME_MATCHER) + next unless match_result + + sequences << match_result[1] + end + end + + sequences +end + +# This method is going to increase the sequence next_value by: +# - increment_by + 1 if the sequence has the attribute is_called = True (which is the common case) +# - increment_by if the sequence has the attribute is_called = False (for example, a newly created sequence) +# It uses ALTER SEQUENCE as a safety mechanism to avoid that no concurrent insertions +# will cause conflicts on the sequence. +# This is because ALTER SEQUENCE blocks concurrent nextval, currval, lastval, and setval calls. +def increment_sequence_by(connection, sequence_name, increment_by) + connection.transaction do + # The first call is to make sure that the sequence's is_called value is set to `true` + # This guarantees that the next call to `nextval` will increase the sequence by `increment_by` + connection.select_value("SELECT nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY #{increment_by}") + connection.select_value("select nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY 1") + end +end diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake new file mode 100644 index 00000000000..b57c2860fe3 --- /dev/null +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + TRIGGER_FUNCTION_NAME = 'gitlab_schema_prevent_write' + + desc "GitLab | DB | Install prevent write triggers on all databases" + task lock_writes: [:environment, 'gitlab:db:validate_config'] do + Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + create_write_trigger_function(connection) + + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + if schemas_for_connection.include?(schema_name.to_sym) + drop_write_trigger(database_name, connection, table_name) + else + create_write_trigger(database_name, connection, table_name) + end + end + end + end + + desc "GitLab | DB | Remove all triggers that prevents writes from all databases" + task unlock_writes: :environment do + Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + drop_write_trigger(database_name, connection, table_name) + end + drop_write_trigger_function(connection) + end + end + + def create_write_trigger_function(connection) + sql = <<-SQL + CREATE OR REPLACE FUNCTION #{TRIGGER_FUNCTION_NAME}() + RETURNS TRIGGER AS + $$ + BEGIN + RAISE EXCEPTION 'Table: "%" is write protected within this Gitlab database.', TG_TABLE_NAME + USING ERRCODE = 'modifying_sql_data_not_permitted', + HINT = 'Make sure you are using the right database connection'; + END + $$ LANGUAGE PLPGSQL + SQL + + connection.execute(sql) + end + + def drop_write_trigger_function(connection) + sql = <<-SQL + DROP FUNCTION IF EXISTS #{TRIGGER_FUNCTION_NAME}() + SQL + + connection.execute(sql) + end + + def create_write_trigger(database_name, connection, table_name) + puts "#{database_name}: '#{table_name}'... Lock Writes".color(:yellow) + sql = <<-SQL + DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; + CREATE TRIGGER #{write_trigger_name(table_name)} + BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE + ON #{table_name} + FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); + SQL + + with_retries(connection) do + connection.execute(sql) + end + end + + def drop_write_trigger(database_name, connection, table_name) + puts "#{database_name}: '#{table_name}'... Allow Writes".color(:green) + sql = <<-SQL + DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name} + SQL + + with_retries(connection) do + connection.execute(sql) + end + end + + def with_retries(connection, &block) + with_statement_timeout_retries do + with_lock_retries(connection) do + yield + end + end + end + + def with_statement_timeout_retries(times = 5) + current_iteration = 1 + begin + yield + rescue ActiveRecord::QueryCanceled => err + puts "Retrying after #{err.message}" + + if current_iteration <= times + current_iteration += 1 + retry + else + raise err + end + end + end + + def with_lock_retries(connection, &block) + Gitlab::Database::WithLockRetries.new( + klass: "gitlab:db:lock_writes", + logger: Gitlab::AppLogger, + connection: connection + ).run(&block) + end + + def write_trigger_name(table_name) + "gitlab_schema_write_trigger_for_#{table_name}" + end + end +end diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index 66aa949cc94..2a3a54b5351 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -4,6 +4,23 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml namespace :gitlab do namespace :db do + DB_CONFIG_NAME_KEY = 'gitlab_db_config_name' + + DB_IDENTIFIER_SQL = <<-SQL + SELECT system_identifier, current_database() + FROM pg_control_system() + SQL + + # We fetch timestamp as a way to properly handle race conditions + # fail in such cases, which should not really happen in production environment + DB_IDENTIFIER_WITH_DB_CONFIG_NAME_SQL = <<-SQL + SELECT + system_identifier, current_database(), + value as db_config_name, created_at as timestamp + FROM pg_control_system() + LEFT JOIN ar_internal_metadata ON ar_internal_metadata.key=$1 + SQL + desc 'Validates `config/database.yml` to ensure a correct behavior is configured' task validate_config: :environment do original_db_config = ActiveRecord::Base.connection_db_config # rubocop:disable Database/MultipleDatabases @@ -14,26 +31,22 @@ namespace :gitlab do db_configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, include_replicas: true) db_configs = db_configs.reject(&:replica?) + # The `pg_control_system()` is not enough to properly discover matching database systems + # since in case of cluster promotion it will return the same identifier as main cluster + # We instead set an `ar_internal_metadata` information with configured database name + db_configs.reverse_each do |db_config| + insert_db_identifier(db_config) + end + # Map each database connection into unique identifier of system+database - # rubocop:disable Database/MultipleDatabases all_connections = db_configs.map do |db_config| - identifier = - begin - ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection - ActiveRecord::Base.connection.select_one("SELECT system_identifier, current_database() FROM pg_control_system()") - rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err - warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" - rescue ActiveRecord::NoDatabaseError - end - { name: db_config.name, config: db_config, database_tasks?: db_config.database_tasks?, - identifier: identifier + identifier: get_db_identifier(db_config) } - end.compact - # rubocop:enable Database/MultipleDatabases + end unique_connections = all_connections.group_by { |connection| connection[:identifier] } primary_connection = all_connections.find { |connection| ActiveRecord::Base.configurations.primary?(connection[:name]) } @@ -111,5 +124,43 @@ namespace :gitlab do Rake::Task["db:schema:load:#{name}"].enhance(['gitlab:db:validate_config']) Rake::Task["db:schema:dump:#{name}"].enhance(['gitlab:db:validate_config']) end + + def insert_db_identifier(db_config) + ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection + + if ActiveRecord::InternalMetadata.table_exists? + ts = Time.zone.now + + ActiveRecord::InternalMetadata.upsert( + { key: DB_CONFIG_NAME_KEY, + value: db_config.name, + created_at: ts, + updated_at: ts } + ) + end + rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err + warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" + rescue ActiveRecord::NoDatabaseError + rescue ActiveRecord::StatementInvalid => err + raise unless err.cause.is_a?(PG::ReadOnlySqlTransaction) + + warn "WARNING: Could not write to the database #{db_config.name}: #{err.message}" + end + + def get_db_identifier(db_config) + ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection + + # rubocop:disable Database/MultipleDatabases + if ActiveRecord::InternalMetadata.table_exists? + ActiveRecord::Base.connection.select_one( + DB_IDENTIFIER_WITH_DB_CONFIG_NAME_SQL, nil, [DB_CONFIG_NAME_KEY]) + else + ActiveRecord::Base.connection.select_one(DB_IDENTIFIER_SQL) + end + # rubocop:enable Database/MultipleDatabases + rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err + warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" + rescue ActiveRecord::NoDatabaseError + end end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index c3828e7eba4..e6fde28e38f 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -4,60 +4,6 @@ require 'logger' namespace :gitlab do namespace :pages do - desc "GitLab | Pages | Migrate legacy storage to zip format" - task migrate_legacy_storage: :gitlab_environment do - logger.info('Starting to migrate legacy pages storage to zip deployments') - - result = ::Pages::MigrateFromLegacyStorageService.new(logger, - ignore_invalid_entries: ignore_invalid_entries, - mark_projects_as_not_deployed: mark_projects_as_not_deployed) - .execute_with_threads(threads: migration_threads, batch_size: batch_size) - - logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.") - logger.info("- The #{result[:migrated]} projects migrated successfully") - logger.info("- The #{result[:errored]} projects failed to be migrated") - end - - desc "GitLab | Pages | DANGER: Removes data which was migrated from legacy storage on zip storage. Can be used if some bugs in migration are discovered and migration needs to be restarted from scratch." - task clean_migrated_zip_storage: :gitlab_environment do - destroyed_deployments = 0 - - logger.info("Starting to delete migrated pages deployments") - - ::PagesDeployment.migrated_from_legacy_storage.each_batch(of: batch_size) do |batch| - destroyed_deployments += batch.count - - # we need to destroy associated files, so can't use delete_all - batch.destroy_all # rubocop: disable Cop/DestroyAll - - logger.info("#{destroyed_deployments} deployments were deleted") - end - end - - def logger - @logger ||= Logger.new($stdout) - end - - def migration_threads - ENV.fetch('PAGES_MIGRATION_THREADS', '3').to_i - end - - def batch_size - ENV.fetch('PAGES_MIGRATION_BATCH_SIZE', '10').to_i - end - - def ignore_invalid_entries - Gitlab::Utils.to_boolean( - ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false') - ) - end - - def mark_projects_as_not_deployed - Gitlab::Utils.to_boolean( - ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false') - ) - end - namespace :deployments do task migrate_to_object_storage: :gitlab_environment do logger = Logger.new($stdout) diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 6574bfd2549..40d88ea8a5b 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -6,6 +6,17 @@ namespace :tw do desc 'Generates a list of codeowners for documentation pages.' task :codeowners do CodeOwnerRule = Struct.new(:category, :writer) + DocumentOwnerMapping = Struct.new(:path, :writer) do + def writer_owns_all_pages?(mappings) + mappings + .select { |mapping| mapping.directory == directory } + .all? { |mapping| mapping.writer == writer } + end + + def directory + @directory ||= File.dirname(path) + end + end CODE_OWNER_RULES = [ CodeOwnerRule.new('Activation', '@kpaizee'), @@ -61,7 +72,6 @@ namespace :tw do CodeOwnerRule.new('Sharding', '@sselhorn'), CodeOwnerRule.new('Source Code', '@aqualls'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), - CodeOwnerRule.new('Static Site Editor', '@aqualls'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Testing', '@eread'), CodeOwnerRule.new('Threat Insights', '@claytoncornell'), @@ -85,6 +95,7 @@ namespace :tw do end errors = [] + mappings = [] path = Rails.root.join("doc/**/*.md") Dir.glob(path) do |file| @@ -99,9 +110,21 @@ namespace :tw do writer = writer_for_group(document.group) next unless writer - puts "#{file.gsub(Dir.pwd, ".")} #{writer}" if document.has_a_valid_group? + mappings << DocumentOwnerMapping.new(file.delete_prefix(Dir.pwd), writer) if document.has_a_valid_group? end + deduplicated_mappings = Set.new + + mappings.each do |mapping| + if mapping.writer_owns_all_pages?(mappings) + deduplicated_mappings.add("#{mapping.directory}/ #{mapping.writer}") + else + deduplicated_mappings.add("#{mapping.path} #{mapping.writer}") + end + end + + deduplicated_mappings.each { |mapping| puts mapping } + if errors.present? puts "-----" puts "ERRORS - the following files are missing the correct metadata:" diff --git a/lib/tasks/migrate/composite_primary_keys.rake b/lib/tasks/migrate/composite_primary_keys.rake deleted file mode 100644 index 68f7c4d6c4a..00000000000 --- a/lib/tasks/migrate/composite_primary_keys.rake +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - namespace :db do - desc 'GitLab | DB | Adds primary keys to tables that only have composite unique keys' - task composite_primary_keys_add: :environment do - require Rails.root.join('db/optional_migrations/composite_primary_keys') - CompositePrimaryKeysMigration.new.up - end - - desc 'GitLab | DB | Removes previously added composite primary keys' - task composite_primary_keys_drop: :environment do - require Rails.root.join('db/optional_migrations/composite_primary_keys') - CompositePrimaryKeysMigration.new.down - end - end -end diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index 6eabdf51dcd..28c370e5ca6 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -36,7 +36,7 @@ unless Rails.env.production? # expected. cop_names = args.to_a - todo_dir = RuboCop::TodoDir.new(RuboCop::TodoDir::DEFAULT_TODO_DIR) + todo_dir = RuboCop::TodoDir.new(RuboCop::Formatter::TodoFormatter.base_directory) if cop_names.any? # We are sorting the cop names to benefit from RuboCop cache which |