diff options
Diffstat (limited to 'app/models')
98 files changed, 1346 insertions, 623 deletions
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index edb9a2053b1..361b1a8dca9 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -120,7 +120,7 @@ class ApplicationSetting < ApplicationRecord if: :help_page_support_url_column_exists? validates :help_page_documentation_base_url, - length: { maximum: 255, message: _("is too long (maximum is %{count} characters)") }, + length: { maximum: 255, message: N_("is too long (maximum is %{count} characters)") }, allow_blank: true, addressable_url: true @@ -148,7 +148,7 @@ class ApplicationSetting < ApplicationRecord if: :akismet_enabled validates :spam_check_api_key, - length: { maximum: 2000, message: _('is too long (maximum is %{count} characters)') }, + length: { maximum: 2000, message: N_('is too long (maximum is %{count} characters)') }, allow_blank: true validates :unique_ips_limit_per_user, @@ -228,7 +228,7 @@ class ApplicationSetting < ApplicationRecord validates :default_artifacts_expire_in, presence: true, duration: true validates :container_expiration_policies_enable_historic_entries, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :container_registry_token_expire_delay, presence: true, @@ -320,8 +320,8 @@ class ApplicationSetting < ApplicationRecord validates :personal_access_token_prefix, format: { with: %r{\A[a-zA-Z0-9_+=/@:.-]+\z}, - message: _("can contain only letters of the Base64 alphabet (RFC4648) with the addition of '@', ':' and '.'") }, - length: { maximum: 20, message: _('is too long (maximum is %{count} characters)') }, + message: N_("can contain only letters of the Base64 alphabet (RFC4648) with the addition of '@', ':' and '.'") }, + length: { maximum: 20, message: N_('is too long (maximum is %{count} characters)') }, allow_blank: true validates :commit_email_hostname, format: { with: /\A[^@]+\z/ } @@ -369,7 +369,7 @@ class ApplicationSetting < ApplicationRecord validates :email_restrictions, untrusted_regexp: true - validates :hashed_storage_enabled, inclusion: { in: [true], message: _("Hashed storage can't be disabled anymore for new projects") } + validates :hashed_storage_enabled, inclusion: { in: [true], message: N_("Hashed storage can't be disabled anymore for new projects") } validates :container_registry_delete_tags_service_timeout, :container_registry_cleanup_tags_service_max_list_size, @@ -377,7 +377,7 @@ class ApplicationSetting < ApplicationRecord numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :container_registry_expiration_policies_caching, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :container_registry_import_max_tags_count, :container_registry_import_max_retries, @@ -404,11 +404,18 @@ class ApplicationSetting < ApplicationRecord numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :invisible_captcha_enabled, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :invitation_flow_enforcement, + validates :invitation_flow_enforcement, :can_create_group, allow_nil: false, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } + + # rubocop:disable Cop/StaticTranslationDefinition + validates :deactivate_dormant_users_period, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 90, message: _("'%{value}' days of inactivity must be greater than or equal to 90") }, + if: :deactivate_dormant_users? + # rubocop:enable Cop/StaticTranslationDefinition Gitlab::SSHPublicKey.supported_types.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } @@ -513,11 +520,11 @@ class ApplicationSetting < ApplicationRecord rsa_key: true, allow_nil: true validates :rate_limiting_response_text, - length: { maximum: 255, message: _('is too long (maximum is %{count} characters)') }, + length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, allow_blank: true validates :jira_connect_application_key, - length: { maximum: 255, message: _('is too long (maximum is %{count} characters)') }, + length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, allow_blank: true with_options(presence: true, numericality: { only_integer: true, greater_than: 0 }) do @@ -561,7 +568,7 @@ class ApplicationSetting < ApplicationRecord allow_nil: false validates :admin_mode, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :external_pipeline_validation_service_url, addressable_url: true, allow_blank: true @@ -574,7 +581,7 @@ class ApplicationSetting < ApplicationRecord inclusion: { in: ApplicationSetting.whats_new_variants.keys } validates :floc_enabled, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } enum sidekiq_job_limiter_mode: { Gitlab::SidekiqMiddleware::SizeLimiter::Validator::TRACK_MODE => 0, @@ -589,7 +596,7 @@ class ApplicationSetting < ApplicationRecord numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :sentry_enabled, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :sentry_dsn, addressable_url: true, presence: true, length: { maximum: 255 }, if: :sentry_enabled? @@ -601,7 +608,7 @@ class ApplicationSetting < ApplicationRecord if: :sentry_enabled? validates :error_tracking_enabled, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :error_tracking_api_url, presence: true, addressable_url: true, @@ -667,9 +674,10 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :arkose_labs_public_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :arkose_labs_private_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :cube_api_key, encryption_options_base_32_aes_256_gcm + attr_encrypted :jitsu_administrator_password, encryption_options_base_32_aes_256_gcm validates :disable_feed_token, - inclusion: { in: [true, false], message: _('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } before_validation :ensure_uuid! before_validation :coerce_repository_storages_weighted, if: :repository_storages_weighted_changed? @@ -791,6 +799,10 @@ class ApplicationSetting < ApplicationRecord ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES.include?(diagram_type) end + def personal_access_tokens_disabled? + false + end + private def parsed_grafana_url diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 4d377855dea..dee4bd07fd9 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -240,7 +240,8 @@ module ApplicationSettingImplementation search_rate_limit: 30, search_rate_limit_unauthenticated: 10, users_get_by_id_limit: 300, - users_get_by_id_limit_allowlist: [] + users_get_by_id_limit_allowlist: [], + can_create_group: true } end diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index 5430575ace7..e9530a80d9f 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -73,4 +73,8 @@ class AwardEmoji < ApplicationRecord awardable.expire_etag_cache if awardable.is_a?(Note) awardable.try(:update_upvotes_count) if upvote? end + + def to_ability_name + 'emoji' + end end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index e0a616b5fb4..a2542e669e1 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -116,8 +116,20 @@ class BulkImports::Entity < ApplicationRecord "/#{pluralized_name}/#{encoded_source_full_path}" end + def base_xid_resource_url_path + "/#{pluralized_name}/#{source_xid}" + end + + def base_resource_path + if source_xid.present? + base_xid_resource_url_path + else + base_resource_url_path + end + end + def export_relations_url_path - "#{base_resource_url_path}/export_relations" + "#{base_resource_path}/export_relations" end def relation_download_url_path(relation) @@ -125,7 +137,7 @@ class BulkImports::Entity < ApplicationRecord end def wikis_url_path - "#{base_resource_url_path}/wikis" + "#{base_resource_path}/wikis" end def project? @@ -149,6 +161,13 @@ class BulkImports::Entity < ApplicationRecord end def validate_imported_entity_type + if project_entity? && !BulkImports::Features.project_migration_enabled?(destination_namespace) + errors.add( + :base, + s_('BulkImport|invalid entity source type') + ) + end + if group.present? && project_entity? errors.add( :group, diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb index 4fea62edb2a..cbd7b189007 100644 --- a/app/models/bulk_imports/export_status.rb +++ b/app/models/bulk_imports/export_status.rb @@ -30,14 +30,18 @@ module BulkImports private - attr_reader :client, :entity, :relation + attr_reader :client, :entity, :relation, :pipeline_tracker def export_status strong_memoize(:export_status) do fetch_export_status&.find { |item| item['relation'] == relation } + rescue BulkImports::NetworkError => e + raise BulkImports::RetryPipelineError.new(e.message, 2.seconds) if e.retriable?(pipeline_tracker) + + default_error_response(e.message) + rescue StandardError => e + default_error_response(e.message) end - rescue StandardError => e - { 'status' => Export::FAILED, 'error' => e.message } end def fetch_export_status @@ -47,5 +51,9 @@ module BulkImports def status_endpoint File.join(entity.export_relations_url_path, 'status') end + + def default_error_response(message) + { 'status' => Export::FAILED, 'error' => message } + end end end diff --git a/app/models/bulk_imports/failure.rb b/app/models/bulk_imports/failure.rb index a6f7582c3b0..44d16618c77 100644 --- a/app/models/bulk_imports/failure.rb +++ b/app/models/bulk_imports/failure.rb @@ -10,4 +10,24 @@ class BulkImports::Failure < ApplicationRecord optional: false validates :entity, presence: true + + def relation + pipeline_relation || default_relation + end + + private + + def pipeline_relation + klass = pipeline_class.constantize + + return unless klass.ancestors.include?(BulkImports::Pipeline) + + klass.relation + rescue NameError + nil + end + + def default_relation + pipeline_class.demodulize.chomp('Pipeline').underscore + end end diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index fa38b7617d2..357f4629078 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -60,6 +60,8 @@ class BulkImports::Tracker < ApplicationRecord event :retry do transition started: :enqueued + # To avoid errors when retrying a pipeline in case of network errors + transition enqueued: :enqueued end event :enqueue do diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4e58f877217..b8511536e32 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -108,10 +108,12 @@ module Ci validates :ref, presence: true scope :not_interruptible, -> do - joins(:metadata).where.not('ci_builds_metadata.id' => Ci::BuildMetadata.scoped_build.with_interruptible.select(:id)) + joins(:metadata) + .where.not(Ci::BuildMetadata.table_name => { id: Ci::BuildMetadata.scoped_build.with_interruptible.select(:id) }) end scope :unstarted, -> { where(runner_id: nil) } + scope :with_downloadable_artifacts, -> do where('EXISTS (?)', Ci::JobArtifact.select(1) @@ -120,6 +122,14 @@ module Ci ) end + scope :with_erasable_artifacts, -> do + where('EXISTS (?)', + Ci::JobArtifact.select(1) + .where('ci_builds.id = ci_job_artifacts.job_id') + .where(file_type: Ci::JobArtifact.erasable_file_types) + ) + end + scope :in_pipelines, ->(pipelines) do where(pipeline: pipelines) end @@ -178,7 +188,7 @@ module Ci scope :license_management_jobs, -> { where(name: %i(license_management license_scanning)) } # handle license rename https://gitlab.com/gitlab-org/gitlab/issues/8911 scope :with_secure_reports_from_config_options, -> (job_types) do - joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) + joins(:metadata).where("#{Ci::BuildMetadata.quoted_table_name}.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) end scope :with_coverage, -> { where.not(coverage: nil) } @@ -218,7 +228,7 @@ module Ci yaml_variables when environment coverage_regex description tag_list protected needs_attributes job_variables_attributes resource_group scheduling_type - ci_stage partition_id].freeze + ci_stage partition_id id_tokens].freeze end end @@ -407,18 +417,10 @@ module Ci pipeline.manual_actions.reject { |action| action.name == self.name } end - def environment_manual_actions - pipeline.manual_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name } - end - def other_scheduled_actions pipeline.scheduled_actions.reject { |action| action.name == self.name } end - def environment_scheduled_actions - pipeline.scheduled_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name } - end - def pages_generator? Gitlab.config.pages.enabled && self.name == 'pages' @@ -445,8 +447,7 @@ module Ci def prevent_rollback_deployment? strong_memoize(:prevent_rollback_deployment) do - Feature.enabled?(:prevent_outdated_deployment_jobs, project) && - starts_environment? && + starts_environment? && project.ci_forward_deployment_enabled? && deployment&.older_than_last_successful_deployment? end @@ -1195,6 +1196,14 @@ module Ci end def job_jwt_variables + if project.ci_cd_settings.opt_in_jwt? + id_tokens_variables + else + legacy_jwt_variables.concat(id_tokens_variables) + end + end + + def legacy_jwt_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless Feature.enabled?(:ci_job_jwt, project) @@ -1208,6 +1217,20 @@ module Ci end end + def id_tokens_variables + return [] unless id_tokens? + + Gitlab::Ci::Variables::Collection.new.tap do |variables| + id_tokens.each do |var_name, token_data| + token = Gitlab::Ci::JwtV2.for_build(self, aud: token_data['id_token']['aud']) + + variables.append(key: var_name, value: token, public: false, masked: true) + end + rescue OpenSSL::PKey::RSAError, Gitlab::Ci::Jwt::NoSigningKeyError => e + Gitlab::ErrorTracking.track_exception(e) + end + end + def cache_for_online_runners(&block) Rails.cache.fetch( ['has-online-runners', id], diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 3bdf2f90acb..33092e881f0 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -6,11 +6,14 @@ module Ci class BuildMetadata < Ci::ApplicationRecord BuildTimeout = Struct.new(:value, :source) + include Ci::Partitionable include Presentable include ChronicDurationAttribute include Gitlab::Utils::StrongMemoize self.table_name = 'ci_builds_metadata' + self.primary_key = 'id' + partitionable scope: :build belongs_to :build, class_name: 'CommitStatus' belongs_to :project @@ -27,7 +30,7 @@ module Ci chronic_duration_attr_reader :timeout_human_readable, :timeout - scope :scoped_build, -> { where('ci_builds_metadata.build_id = ci_builds.id') } + scope :scoped_build, -> { where("#{quoted_table_name}.build_id = #{Ci::Build.quoted_table_name}.id") } scope :with_interruptible, -> { where(interruptible: true) } scope :with_exposed_artifacts, -> { where(has_exposed_artifacts: true) } diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index c2ab8ca0929..3fdf07123e6 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -19,6 +19,11 @@ module Ci validates :target_project, presence: true validate :not_self_referential_link + enum direction: { + outbound: 0, + inbound: 1 + } + def self.for_source_and_target(source_project, target_project) self.find_by(source_project: source_project, target_project: target_project) end diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb index 26a49d6a730..1aa49b95201 100644 --- a/app/models/ci/job_token/scope.rb +++ b/app/models/ci/job_token/scope.rb @@ -23,7 +23,7 @@ module Ci def includes?(target_project) # if the setting is disabled any project is considered to be in scope. - return true unless source_project.ci_job_token_scope_enabled? + return true unless source_project.ci_outbound_job_token_scope_enabled? target_project.id == source_project.id || Ci::JobToken::ProjectScopeLink.from_project(source_project).to_project(target_project).exists? diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1e328c3c573..950e0a583bc 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -112,6 +112,8 @@ module Ci has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline + has_one :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :pipeline + has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id has_many :latest_builds_report_results, through: :latest_builds, source: :report_results has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -119,6 +121,7 @@ module Ci accepts_nested_attributes_for :variables, reject_if: :persisted? delegate :full_path, to: :project, prefix: true + delegate :title, to: :pipeline_metadata, allow_nil: true validates :sha, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } @@ -614,6 +617,15 @@ module Ci # auto_canceled_by_pipeline_id - store the pipeline_id of the pipeline that triggered cancellation # execute_async - if true cancel the children asyncronously def cancel_running(retries: 1, cascade_to_children: true, auto_canceled_by_pipeline_id: nil, execute_async: true) + Gitlab::AppJsonLogger.info( + event: 'pipeline_cancel_running', + pipeline_id: id, + auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id, + cascade_to_children: cascade_to_children, + execute_async: execute_async, + **Gitlab::ApplicationContext.current + ) + update(auto_canceled_by_id: auto_canceled_by_pipeline_id) if auto_canceled_by_pipeline_id cancel_jobs(cancelable_statuses, retries: retries, auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id) @@ -760,8 +772,14 @@ module Ci # There is no ActiveRecord relation between Ci::Pipeline and notes # as they are related to a commit sha. This method helps importing # them using the +Gitlab::ImportExport::Project::RelationFactory+ class. - def notes=(notes) - notes.each do |note| + def notes=(notes_to_save) + notes_to_save.reject! do |note_to_save| + notes.any? do |note| + [note_to_save.note, note_to_save.created_at.to_i] == [note.note, note.created_at.to_i] + end + end + + notes_to_save.each do |note| note[:id] = nil note[:commit_id] = sha note[:noteable_id] = self['id'] @@ -850,7 +868,6 @@ module Ci variables.append(key: 'CI_COMMIT_REF_NAME', value: source_ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: source_ref_slug) variables.append(key: 'CI_COMMIT_BRANCH', value: ref) if branch? - variables.append(key: 'CI_COMMIT_TAG', value: ref) if tag? variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) @@ -863,7 +880,8 @@ module Ci variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) variables.append(key: 'CI_BUILD_REF_NAME', value: source_ref) variables.append(key: 'CI_BUILD_REF_SLUG', value: source_ref_slug) - variables.append(key: 'CI_BUILD_TAG', value: ref) if tag? + + variables.concat(predefined_commit_tag_variables) end end end @@ -888,6 +906,20 @@ module Ci end end + def predefined_commit_tag_variables + strong_memoize(:predefined_commit_ref_variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless tag? + + variables.append(key: 'CI_COMMIT_TAG', value: ref) + variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: project.repository.find_tag(ref).message) + + # legacy variable + variables.append(key: 'CI_BUILD_TAG', value: ref) + end + end + end + def queued_duration return unless started_at @@ -972,8 +1004,8 @@ module Ci # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 expanded_environment_names = builds_in_self_and_project_descendants.joins(:metadata) - .where.not('ci_builds_metadata.expanded_environment_name' => nil) - .distinct('ci_builds_metadata.expanded_environment_name') + .where.not(Ci::BuildMetadata.table_name => { expanded_environment_name: nil }) + .distinct("#{Ci::BuildMetadata.quoted_table_name}.expanded_environment_name") .limit(100) .pluck(:expanded_environment_name) @@ -1162,6 +1194,10 @@ module Ci complete? && builds.latest.with_exposed_artifacts.exists? end + def has_erasable_artifacts? + complete? && builds.latest.with_erasable_artifacts.exists? + end + def branch_updated? strong_memoize(:branch_updated) do push_details.branch_updated? @@ -1328,9 +1364,9 @@ module Ci self.builds.latest.build_matchers(project) end - def authorized_cluster_agents - strong_memoize(:authorized_cluster_agents) do - ::Clusters::AgentAuthorizationsFinder.new(project).execute.map(&:agent) + def cluster_agent_authorizations + strong_memoize(:cluster_agent_authorizations) do + ::Clusters::AgentAuthorizationsFinder.new(project).execute end end diff --git a/app/models/ci/pipeline_metadata.rb b/app/models/ci/pipeline_metadata.rb new file mode 100644 index 00000000000..c96b395b45f --- /dev/null +++ b/app/models/ci/pipeline_metadata.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class PipelineMetadata < Ci::ApplicationRecord + self.primary_key = :pipeline_id + + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_metadata + belongs_to :project, class_name: "Project", inverse_of: :pipeline_metadata + + validates :pipeline, presence: true + validates :project, presence: true + validates :title, presence: true, length: { minimum: 1, maximum: 255 } + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 28d9edcc135..3be627989b1 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -14,7 +14,7 @@ module Ci include Presentable include EachBatch - add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration, expiration_enforced?: :token_expiration_enforced? + add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration enum access_level: { not_protected: 0, @@ -99,27 +99,26 @@ module Ci } scope :belonging_to_group, -> (group_id) { - joins(:runner_namespaces) - .where(ci_runner_namespaces: { namespace_id: group_id }) + joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_id }) } scope :belonging_to_group_or_project_descendants, -> (group_id) { group_ids = Ci::NamespaceMirror.by_group_and_descendants(group_id).select(:namespace_id) project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id) - group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_ids }) - project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_ids }) + group_runners = belonging_to_group(group_ids) + project_runners = belonging_to_project(project_ids).distinct - union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql - - from("(#{union_sql}) #{table_name}") + from_union( + [group_runners, project_runners], + remove_duplicates: false + ) } scope :belonging_to_group_and_ancestors, -> (group_id) { group_self_and_ancestors_ids = ::Group.find_by(id: group_id)&.self_and_ancestor_ids - joins(:runner_namespaces) - .where(ci_runner_namespaces: { namespace_id: group_self_and_ancestors_ids }) + belonging_to_group(group_self_and_ancestors_ids) } scope :belonging_to_parent_group_of_project, -> (project_id) { @@ -153,6 +152,17 @@ module Ci ) end + scope :usable_from_scope, -> (group) do + from_union( + [ + belonging_to_group(group.ancestor_ids), + belonging_to_group_or_project_descendants(group.id), + group.shared_runners + ], + remove_duplicates: false + ) + end + scope :assignable_for, ->(project) do # FIXME: That `to_sql` is needed to workaround a weird Rails bug. # Without that, placeholders would miss one and couldn't match. @@ -205,7 +215,7 @@ module Ci validates :maintenance_note, length: { maximum: 1024 } - alias_attribute :maintenance_note, :maintainer_note + alias_attribute :maintenance_note, :maintainer_note # NOTE: Need to keep until REST v5 is implemented # Searches for runners matching the given query. # @@ -335,7 +345,7 @@ module Ci end # DEPRECATED - # TODO Remove in %16.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + # TODO Remove in v5 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 def deprecated_rest_status return :stale if stale? @@ -470,10 +480,6 @@ module Ci end end - def self.token_expiration_enforced? - Feature.enabled?(:enforce_runner_token_expires_at) - end - private scope :with_upgrade_status, ->(upgrade_status) do diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 9a35f1876c9..ffff7eebbee 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -7,6 +7,7 @@ module Ci FILE_SIZE_LIMIT = 5.megabytes.freeze CHECKSUM_ALGORITHM = 'sha256' + PARSABLE_EXTENSIONS = %w[cer p12 mobileprovision].freeze self.limit_scope = :project self.limit_name = 'project_ci_secure_files' @@ -16,6 +17,7 @@ module Ci validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } validates :checksum, :file_store, :name, :project_id, presence: true validates :name, uniqueness: { scope: :project } + validates :metadata, json_schema: { filename: "ci_secure_file_metadata" }, allow_nil: true after_initialize :generate_key_data before_validation :assign_checksum @@ -23,6 +25,8 @@ module Ci scope :order_by_created_at, -> { order(created_at: :desc) } scope :project_id_in, ->(ids) { where(project_id: ids) } + serialize :metadata, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize + default_value_for(:file_store) { Ci::SecureFileUploader.default_store } mount_file_store_uploader Ci::SecureFileUploader @@ -31,6 +35,41 @@ module Ci CHECKSUM_ALGORITHM end + def file_extension + File.extname(name).delete_prefix('.') + end + + def metadata_parsable? + PARSABLE_EXTENSIONS.include?(file_extension) + end + + def metadata_parser + return unless metadata_parsable? + + case file_extension + when 'cer' + Gitlab::Ci::SecureFiles::Cer.new(file.read) + when 'p12' + Gitlab::Ci::SecureFiles::P12.new(file.read) + when 'mobileprovision' + Gitlab::Ci::SecureFiles::MobileProvision.new(file.read) + end + end + + def update_metadata! + return unless metadata_parser + + begin + parser = metadata_parser + self.metadata = parser.metadata + self.expires_at = parser.metadata[:expires_at] + save! + rescue StandardError => err + Gitlab::AppLogger.error("Secure File Parser Failure (#{id}): #{err.message} - #{parser.error}.") + nil + end + end + private def assign_checksum diff --git a/app/models/clusters/agents/implicit_authorization.rb b/app/models/clusters/agents/implicit_authorization.rb index 9f7f653ed65..a365ccdc568 100644 --- a/app/models/clusters/agents/implicit_authorization.rb +++ b/app/models/clusters/agents/implicit_authorization.rb @@ -16,7 +16,7 @@ module Clusters end def config - nil + {} end end end diff --git a/app/models/concerns/approvable.rb b/app/models/concerns/approvable.rb index 1566c53217d..55e138d84fb 100644 --- a/app/models/concerns/approvable.rb +++ b/app/models/concerns/approvable.rb @@ -50,11 +50,11 @@ module Approvable approvals.where(user: user).any? end - def can_be_approved_by?(user) + def eligible_for_approval_by?(user) user && !approved_by?(user) && user.can?(:approve_merge_request, self) end - def can_be_unapproved_by?(user) + def eligible_for_unapproval_by?(user) user && approved_by?(user) && user.can?(:approve_merge_request, self) end end diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 88f577c3e23..14be924f9da 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -174,6 +174,13 @@ module AtomicInternalId # # bulk_insert(attributes) # end + # + # - track_#{scope}_#{column}! + # This method can be used to set a new greatest IID value during import operations. + # + # Example: + # + # MyClass.track_project_iid!(project, value) def define_singleton_internal_id_methods(scope, column, init) define_singleton_method("with_#{scope}_#{column}_supply") do |scope_value, &block| subject = find_by(scope => scope_value) || self @@ -183,6 +190,16 @@ module AtomicInternalId supply = Supply.new(-> { InternalId.generate_next(subject, scope_attrs, usage, init) }) block.call(supply) end + + define_singleton_method("track_#{scope}_#{column}!") do |scope_value, value| + InternalId.track_greatest( + self, + ::AtomicInternalId.scope_attrs(scope_value), + ::AtomicInternalId.scope_usage(self), + value, + init + ) + end end end diff --git a/app/models/concerns/boards/listable.rb b/app/models/concerns/boards/listable.rb index b9827a79422..b09ef7e612d 100644 --- a/app/models/concerns/boards/listable.rb +++ b/app/models/concerns/boards/listable.rb @@ -13,7 +13,7 @@ module Boards scope :ordered, -> { order(:list_type, :position) } scope :destroyable, -> { where(list_type: list_types.slice(*destroyable_types).values) } scope :movable, -> { where(list_type: list_types.slice(*movable_types).values) } - scope :without_types, ->(list_types) { where.not(list_type: list_types) } + scope :with_types, ->(list_types) { where(list_type: list_types) } class << self def preload_preferences_for_user(lists, user) diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 9ee0fd1db1d..ec0cf36d875 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -237,3 +237,5 @@ module CacheMarkdownField end end end + +CacheMarkdownField.prepend_mod diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 71b26b70bbf..ff884984099 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -80,7 +80,7 @@ module Ci end def id_tokens? - !!metadata&.id_tokens? + metadata&.id_tokens.present? end def id_tokens=(value) diff --git a/app/models/concerns/ci/partitionable.rb b/app/models/concerns/ci/partitionable.rb index 710ee1ba64f..df803180e77 100644 --- a/app/models/concerns/ci/partitionable.rb +++ b/app/models/concerns/ci/partitionable.rb @@ -19,7 +19,32 @@ module Ci extend ActiveSupport::Concern include ::Gitlab::Utils::StrongMemoize + module Testing + InclusionError = Class.new(StandardError) + + PARTITIONABLE_MODELS = %w[ + CommitStatus + Ci::BuildMetadata + Ci::Stage + Ci::JobArtifact + Ci::PipelineVariable + Ci::Pipeline + ].freeze + + def self.check_inclusion(klass) + return if PARTITIONABLE_MODELS.include?(klass.name) + + raise Partitionable::Testing::InclusionError, + "#{klass} must be included in PARTITIONABLE_MODELS" + + rescue InclusionError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + end + included do + Partitionable::Testing.check_inclusion(self) + before_validation :set_partition_id, on: :create validates :partition_id, presence: true @@ -37,6 +62,8 @@ module Ci def partitionable(scope:) define_method(:partition_scope_value) do strong_memoize(:partition_scope_value) do + next Ci::Pipeline.current_partition_value if respond_to?(:importing?) && importing? + record = scope.to_proc.call(self) record.respond_to?(:partition_id) ? record.partition_id : record end diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 64d178b7507..03e062a9855 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -95,7 +95,7 @@ module CounterAttribute next if increment_value == 0 transaction do - unsafe_update_counters(id, attribute => increment_value) + update_counters_with_lease({ attribute => increment_value }) redis_state { |redis| redis.del(flushed_key) } new_db_value = reset.read_attribute(attribute) end @@ -130,9 +130,18 @@ module CounterAttribute end end - def clear_counter!(attribute) + def update_counters_with_lease(increments) + detect_race_on_record(log_fields: { caller: __method__, attributes: increments.keys }) do + self.class.update_counters(id, increments) + end + end + + def reset_counter!(attribute) if counter_attribute_enabled?(attribute) - redis_state { |redis| redis.del(counter_key(attribute)) } + detect_race_on_record(log_fields: { caller: __method__, attributes: attribute }) do + update!(attribute => 0) + clear_counter!(attribute) + end log_clear_counter(attribute) end @@ -164,14 +173,20 @@ module CounterAttribute private + def database_lock_key + "project:{#{project_id}}:#{self.class}:#{id}" + end + def steal_increments(increment_key, flushed_key) redis_state do |redis| redis.eval(LUA_STEAL_INCREMENT_SCRIPT, keys: [increment_key, flushed_key]) end end - def unsafe_update_counters(id, increments) - self.class.update_counters(id, increments) + def clear_counter!(attribute) + redis_state do |redis| + redis.del(counter_key(attribute)) + end end def execute_after_flush_callbacks @@ -192,6 +207,44 @@ module CounterAttribute # a worker is already updating the counters end + # detect_race_on_record uses a lease to monitor access + # to the project statistics row. This is needed to detect + # concurrent attempts to increment columns, which could result in a + # race condition. + # + # As the purpose is to detect and warn concurrent attempts, + # it falls back to direct update on the row if it fails to obtain the lease. + # + # It does not guarantee that there will not be any concurrent updates. + def detect_race_on_record(log_fields: {}) + return yield unless Feature.enabled?(:counter_attribute_db_lease_for_update, project) + + # Ensure attributes is always an array before we log + log_fields[:attributes] = Array(log_fields[:attributes]) + + Gitlab::AppLogger.info( + message: 'Acquiring lease for project statistics update', + project_statistics_id: id, + project_id: project.id, + **log_fields, + **Gitlab::ApplicationContext.current + ) + + in_lock(database_lock_key, retries: 0) do + yield + end + rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError + Gitlab::AppLogger.warn( + message: 'Concurrent project statistics update detected', + project_statistics_id: id, + project_id: project.id, + **log_fields, + **Gitlab::ApplicationContext.current + ) + + yield + end + def log_increment_counter(attribute, increment, new_value) payload = Gitlab::ApplicationContext.current.merge( message: 'Increment counter attribute', diff --git a/app/models/concerns/has_wiki.rb b/app/models/concerns/has_wiki.rb index 89bcabafb84..53016ce62f4 100644 --- a/app/models/concerns/has_wiki.rb +++ b/app/models/concerns/has_wiki.rb @@ -8,7 +8,7 @@ module HasWiki end def create_wiki - wiki.wiki + wiki.create_wiki_repository true rescue Wiki::CouldNotCreateWikiError errors.add(:base, _('Failed to create wiki')) diff --git a/app/models/concerns/integrations/base_data_fields.rb b/app/models/concerns/integrations/base_data_fields.rb index 2870922d90d..4319d63abb9 100644 --- a/app/models/concerns/integrations/base_data_fields.rb +++ b/app/models/concerns/integrations/base_data_fields.rb @@ -5,8 +5,6 @@ module Integrations extend ActiveSupport::Concern included do - # TODO: Once we rename the tables we can't rely on `table_name` anymore. - # https://gitlab.com/gitlab-org/gitlab/-/issues/331953 belongs_to :integration, inverse_of: self.table_name.to_sym, foreign_key: :integration_id validates :integration, presence: true diff --git a/app/models/concerns/integrations/has_web_hook.rb b/app/models/concerns/integrations/has_web_hook.rb index 5fd71f3d72f..e622faf4a51 100644 --- a/app/models/concerns/integrations/has_web_hook.rb +++ b/app/models/concerns/integrations/has_web_hook.rb @@ -6,7 +6,7 @@ module Integrations included do after_save :update_web_hook!, if: :activated? - has_one :service_hook, inverse_of: :integration, foreign_key: :service_id + has_one :service_hook, inverse_of: :integration, foreign_key: :integration_id end # Return the URL to be used for the webhook. diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index b81a9b51e1c..f8389865f91 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -33,6 +33,7 @@ module Issuable DESCRIPTION_LENGTH_MAX = 1.megabyte DESCRIPTION_HTML_LENGTH_MAX = 5.megabytes SEARCHABLE_FIELDS = %w(title description).freeze + MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS = 200 STATE_ID_MAP = { opened: 1, @@ -95,6 +96,7 @@ module Issuable # to avoid breaking the existing Issuables which may have their descriptions longer validates :description, length: { maximum: DESCRIPTION_LENGTH_MAX }, allow_blank: true, on: :create validate :description_max_length_for_new_records_is_valid, on: :update + validate :validate_assignee_size_length, unless: :importing? before_validation :truncate_description_on_import! @@ -166,6 +168,11 @@ module Issuable def locking_enabled? false end + + def max_number_of_assignees_or_reviewers_message + # Assignees will be included in https://gitlab.com/gitlab-org/gitlab/-/issues/368936 + format(_("total must be less than or equal to %{size}"), size: MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS) + end end # We want to use optimistic lock for cases when only title or description are involved @@ -227,11 +234,19 @@ module Issuable def truncate_description_on_import! self.description = description&.slice(0, Issuable::DESCRIPTION_LENGTH_MAX) if importing? end + + def validate_assignee_size_length + return true unless Feature.enabled?(:limit_assignees_per_issuable) + return true unless assignees.size > MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS + + errors.add :assignees, + -> (_object, _data) { self.class.max_number_of_assignees_or_reviewers_message } + end end class_methods do def participant_includes - [:assignees, :author, { notes: [:author, :award_emoji] }] + [:author, :award_emoji, { notes: [:author, :award_emoji, :system_note_metadata] }] end # Searches for records with a matching title. @@ -383,10 +398,12 @@ module Issuable milestone_table = Milestone.arel_table grouping_columns << milestone_table[:id] grouping_columns << milestone_table[:due_date] - elsif %w(merged_at_desc merged_at_asc).include?(sort) + elsif %w(merged_at_desc merged_at_asc merged_at).include?(sort) + grouping_columns << MergeRequest::Metrics.arel_table[:id] grouping_columns << MergeRequest::Metrics.arel_table[:merged_at] - elsif %w(closed_at_desc closed_at_asc).include?(sort) - grouping_columns << MergeRequest::Metrics.arel_table[:closed_at] + elsif %w(closed_at_desc closed_at_asc closed_at).include?(sort) + grouping_columns << MergeRequest::Metrics.arel_table[:id] + grouping_columns << MergeRequest::Metrics.arel_table[:latest_closed_at] end grouping_columns @@ -431,7 +448,16 @@ module Issuable end def assignee_or_author?(user) - author_id == user.id || assignees.exists?(user.id) + author_id == user.id || assignee?(user) + end + + def assignee?(user) + # Necessary so we can preload the association and avoid N + 1 queries + if assignees.loaded? + assignees.to_a.include?(user) + else + assignees.exists?(user.id) + end end def today? @@ -630,6 +656,14 @@ module Issuable def draftless_title_changed(old_title) old_title != title end + + def read_ability_for(participable_source) + return super if participable_source == self + + name = participable_source.try(:issuable_ability_name) || :read_issuable_participables + + { name: name, subject: self } + end end Issuable.prepend_mod_with('Issuable') diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index 8130adf05f1..6035cb87c9b 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -152,7 +152,9 @@ module Participable end def source_visible_to_user?(source, user) - Ability.allowed?(user, "read_#{source.model_name.element}".to_sym, source) + ability = read_ability_for(source) + + Ability.allowed?(user, ability[:name], ability[:subject]) end def filter_by_ability(participants) @@ -172,6 +174,14 @@ module Participable participant.can?(:read_project, project) end end + + # Returns Hash containing ability name and subject needed to read a specific participable. + # Should be overridden if a different ability is required. + def read_ability_for(participable_source) + name = participable_source.try(:to_ability_name) || participable_source.model_name.element + + { name: "read_#{name}".to_sym, subject: participable_source } + end end Participable.prepend_mod_with('Participable') diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 5b759dedb26..262839a3fa6 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -17,6 +17,9 @@ module Routable def self.find_by_full_path(path, follow_redirects: false, route_scope: Route, redirect_route_scope: RedirectRoute) return unless path.present? + # Convert path to string to prevent DB error: function lower(integer) does not exist + path = path.to_s + # Case sensitive match first (it's cheaper and the usual case) # If we didn't have an exact match, we perform a case insensitive search # diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index d53594eb5af..5b74e88429c 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -3,13 +3,10 @@ module Timebox extend ActiveSupport::Concern - include AtomicInternalId include CacheMarkdownField include Gitlab::SQL::Pattern - include IidRoutes include Referable include StripAttribute - include FromUnion TimeboxStruct = Struct.new(:title, :name, :id, :class_name) do # Ensure these models match the interface required for exporting @@ -42,39 +39,19 @@ module Timebox alias_method :timebox_id, :id - validates :group, presence: true, unless: :project - validates :project, presence: true, unless: :group - - validate :timebox_type_check validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } validate :dates_within_4_digits cache_markdown_field :title, pipeline: :single_line cache_markdown_field :description, issuable_reference_expansion_enabled: true - belongs_to :project - belongs_to :group - has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests - scope :of_projects, ->(ids) { where(project_id: ids) } - scope :of_groups, ->(ids) { where(group_id: ids) } scope :closed, -> { with_state(:closed) } - scope :for_projects, -> { where(group: nil).includes(:project) } scope :with_title, -> (title) { where(title: title) } - scope :for_projects_and_groups, -> (projects, groups) do - projects = projects.compact if projects.is_a? Array - projects = [] if projects.nil? - - groups = groups.compact if groups.is_a? Array - groups = [] if groups.nil? - - from_union([where(project_id: projects), where(group_id: groups)], remove_duplicates: false) - end - # A timebox is within the timeframe (start_date, end_date) if it overlaps # with that timeframe: # @@ -132,10 +109,6 @@ module Timebox end end - def count_by_state - reorder(nil).group(:state).count - end - def predefined_id?(id) [Any.id, None.id, Upcoming.id, Started.id].include?(id) end @@ -145,29 +118,8 @@ module Timebox end end - ## - # Returns the String necessary to reference a Timebox in Markdown. Group - # timeboxes only support name references, and do not support cross-project - # references. - # - # format - Symbol format to use (default: :iid, optional: :name) - # - # Examples: - # - # Milestone.first.to_reference # => "%1" - # Iteration.first.to_reference(format: :name) # => "*iteration:\"goal\"" - # Milestone.first.to_reference(cross_namespace_project) # => "gitlab-org/gitlab-foss%1" - # Iteration.first.to_reference(same_namespace_project) # => "gitlab-foss*iteration:1" - # - def to_reference(from = nil, format: :name, full: false) - format_reference = timebox_format_reference(format) - reference = "#{self.class.reference_prefix}#{format_reference}" - - if project - "#{project.to_reference_base(from, full: full)}#{reference}" - else - reference - end + def to_reference + raise NotImplementedError end def reference_link_text(from = nil) @@ -182,20 +134,12 @@ module Timebox model_name.singular end - def group_timebox? - group_id.present? - end - - def project_timebox? - project_id.present? - end - def safe_title title.to_slug.normalize.to_s end def resource_parent - group || project + raise NotImplementedError end def to_ability_name @@ -203,13 +147,7 @@ module Timebox end def merge_requests_enabled? - if group_timebox? - # Assume that groups have at least one project with merge requests enabled. - # Otherwise, we would need to load all of the projects from the database. - true - elsif project_timebox? - project&.merge_requests_enabled? - end + raise NotImplementedError end def weight_available? @@ -218,28 +156,6 @@ module Timebox private - def timebox_format_reference(format = :iid) - raise ArgumentError, _('Unknown format') unless [:iid, :name].include?(format) - - if group_timebox? && format == :iid - raise ArgumentError, _('Cannot refer to a group %{timebox_type} by an internal id!') % { timebox_type: timebox_name } - end - - if format == :name && !name.include?('"') - %("#{name}") - else - iid - end - end - - # Timebox should be either a project timebox or a group timebox - def timebox_type_check - if group_id && project_id - field = project_id_changed? ? :project_id : :group_id - errors.add(field, _("%{timebox_name} should belong either to a project or a group.") % { timebox_name: timebox_name }) - end - end - def start_date_should_be_less_than_due_date if due_date <= start_date errors.add(:due_date, _("must be greater than start date")) diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 94ac2405f61..2563fd484f1 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -4,6 +4,7 @@ class DeployKey < Key include FromUnion include IgnorableColumns include PolicyActor + include Presentable has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :deploy_keys_projects diff --git a/app/models/deployment.rb b/app/models/deployment.rb index dafcbc593be..20841bc14cd 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -105,6 +105,7 @@ class Deployment < ApplicationRecord after_transition any => :running do |deployment| next unless deployment.project.ci_forward_deployment_enabled? + next if Feature.enabled?(:prevent_outdated_deployment_jobs, deployment.project) deployment.run_after_commit do Deployments::DropOlderDeploymentsWorker.perform_async(id) @@ -282,27 +283,11 @@ class Deployment < ApplicationRecord end def manual_actions - environment_manual_actions - end - - def other_manual_actions - @other_manual_actions ||= deployable.try(:other_manual_actions) - end - - def environment_manual_actions - @environment_manual_actions ||= deployable.try(:environment_manual_actions) + @manual_actions ||= deployable.try(:other_manual_actions) end def scheduled_actions - environment_scheduled_actions - end - - def environment_scheduled_actions - @environment_scheduled_actions ||= deployable.try(:environment_scheduled_actions) - end - - def other_scheduled_actions - @other_scheduled_actions ||= deployable.try(:other_scheduled_actions) + @scheduled_actions ||= deployable.try(:other_scheduled_actions) end def playable_build diff --git a/app/models/diff_viewer/server_side.rb b/app/models/diff_viewer/server_side.rb index 0877c9dddec..a1defb2594f 100644 --- a/app/models/diff_viewer/server_side.rb +++ b/app/models/diff_viewer/server_side.rb @@ -10,6 +10,9 @@ module DiffViewer end def prepare! + return if Feature.enabled?(:disable_load_entire_blob_for_diff_viewer, diff_file.repository.project) + + # TODO: remove this after resolving #342703 diff_file.old_blob&.load_all_data! diff_file.new_blob&.load_all_data! end diff --git a/app/models/environment.rb b/app/models/environment.rb index 4b98cd02e3b..2d3f342953f 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -71,7 +71,7 @@ class Environment < ApplicationRecord validate :safe_external_url validate :merge_request_not_changed - delegate :manual_actions, :other_manual_actions, to: :last_deployment, allow_nil: true + delegate :manual_actions, to: :last_deployment, allow_nil: true delegate :auto_rollback_enabled?, to: :project scope :available, -> { with_state(:available) } @@ -332,9 +332,9 @@ class Environment < ApplicationRecord end def actions_for(environment) - return [] unless other_manual_actions + return [] unless manual_actions - other_manual_actions.select do |action| + manual_actions.select do |action| action.expanded_environment_name == environment end end @@ -441,11 +441,15 @@ class Environment < ApplicationRecord end def auto_stop_in=(value) - return unless value + if value.nil? + # Handles edge case when auto_stop_at is already set and the new value is nil. + # Possible by setting `auto_stop_in: null` in the CI configuration yml. + self.auto_stop_at = nil - parser = ::Gitlab::Ci::Build::DurationParser.new(value) + return + end - return if parser.seconds_from_now.nil? && auto_stop_at.nil? + parser = ::Gitlab::Ci::Build::DurationParser.new(value) self.auto_stop_at = parser.seconds_from_now rescue ChronicDuration::DurationParseError => ex @@ -540,7 +544,7 @@ class Environment < ApplicationRecord self.class.tiers[:development] when /(test|tst|int|ac(ce|)pt|qa|qc|control|quality)/i self.class.tiers[:testing] - when /(st(a|)g|mod(e|)l|pre|demo)/i + when /(st(a|)g|mod(e|)l|pre|demo|non)/i self.class.tiers[:staging] when /(pr(o|)d|live)/i self.class.tiers[:production] diff --git a/app/models/event.rb b/app/models/event.rb index a20ca0dc423..4c1793d3f13 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -10,9 +10,6 @@ class Event < ApplicationRecord include UsageStatistics include ShaAttribute - # TODO https://gitlab.com/gitlab-org/gitlab/-/issues/358088 - default_scope { reorder(nil) } # rubocop:disable Cop/DefaultScope - ACTIONS = HashWithIndifferentAccess.new( created: 1, updated: 2, @@ -281,6 +278,7 @@ class Event < ApplicationRecord "opened" end end + # rubocop: enable Metrics/CyclomaticComplexity # rubocop: enable Metrics/PerceivedComplexity @@ -448,9 +446,9 @@ class Event < ApplicationRecord def design_action_names { - created: _('added'), - updated: _('updated'), - destroyed: _('removed') + created: 'added', + updated: 'updated', + destroyed: 'removed' } end diff --git a/app/models/group.rb b/app/models/group.rb index 1445e71b0bc..38623d91705 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -904,11 +904,7 @@ class Group < Namespace end def packages_policy_subject - if Feature.enabled?(:read_package_policy_rule, self) - ::Packages::Policies::Group.new(self) - else - self - end + ::Packages::Policies::Group.new(self) end def update_two_factor_requirement_for_members diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index 7005c8593bd..15949570f9c 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -8,7 +8,7 @@ class GroupGroupLink < ApplicationRecord validates :shared_group, presence: true validates :shared_group_id, uniqueness: { scope: [:shared_with_group_id], - message: _('The group has already been shared with this group') } + message: N_('The group has already been shared with this group') } validates :shared_with_group, presence: true validates :group_access, inclusion: { in: Gitlab::Access.all_values }, presence: true diff --git a/app/models/group_label.rb b/app/models/group_label.rb index ff14529c6e6..0d2eb524929 100644 --- a/app/models/group_label.rb +++ b/app/models/group_label.rb @@ -2,6 +2,7 @@ class GroupLabel < Label belongs_to :group + belongs_to :parent_container, foreign_key: :group_id, class_name: 'Group' validates :group, presence: true diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index bcbf43ee38b..dcba136d163 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -55,13 +55,6 @@ class ProjectHook < WebHook redis.set(key, time) if !prev || prev < time end end - - private - - override :web_hooks_disable_failed? - def web_hooks_disable_failed? - Feature.enabled?(:web_hooks_disable_failed, project) - end end ProjectHook.prepend_mod_with('ProjectHook') diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb index 80e167b350b..27119d3a95a 100644 --- a/app/models/hooks/service_hook.rb +++ b/app/models/hooks/service_hook.rb @@ -4,7 +4,7 @@ class ServiceHook < WebHook include Presentable extend ::Gitlab::Utils::Override - belongs_to :integration, foreign_key: :service_id + belongs_to :integration validates :integration, presence: true def execute(data, hook_name = 'service_hook') diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 84ee23d77ce..71794964c99 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -7,7 +7,7 @@ class WebHook < ApplicationRecord MAX_FAILURES = 100 FAILURE_THRESHOLD = 3 # three strikes - INITIAL_BACKOFF = 10.minutes + INITIAL_BACKOFF = 1.minute MAX_BACKOFF = 1.day BACKOFF_GROWTH_FACTOR = 2.0 @@ -53,18 +53,24 @@ class WebHook < ApplicationRecord where('recent_failures > ? OR disabled_until >= ?', FAILURE_THRESHOLD, Time.current) end + def self.web_hooks_disable_failed?(hook) + Feature.enabled?(:web_hooks_disable_failed, hook.parent) + end + def executable? !temporarily_disabled? && !permanently_disabled? end def temporarily_disabled? return false unless web_hooks_disable_failed? + return false if recent_failures <= FAILURE_THRESHOLD disabled_until.present? && disabled_until >= Time.current end def permanently_disabled? return false unless web_hooks_disable_failed? + return false if disabled_until.present? recent_failures > FAILURE_THRESHOLD end @@ -112,17 +118,26 @@ class WebHook < ApplicationRecord save(validate: false) end + # Don't actually back-off until FAILURE_THRESHOLD failures have been seen + # we mark the grace-period using the recent_failures counter def backoff! return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?) - assign_attributes(disabled_until: next_backoff.from_now, backoff_count: backoff_count.succ.clamp(0, MAX_FAILURES)) + attrs = { recent_failures: recent_failures + 1 } + + if recent_failures >= FAILURE_THRESHOLD + attrs[:backoff_count] = backoff_count.succ.clamp(1, MAX_FAILURES) + attrs[:disabled_until] = next_backoff.from_now + end + + assign_attributes(attrs) save(validate: false) end def failed! return unless recent_failures < MAX_FAILURES - assign_attributes(recent_failures: recent_failures + 1) + assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: recent_failures + 1) save(validate: false) end @@ -186,7 +201,7 @@ class WebHook < ApplicationRecord private def web_hooks_disable_failed? - Feature.enabled?(:web_hooks_disable_failed) + self.class.web_hooks_disable_failed?(self) end def initialize_url_variables diff --git a/app/models/incident_management/timeline_event.rb b/app/models/incident_management/timeline_event.rb index dd0d3c6585d..735d4e4298c 100644 --- a/app/models/incident_management/timeline_event.rb +++ b/app/models/incident_management/timeline_event.rb @@ -18,7 +18,13 @@ module IncidentManagement validates :project, :incident, :occurred_at, presence: true validates :action, presence: true, length: { maximum: 128 } - validates :note, :note_html, presence: true, length: { maximum: 10_000 } + validates :note, presence: true, length: { maximum: 10_000 } + validates :note_html, length: { maximum: 10_000 } + + has_many :timeline_event_tag_links, class_name: 'IncidentManagement::TimelineEventTagLink' + has_many :timeline_event_tags, + class_name: 'IncidentManagement::TimelineEventTag', + through: :timeline_event_tag_links scope :order_occurred_at_asc_id_asc, -> { reorder(occurred_at: :asc, id: :asc) } end diff --git a/app/models/incident_management/timeline_event_tag.rb b/app/models/incident_management/timeline_event_tag.rb new file mode 100644 index 00000000000..cde3afcaa16 --- /dev/null +++ b/app/models/incident_management/timeline_event_tag.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module IncidentManagement + class TimelineEventTag < ApplicationRecord + self.table_name = 'incident_management_timeline_event_tags' + + belongs_to :project, inverse_of: :incident_management_timeline_event_tags + + has_many :timeline_event_tag_links, + class_name: 'IncidentManagement::TimelineEventTagLink' + + has_many :timeline_events, + class_name: 'IncidentManagement::TimelineEvent', + through: :timeline_event_tag_links + + validates :name, presence: true, format: { with: /\A[^,]+\z/ } + validates :name, uniqueness: { scope: :project_id } + validates :name, length: { maximum: 255 } + end +end diff --git a/app/models/incident_management/timeline_event_tag_link.rb b/app/models/incident_management/timeline_event_tag_link.rb new file mode 100644 index 00000000000..912339717a8 --- /dev/null +++ b/app/models/incident_management/timeline_event_tag_link.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module IncidentManagement + class TimelineEventTagLink < ApplicationRecord + self.table_name = 'incident_management_timeline_event_tag_links' + + belongs_to :timeline_event_tag, class_name: 'IncidentManagement::TimelineEventTag' + + belongs_to :timeline_event, class_name: 'IncidentManagement::TimelineEvent' + end +end diff --git a/app/models/integration.rb b/app/models/integration.rb index aecf9529a14..23688a87cbd 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -147,6 +147,8 @@ class Integration < ApplicationRecord fields << ::Integrations::Field.new(name: name, integration_class: self, **attrs) case storage + when :attribute + # noop when :properties prop_accessor(name) when :data_fields @@ -155,7 +157,7 @@ class Integration < ApplicationRecord raise ArgumentError, "Unknown field storage: #{storage}" end - boolean_accessor(name) if attrs[:type] == 'checkbox' + boolean_accessor(name) if attrs[:type] == 'checkbox' && storage != :attribute end # :nocov: diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index c9407aa738e..ab0fdbd777f 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -15,7 +15,77 @@ module Integrations TAG_KEY_VALUE_RE = %r{\A [\w-]+ : .*\S.* \z}x.freeze - prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env, :datadog_tags + field :datadog_site, + placeholder: DEFAULT_DOMAIN, + help: -> do + ERB::Util.html_escape( + s_('DatadogIntegration|The Datadog site to send data to. To send data to the EU site, use %{codeOpen}datadoghq.eu%{codeClose}.') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe + } + end + + field :api_url, + exposes_secrets: true, + title: -> { s_('DatadogIntegration|API URL') }, + help: -> { s_('DatadogIntegration|(Advanced) The full URL for your Datadog site.') } + + field :api_key, + type: 'password', + title: -> { _('API key') }, + non_empty_password_title: -> { s_('ProjectService|Enter new API key') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current API key') }, + help: -> do + ERB::Util.html_escape( + s_('DatadogIntegration|%{linkOpen}API key%{linkClose} used for authentication with Datadog.') + ) % { + linkOpen: %Q{<a href="#{URL_API_KEYS_DOCS}" target="_blank" rel="noopener noreferrer">}.html_safe, + linkClose: '</a>'.html_safe + } + end, + required: true + + field :archive_trace_events, + storage: :attribute, + type: 'checkbox', + title: -> { s_('Logs') }, + checkbox_label: -> { s_('Enable logs collection') }, + help: -> { s_('When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces.') } + + field :datadog_service, + title: -> { s_('DatadogIntegration|Service') }, + placeholder: 'gitlab-ci', + help: -> { s_('DatadogIntegration|Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments.') } + + field :datadog_env, + title: -> { s_('DatadogIntegration|Environment') }, + placeholder: 'ci', + help: -> do + ERB::Util.html_escape( + s_('DatadogIntegration|For self-managed deployments, set the %{codeOpen}env%{codeClose} tag for all the data sent to Datadog. %{linkOpen}How do I use tags?%{linkClose}') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe, + linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, + linkClose: '</a>'.html_safe + } + end + + field :datadog_tags, + type: 'textarea', + title: -> { s_('DatadogIntegration|Tags') }, + placeholder: "tag:value\nanother_tag:value", + help: -> do + ERB::Util.html_escape( + s_('DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe, + linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, + linkClose: '</a>'.html_safe + } + end before_validation :strip_properties @@ -68,87 +138,6 @@ module Integrations 'datadog' end - def fields - [ - { - type: 'text', - name: 'datadog_site', - placeholder: DEFAULT_DOMAIN, - help: ERB::Util.html_escape( - s_('DatadogIntegration|The Datadog site to send data to. To send data to the EU site, use %{codeOpen}datadoghq.eu%{codeClose}.') - ) % { - codeOpen: '<code>'.html_safe, - codeClose: '</code>'.html_safe - }, - required: false - }, - { - type: 'text', - name: 'api_url', - title: s_('DatadogIntegration|API URL'), - help: s_('DatadogIntegration|(Advanced) The full URL for your Datadog site.'), - required: false - }, - { - type: 'password', - name: 'api_key', - title: _('API key'), - non_empty_password_title: s_('ProjectService|Enter new API key'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current API key'), - help: ERB::Util.html_escape( - s_('DatadogIntegration|%{linkOpen}API key%{linkClose} used for authentication with Datadog.') - ) % { - linkOpen: %Q{<a href="#{URL_API_KEYS_DOCS}" target="_blank" rel="noopener noreferrer">}.html_safe, - linkClose: '</a>'.html_safe - }, - required: true - }, - { - type: 'checkbox', - name: 'archive_trace_events', - title: s_('Logs'), - checkbox_label: s_('Enable logs collection'), - help: s_('When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces.'), - required: false - }, - { - type: 'text', - name: 'datadog_service', - title: s_('DatadogIntegration|Service'), - placeholder: 'gitlab-ci', - help: s_('DatadogIntegration|Tag all data from this GitLab instance in Datadog. Useful when managing several self-managed deployments.') - }, - { - type: 'text', - name: 'datadog_env', - title: s_('DatadogIntegration|Environment'), - placeholder: 'ci', - help: ERB::Util.html_escape( - s_('DatadogIntegration|For self-managed deployments, set the %{codeOpen}env%{codeClose} tag for all the data sent to Datadog. %{linkOpen}How do I use tags?%{linkClose}') - ) % { - codeOpen: '<code>'.html_safe, - codeClose: '</code>'.html_safe, - linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, - linkClose: '</a>'.html_safe - } - }, - { - type: 'textarea', - name: 'datadog_tags', - title: s_('DatadogIntegration|Tags'), - placeholder: "tag:value\nanother_tag:value", - help: ERB::Util.html_escape( - s_('DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}') - ) % { - codeOpen: '<code>'.html_safe, - codeClose: '</code>'.html_safe, - linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, - linkClose: '</a>'.html_safe - } - } - ] - end - override :hook_url def hook_url url = api_url.presence || sprintf(URL_TEMPLATE, datadog_domain: datadog_domain) diff --git a/app/models/integrations/harbor.rb b/app/models/integrations/harbor.rb index 58eabcfd378..01a04743d5d 100644 --- a/app/models/integrations/harbor.rb +++ b/app/models/integrations/harbor.rb @@ -3,14 +3,33 @@ require 'uri' module Integrations class Harbor < Integration - prop_accessor :url, :project_name, :username, :password - validates :url, public_url: true, presence: true, addressable_url: { allow_localhost: false, allow_local_network: false }, if: :activated? validates :project_name, presence: true, if: :activated? validates :username, presence: true, if: :activated? validates :password, format: { with: ::Ci::Maskable::REGEX }, if: :activated? - before_validation :reset_username_and_password + field :url, + title: -> { s_('HarborIntegration|Harbor URL') }, + placeholder: 'https://demo.goharbor.io', + help: -> { s_('HarborIntegration|Base URL of the Harbor instance.') }, + exposes_secrets: true, + required: true + + field :project_name, + title: -> { s_('HarborIntegration|Harbor project name') }, + help: -> { s_('HarborIntegration|The name of the project in Harbor.') } + + field :username, + title: -> { s_('HarborIntegration|Harbor username') }, + required: true + + field :password, + type: 'password', + title: -> { s_('HarborIntegration|Harbor password') }, + help: -> { s_('HarborIntegration|Password for your Harbor username.') }, + non_empty_password_title: -> { s_('HarborIntegration|Enter new Harbor password') }, + non_empty_password_help: -> { s_('HarborIntegration|Leave blank to use your current password.') }, + required: true def title 'Harbor' @@ -21,7 +40,7 @@ module Integrations end def help - s_("HarborIntegration|After the Harbor integration is activated, global variables '$HARBOR_USERNAME', '$HARBOR_HOST', '$HARBOR_OCI', '$HARBOR_PASSWORD', '$HARBOR_URL' and '$HARBOR_PROJECT' will be created for CI/CD use.") + s_("HarborIntegration|After the Harbor integration is activated, global variables `$HARBOR_USERNAME`, `$HARBOR_HOST`, `$HARBOR_OCI`, `$HARBOR_PASSWORD`, `$HARBOR_URL` and `$HARBOR_PROJECT` will be created for CI/CD use.") end def hostname @@ -46,40 +65,6 @@ module Integrations client.ping end - def fields - [ - { - type: 'text', - name: 'url', - title: s_('HarborIntegration|Harbor URL'), - placeholder: 'https://demo.goharbor.io', - help: s_('HarborIntegration|Base URL of the Harbor instance.'), - required: true - }, - { - type: 'text', - name: 'project_name', - title: s_('HarborIntegration|Harbor project name'), - help: s_('HarborIntegration|The name of the project in Harbor.') - }, - { - type: 'text', - name: 'username', - title: s_('HarborIntegration|Harbor username'), - required: true - }, - { - type: 'password', - name: 'password', - title: s_('HarborIntegration|Harbor password'), - help: s_('HarborIntegration|Password for your Harbor username.'), - non_empty_password_title: s_('HarborIntegration|Enter new Harbor password'), - non_empty_password_help: s_('HarborIntegration|Leave blank to use your current password.'), - required: true - } - ] - end - def ci_variables return [] unless activated? @@ -100,15 +85,5 @@ module Integrations def client @client ||= ::Gitlab::Harbor::Client.new(self) end - - def reset_username_and_password - if url_changed? && !password_touched? - self.password = nil - end - - if url_changed? && !username_touched? - self.username = nil - end - end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 754591b8017..ea7acf9a5d1 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -40,10 +40,15 @@ class Issue < ApplicationRecord SORTING_PREFERENCE_FIELD = :issues_sort - # Types of issues that should be displayed on lists across the app - # for example, project issues list, group issues list and issue boards. - # Some issue types, like test cases, should be hidden by default. - TYPES_FOR_LIST = %w(issue incident).freeze + # Types of issues that should be displayed on issue lists across the app + # for example, project issues list, group issues list, and issues dashboard. + # + # This should be kept consistent with the enums used for the GraphQL issue list query in + # https://gitlab.com/gitlab-org/gitlab/-/blob/1379c2d7bffe2a8d809f23ac5ef9b4114f789c07/app/assets/javascripts/issues/list/constants.js#L154-158 + TYPES_FOR_LIST = %w(issue incident test_case task).freeze + + # Types of issues that should be displayed on issue board lists + TYPES_FOR_BOARD_LIST = %w(issue incident).freeze belongs_to :project belongs_to :namespace, inverse_of: :issues @@ -107,6 +112,7 @@ class Issue < ApplicationRecord enum issue_type: WorkItems::Type.base_types alias_method :issuing_parent, :project + alias_attribute :issuing_parent_id, :project_id alias_attribute :external_author, :service_desk_reply_to @@ -270,6 +276,10 @@ class Issue < ApplicationRecord end end + def self.participant_includes + [:assignees] + super + end + def next_object_by_relative_position(ignoring: nil, order: :asc) array_mapping_scope = -> (id_expression) do relation = Issue.where(Issue.arel_table[:project_id].eq(id_expression)) diff --git a/app/models/iteration.rb b/app/models/iteration.rb index 71ecbcf1c1a..ed73793c78f 100644 --- a/app/models/iteration.rb +++ b/app/models/iteration.rb @@ -2,6 +2,12 @@ # Placeholder class for model that is implemented in EE class Iteration < ApplicationRecord + include IgnorableColumns + + # TODO https://gitlab.com/gitlab-org/gitlab/-/issues/372125 + # TODO https://gitlab.com/gitlab-org/gitlab/-/issues/372126 + ignore_column :project_id, remove_with: '15.6', remove_after: '2022-09-17' + self.table_name = 'sprints' def self.reference_prefix diff --git a/app/models/jira_connect/public_key.rb b/app/models/jira_connect/public_key.rb new file mode 100644 index 00000000000..8959884861b --- /dev/null +++ b/app/models/jira_connect/public_key.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module JiraConnect + class PublicKey + # Public keys are created with JWT tokens via JiraConnect::CreateAsymmetricJwtService + # They need to be available for third party applications to verify the token. + # This should happen right after the application received the token so public keys + # only need to exist for a few minutes. + REDIS_EXPIRY_TIME = 5.minutes.to_i.freeze + + attr_reader :key, :uuid + + def self.create!(key:) + new(key: key, uuid: Gitlab::UUID.v5(SecureRandom.hex)).save! + end + + def self.find(uuid) + Gitlab::Redis::SharedState.with do |redis| + key = redis.get(redis_key(uuid)) + + raise ActiveRecord::RecordNotFound if key.nil? + + new(key: key, uuid: uuid) + end + end + + def initialize(key:, uuid:) + key = OpenSSL::PKey.read(key) unless key.is_a?(OpenSSL::PKey::RSA) + + @key = key.to_s + @uuid = uuid + rescue OpenSSL::PKey::PKeyError + raise ArgumentError, 'Invalid public key' + end + + def save! + Gitlab::Redis::SharedState.with do |redis| + redis.set(self.class.redis_key(uuid), key, ex: REDIS_EXPIRY_TIME) + end + + self + end + + def self.redis_key(uuid) + "JiraConnect:public_key:uuid=#{uuid}" + end + end +end diff --git a/app/models/jira_connect_installation.rb b/app/models/jira_connect_installation.rb index 0a2d3ba0749..23813fa138f 100644 --- a/app/models/jira_connect_installation.rb +++ b/app/models/jira_connect_installation.rb @@ -21,6 +21,9 @@ class JiraConnectInstallation < ApplicationRecord }) } + scope :direct_installations, -> { joins(:subscriptions) } + scope :proxy_installations, -> { where.not(instance_url: nil) } + def client Atlassian::JiraConnect::Client.new(base_url, shared_secret) end @@ -30,4 +33,20 @@ class JiraConnectInstallation < ApplicationRecord instance_url end + + def audience_url + return unless proxy? + + Gitlab::Utils.append_path(instance_url, '/-/jira_connect') + end + + def audience_installed_event_url + return unless proxy? + + Gitlab::Utils.append_path(instance_url, '/-/jira_connect/events/installed') + end + + def proxy? + instance_url.present? + end end diff --git a/app/models/jira_import_state.rb b/app/models/jira_import_state.rb index 76b5f1def6a..97d6cd00fb8 100644 --- a/app/models/jira_import_state.rb +++ b/app/models/jira_import_state.rb @@ -24,7 +24,7 @@ class JiraImportState < ApplicationRecord validates :project, uniqueness: { conditions: -> { where.not(status: STATUSES.values_at(:failed, :finished)) }, - message: _('Cannot have multiple Jira imports running at the same time') + message: N_('Cannot have multiple Jira imports running at the same time') } before_save :ensure_error_message_size diff --git a/app/models/label.rb b/app/models/label.rb index 6608a0573cb..483d51099b1 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -42,6 +42,7 @@ class Label < ApplicationRecord scope :order_name_asc, -> { reorder(title: :asc) } scope :order_name_desc, -> { reorder(title: :desc) } scope :subscribed_by, ->(user_id) { joins(:subscriptions).where(subscriptions: { user_id: user_id, subscribed: true }) } + scope :with_preloaded_container, -> { preload(parent_container: :route) } scope :top_labels_by_target, -> (target_relation) { label_id_column = arel_table[:id] @@ -59,6 +60,13 @@ class Label < ApplicationRecord .distinct } + scope :for_targets, ->(target_relation) do + joins(:label_links) + .merge(LabelLink.where(target: target_relation)) + .select(arel_table[Arel.star], LabelLink.arel_table[:target_id]) + .with_preloaded_container + end + def self.prioritized(project) joins(:priorities) .where(label_priorities: { project_id: project }) diff --git a/app/models/member.rb b/app/models/member.rb index c5351d5447b..ff1d8f18c25 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -55,7 +55,7 @@ class Member < ApplicationRecord validate :signup_email_valid?, on: :create, if: ->(member) { member.invite_email.present? } validates :user_id, uniqueness: { - message: _('project bots cannot be added to other groups / projects') + message: N_('project bots cannot be added to other groups / projects') }, if: :project_bot? validate :access_level_inclusion @@ -627,7 +627,6 @@ class Member < ApplicationRecord end def blocking_refresh - return true unless Feature.enabled?(:allow_non_blocking_member_refresh) return true if @blocking_refresh.nil? @blocking_refresh diff --git a/app/models/members/member_role.rb b/app/models/members/member_role.rb index 2e8532fa739..b4e3d6874ef 100644 --- a/app/models/members/member_role.rb +++ b/app/models/members/member_role.rb @@ -4,6 +4,15 @@ class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass has_many :members belongs_to :namespace - validates :namespace_id, presence: true + validates :namespace, presence: true validates :base_access_level, presence: true + validate :belongs_to_top_level_namespace + + private + + def belongs_to_top_level_namespace + return if !namespace || namespace.root? + + errors.add(:namespace, s_("must be top-level namespace")) + end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a57cb97e936..fb20d91fa20 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -41,8 +41,6 @@ class MergeRequest < ApplicationRecord 'Ci::CompareCodequalityReportsService' => ->(project) { true } }.freeze - MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS = 100 - belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" @@ -73,6 +71,11 @@ class MergeRequest < ApplicationRecord belongs_to :latest_merge_request_diff, class_name: 'MergeRequestDiff' manual_inverse_association :latest_merge_request_diff, :merge_request + # method overriden in EE + def suggested_reviewer_users + User.none + end + # This is the same as latest_merge_request_diff unless: # 1. There are arguments - in which case we might be trying to force-reload. # 2. This association is already loaded. @@ -238,6 +241,12 @@ class MergeRequest < ApplicationRecord Gitlab::Timeless.timeless(merge_request, &block) end + after_transition any => [:unchecked, :cannot_be_merged_recheck, :checking, :cannot_be_merged_rechecking, :can_be_merged, :cannot_be_merged] do |merge_request, transition| + if Feature.enabled?(:trigger_mr_subscription_on_merge_status_change, merge_request.project) + GraphqlTriggers.merge_request_merge_status_updated(merge_request) + end + end + # rubocop: disable CodeReuse/ServiceClass after_transition [:unchecked, :checking] => :cannot_be_merged do |merge_request, transition| if merge_request.notify_conflict? @@ -269,7 +278,7 @@ class MergeRequest < ApplicationRecord validate :validate_branches, unless: [:allow_broken, :importing?, :closed_or_merged_without_fork?] validate :validate_fork, unless: :closed_or_merged_without_fork? validate :validate_target_project, on: :create - validate :validate_reviewer_and_assignee_size_length, unless: :importing? + validate :validate_reviewer_size_length, unless: :importing? scope :by_source_or_target_branch, ->(branch_name) do where("source_branch = :branch OR target_branch = :branch", branch: branch_name) @@ -438,6 +447,7 @@ class MergeRequest < ApplicationRecord # we'd eventually rename the column for avoiding confusions, but in the mean time # please use `auto_merge_enabled` alias instead of `merge_when_pipeline_succeeds`. alias_attribute :auto_merge_enabled, :merge_when_pipeline_succeeds + alias_attribute :issuing_parent_id, :target_project_id alias_method :issuing_parent, :target_project delegate :builds_with_coverage, to: :head_pipeline, prefix: true, allow_nil: true @@ -602,7 +612,7 @@ class MergeRequest < ApplicationRecord end def self.participant_includes - [:reviewers, :award_emoji] + super + [:assignees, :reviewers] + super end def committers @@ -988,18 +998,12 @@ class MergeRequest < ApplicationRecord 'Source project is not a fork of the target project' end - def self.max_number_of_assignees_or_reviewers_message - # Assignees will be included in https://gitlab.com/gitlab-org/gitlab/-/issues/368936 - _("total must be less than or equal to %{size}") % { size: MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS } - end - - def validate_reviewer_and_assignee_size_length - # Assigness will be added in a subsequent MR https://gitlab.com/gitlab-org/gitlab/-/issues/368936 + def validate_reviewer_size_length return true unless Feature.enabled?(:limit_reviewer_and_assignee_size) return true unless reviewers.size > MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS errors.add :reviewers, - -> (_object, _data) { MergeRequest.max_number_of_assignees_or_reviewers_message } + -> (_object, _data) { self.class.max_number_of_assignees_or_reviewers_message } end def merge_ongoing? @@ -1989,6 +1993,10 @@ class MergeRequest < ApplicationRecord # rubocop: enable CodeReuse/ServiceClass end + def can_suggest_reviewers? + false # overridden in EE + end + private attr_accessor :skip_fetch_ref diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index 36902e43a77..04b322ef5a6 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -25,6 +25,10 @@ class MergeRequestDiffFile < ApplicationRecord return '' if fetched_diff.blank? encode_utf8(fetched_diff) if fetched_diff.respond_to?(:encoding) + rescue StandardError => e + log_exception('Failed fetching merge request diff', e) + + '' end def diff @@ -75,15 +79,19 @@ class MergeRequestDiffFile < ApplicationRecord content rescue StandardError => e + log_exception('Cached external diff export failed', e) + + diff + end + + def log_exception(message, exception) log_payload = { - message: 'Cached external diff export failed', + message: message, merge_request_diff_file_id: id, merge_request_diff_id: merge_request_diff&.id } - Gitlab::ExceptionLogFormatter.format!(e, log_payload) + Gitlab::ExceptionLogFormatter.format!(exception, log_payload) Gitlab::AppLogger.warn(log_payload) - - diff end end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index ff4fadb0f13..da07d8dd9fc 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true class Milestone < ApplicationRecord + include AtomicInternalId include Sortable include Timebox include Milestoneish include FromUnion include Importable + include IidRoutes prepend_mod_with('Milestone') # rubocop: disable Cop/InjectEnterpriseEditionModule @@ -13,6 +15,9 @@ class Milestone < ApplicationRecord ALL = [::Timebox::None, ::Timebox::Any, ::Timebox::Started, ::Timebox::Upcoming].freeze end + belongs_to :project + belongs_to :group + has_many :milestone_releases has_many :releases, through: :milestone_releases @@ -30,13 +35,28 @@ class Milestone < ApplicationRecord .order(:project_id, :group_id, :due_date) end + scope :of_projects, ->(ids) { where(project_id: ids) } + scope :for_projects, -> { where(group: nil).includes(:project) } + scope :for_projects_and_groups, -> (projects, groups) do + projects = projects.compact if projects.is_a? Array + projects = [] if projects.nil? + + groups = groups.compact if groups.is_a? Array + groups = [] if groups.nil? + + from_union([where(project_id: projects), where(group_id: groups)], remove_duplicates: false) + end + scope :order_by_name_asc, -> { order(Arel::Nodes::Ascending.new(arel_table[:title].lower)) } scope :reorder_by_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } scope :with_api_entity_associations, -> { preload(project: [:project_feature, :route, namespace: :route]) } scope :order_by_dates_and_title, -> { order(due_date: :asc, start_date: :asc, title: :asc) } + validates :group, presence: true, unless: :project + validates :project, presence: true, unless: :group validates :title, presence: true validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } + validate :parent_type_check validate :uniqueness_of_title, if: :title_changed? state_machine :state, initial: :active do @@ -176,10 +196,18 @@ class Milestone < ApplicationRecord # TODO: remove after all code paths use `timebox_id` # https://gitlab.com/gitlab-org/gitlab/-/issues/215688 alias_method :milestoneish_id, :timebox_id - # TODO: remove after all code paths use (group|project)_timebox? - # https://gitlab.com/gitlab-org/gitlab/-/issues/215690 - alias_method :group_milestone?, :group_timebox? - alias_method :project_milestone?, :project_timebox? + + def group_milestone? + group_id.present? + end + + def project_milestone? + project_id.present? + end + + def resource_parent + group || project + end def parent if group_milestone? @@ -193,8 +221,63 @@ class Milestone < ApplicationRecord group_milestone? && parent.subgroup? end + def merge_requests_enabled? + if group_milestone? + # Assume that groups have at least one project with merge requests enabled. + # Otherwise, we would need to load all of the projects from the database. + true + elsif project_milestone? + project&.merge_requests_enabled? + end + end + + ## + # Returns the String necessary to reference a milestone in Markdown. Group + # milestones only support name references, and do not support cross-project + # references. + # + # format - Symbol format to use (default: :iid, optional: :name) + # + # Examples: + # + # Milestone.first.to_reference # => "%1" + # Milestone.first.to_reference(cross_namespace_project) # => "gitlab-org/gitlab-foss%1" + # + def to_reference(from = nil, format: :name, full: false) + format_reference = timebox_format_reference(format) + reference = "#{self.class.reference_prefix}#{format_reference}" + + if project + "#{project.to_reference_base(from, full: full)}#{reference}" + else + reference + end + end + private + def timebox_format_reference(format = :iid) + raise ArgumentError, _('Unknown format') unless [:iid, :name].include?(format) + + if group_milestone? && format == :iid + raise ArgumentError, _('Cannot refer to a group milestone by an internal id!') + end + + if format == :name && !name.include?('"') + %("#{name}") + else + iid + end + end + + # Milestone should be either a project milestone or a group milestone + def parent_type_check + return unless group_id && project_id + + field = project_id_changed? ? :project_id : :group_id + errors.add(field, _("milestone should belong either to a project or a group.") % { timebox_name: timebox_name }) + end + def issues_finder_params { project_id: project_id, group_id: group_id, include_subgroups: group_id.present? }.compact end diff --git a/app/models/ml/candidate_param.rb b/app/models/ml/candidate_param.rb index cbdddcc8a1a..a259e059379 100644 --- a/app/models/ml/candidate_param.rb +++ b/app/models/ml/candidate_param.rb @@ -3,6 +3,7 @@ module Ml class CandidateParam < ApplicationRecord validates :candidate, presence: true + validates :name, uniqueness: { scope: :candidate } validates :name, :value, length: { maximum: 250 }, presence: true belongs_to :candidate, class_name: 'Ml::Candidate' diff --git a/app/models/ml/experiment.rb b/app/models/ml/experiment.rb index e4e9baac4c8..a32099e8a0c 100644 --- a/app/models/ml/experiment.rb +++ b/app/models/ml/experiment.rb @@ -13,10 +13,6 @@ module Ml has_internal_id :iid, scope: :project - def artifact_location - 'not_implemented' - end - class << self def by_project_id_and_iid(project_id, iid) find_by(project_id: project_id, iid: iid) @@ -26,8 +22,8 @@ module Ml find_by(project_id: project_id, name: name) end - def has_record?(project_id, name) - where(project_id: project_id, name: name).exists? + def by_project_id(project_id) + where(project_id: project_id) end end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 0ffd5c446d3..42f362876bb 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -130,6 +130,10 @@ class Namespace < ApplicationRecord to: :namespace_settings, allow_nil: true delegate :show_diff_preview_in_email, :show_diff_preview_in_email?, :show_diff_preview_in_email=, to: :namespace_settings + delegate :maven_package_requests_forwarding, + :pypi_package_requests_forwarding, + :npm_package_requests_forwarding, + to: :package_settings after_save :reload_namespace_details diff --git a/app/models/namespace/aggregation_schedule.rb b/app/models/namespace/aggregation_schedule.rb index ed61c807519..cd7d4fc409a 100644 --- a/app/models/namespace/aggregation_schedule.rb +++ b/app/models/namespace/aggregation_schedule.rb @@ -6,13 +6,20 @@ class Namespace::AggregationSchedule < ApplicationRecord self.primary_key = :namespace_id - DEFAULT_LEASE_TIMEOUT = 1.5.hours.to_i REDIS_SHARED_KEY = 'gitlab:update_namespace_statistics_delay' belongs_to :namespace after_create :schedule_root_storage_statistics + def self.default_lease_timeout + if Feature.enabled?(:remove_namespace_aggregator_delay) + 30.minutes.to_i + else + 1.hour.to_i + end + end + def schedule_root_storage_statistics run_after_commit_or_now do try_obtain_lease do @@ -20,7 +27,7 @@ class Namespace::AggregationSchedule < ApplicationRecord .perform_async(namespace_id) Namespaces::RootStatisticsWorker - .perform_in(DEFAULT_LEASE_TIMEOUT, namespace_id) + .perform_in(self.class.default_lease_timeout, namespace_id) end end end @@ -29,7 +36,7 @@ class Namespace::AggregationSchedule < ApplicationRecord # Used by ExclusiveLeaseGuard def lease_timeout - DEFAULT_LEASE_TIMEOUT + self.class.default_lease_timeout end # Used by ExclusiveLeaseGuard diff --git a/app/models/namespace/detail.rb b/app/models/namespace/detail.rb index dbbf9f4944a..a5643ab9f79 100644 --- a/app/models/namespace/detail.rb +++ b/app/models/namespace/detail.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class Namespace::Detail < ApplicationRecord + include IgnorableColumns + + ignore_column :free_user_cap_over_limt_notified_at, remove_with: '15.7', remove_after: '2022-11-22' + belongs_to :namespace, inverse_of: :namespace_details validates :namespace, presence: true validates :description, length: { maximum: 255 } diff --git a/app/models/namespace/package_setting.rb b/app/models/namespace/package_setting.rb index 881b2f3acb3..22c3e41ff21 100644 --- a/app/models/namespace/package_setting.rb +++ b/app/models/namespace/package_setting.rb @@ -1,9 +1,15 @@ # frozen_string_literal: true class Namespace::PackageSetting < ApplicationRecord + include CascadingNamespaceSettingAttribute + self.primary_key = :namespace_id self.table_name = 'namespace_package_settings' + cascading_attr :maven_package_requests_forwarding + cascading_attr :npm_package_requests_forwarding + cascading_attr :pypi_package_requests_forwarding + PackageSettingNotImplemented = Class.new(StandardError) PACKAGES_WITH_SETTINGS = %w[maven generic].freeze diff --git a/app/models/note.rb b/app/models/note.rb index daac489757b..e444111119b 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -22,6 +22,7 @@ class Note < ApplicationRecord include ThrottledTouch include FromUnion include Sortable + include EachBatch ISSUE_TASK_SYSTEM_NOTE_PATTERN = /\A.*marked\sthe\stask.+as\s(completed|incomplete).*\z/.freeze @@ -693,7 +694,7 @@ class Note < ApplicationRecord # Method necesary while we transition into the new format for task system notes # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/369923 def note - return super unless system? && for_issue? && super.match?(ISSUE_TASK_SYSTEM_NOTE_PATTERN) + return super unless system? && for_issue? && super&.match?(ISSUE_TASK_SYSTEM_NOTE_PATTERN) super.sub!('task', 'checklist item') end @@ -701,11 +702,15 @@ class Note < ApplicationRecord # Method necesary while we transition into the new format for task system notes # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/369923 def note_html - return super unless system? && for_issue? && super.match?(ISSUE_TASK_SYSTEM_NOTE_PATTERN) + return super unless system? && for_issue? && super&.match?(ISSUE_TASK_SYSTEM_NOTE_PATTERN) super.sub!('task', 'checklist item') end + def issuable_ability_name + confidential? ? :read_internal_note : :read_note + end + private def system_note_viewable_by?(user) diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index caa24377791..20d5a5ae1a1 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -97,8 +97,6 @@ class NotificationRecipient end def email_blocked? - return false if Feature.disabled?(:block_emails_with_failures) - recipient_email = user.notification_email_for(@group) Gitlab::ApplicationRateLimiter.peek(:permanent_email_failure, scope: recipient_email) || diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index b4c09d99bb0..317db51f4ef 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -114,13 +114,18 @@ class Packages::Package < ApplicationRecord ) end + scope :with_case_insensitive_version, ->(version) do + where('LOWER(version) = ?', version.downcase) + end + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :with_version, ->(version) { where(version: version) } scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } scope :with_package_type, ->(package_type) { where(package_type: package_type) } scope :without_package_type, ->(package_type) { where.not(package_type: package_type) } scope :displayable, -> { with_status(DISPLAYABLE_STATUSES) } - scope :including_project_route, -> { includes(project: { namespace: :route }) } + scope :including_project_route, -> { includes(project: :route) } + scope :including_project_namespace_route, -> { includes(project: { namespace: :route }) } scope :including_tags, -> { includes(:tags) } scope :including_dependency_links, -> { includes(dependency_links: :dependency) } diff --git a/app/models/packages/rpm/repository_file.rb b/app/models/packages/rpm/repository_file.rb new file mode 100644 index 00000000000..4b5fa59c6ee --- /dev/null +++ b/app/models/packages/rpm/repository_file.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +module Packages + module Rpm + class RepositoryFile < ApplicationRecord + include EachBatch + include UpdateProjectStatistics + include FileStoreMounter + include Packages::Installable + + INSTALLABLE_STATUSES = [:default].freeze + + enum status: { default: 0, pending_destruction: 1, processing: 2, error: 3 } + + belongs_to :project, inverse_of: :repository_files + + validates :project, presence: true + validates :file, presence: true + validates :file_name, presence: true + + mount_file_store_uploader Packages::Rpm::RepositoryFileUploader + + update_project_statistics project_statistics_name: :packages_size + end + end +end diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index e7d455085c0..c1056d4f6cb 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -32,7 +32,9 @@ module Pages { type: 'zip', - path: deployment.file.url_or_file_path(expire_at: 1.day.from_now), + path: deployment.file.url_or_file_path( + expire_at: ::Gitlab::Pages::CacheControl::DEPLOYMENT_EXPIRATION.from_now + ), global_id: global_id, sha256: deployment.file_sha256, file_size: deployment.size, diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 9ed25c56ed6..f0ed1822da6 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -5,6 +5,8 @@ class PersonalAccessToken < ApplicationRecord include TokenAuthenticatable include Sortable include EachBatch + include CreatedAtFilterable + include Gitlab::SQL::Pattern extend ::Gitlab::Utils::Override add_authentication_token_field :token, digest: true @@ -24,7 +26,6 @@ class PersonalAccessToken < ApplicationRecord scope :expiring_and_not_notified, ->(date) { where(["revoked = false AND expire_notification_delivered = false AND expires_at >= CURRENT_DATE AND expires_at <= ?", date]) } scope :expired_today_and_not_notified, -> { where(["revoked = false AND expires_at = CURRENT_DATE AND after_expiry_notification_delivered = false"]) } scope :inactive, -> { where("revoked = true OR expires_at < CURRENT_DATE") } - scope :created_before, -> (date) { where("personal_access_tokens.created_at < :date", date: date) } scope :last_used_before_or_unused, -> (date) { where("personal_access_tokens.created_at < :date AND (last_used_at < :date OR last_used_at IS NULL)", date: date) } scope :with_impersonation, -> { where(impersonation: true) } scope :without_impersonation, -> { where(impersonation: false) } @@ -38,6 +39,8 @@ class PersonalAccessToken < ApplicationRecord scope :order_expires_at_asc_id_desc, -> { reorder(expires_at: :asc, id: :desc) } scope :project_access_token, -> { includes(:user).where(user: { user_type: :project_bot }) } scope :owner_is_human, -> { includes(:user).where(user: { user_type: :human }) } + scope :last_used_before, -> (date) { where("last_used_at <= ?", date) } + scope :last_used_after, -> (date) { where("last_used_at >= ?", date) } validates :scopes, presence: true validate :validate_scopes @@ -90,6 +93,10 @@ class PersonalAccessToken < ApplicationRecord Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix end + def self.search(query) + fuzzy_search(query, [:name]) + end + override :format_token def format_token(token) "#{self.class.token_prefix}#{token}" diff --git a/app/models/preloaders/labels_preloader.rb b/app/models/preloaders/labels_preloader.rb index 722d588d8bc..b6e73c1cd02 100644 --- a/app/models/preloaders/labels_preloader.rb +++ b/app/models/preloaders/labels_preloader.rb @@ -21,8 +21,10 @@ module Preloaders def preload_all preloader = ActiveRecord::Associations::Preloader.new + preloader.preload(labels, parent_container: :route) preloader.preload(labels.select { |l| l.is_a? ProjectLabel }, { project: [:project_feature, namespace: :route] }) preloader.preload(labels.select { |l| l.is_a? GroupLabel }, { group: :route }) + labels.each do |label| label.lazy_subscription(user) label.lazy_subscription(user, project) if project.present? diff --git a/app/models/preloaders/project_root_ancestor_preloader.rb b/app/models/preloaders/project_root_ancestor_preloader.rb index 8d04e71774c..1e935249407 100644 --- a/app/models/preloaders/project_root_ancestor_preloader.rb +++ b/app/models/preloaders/project_root_ancestor_preloader.rb @@ -21,7 +21,8 @@ module Preloaders ActiveRecord::Associations::Preloader.new.preload(@projects, :namespace) @projects.each do |project| - project.namespace.root_ancestor = root_ancestors_by_id[project.id]&.first + root_ancestor = root_ancestors_by_id[project.id]&.first + project.namespace.root_ancestor = root_ancestor if root_ancestor.present? end end diff --git a/app/models/project.rb b/app/models/project.rb index c5fad189f87..7b61010ab01 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -32,7 +32,6 @@ class Project < ApplicationRecord include FeatureGate include OptionallySearch include FromUnion - include IgnorableColumns include Repositories::CanHousekeepRepository include EachBatch include GitlabRoutingHelper @@ -49,8 +48,6 @@ class Project < ApplicationRecord BoardLimitExceeded = Class.new(StandardError) ExportLimitExceeded = Class.new(StandardError) - ignore_columns :build_coverage_regex, remove_after: '2022-10-22', remove_with: '15.5' - STATISTICS_ATTRIBUTE = 'repositories_count' UNKNOWN_IMPORT_URL = 'http://unknown.git' # Hashed Storage versions handle rolling out new storage to project and dependents models: @@ -239,6 +236,9 @@ class Project < ApplicationRecord # Packages has_many :packages, class_name: 'Packages::Package' has_many :package_files, through: :packages, class_name: 'Packages::PackageFile' + # repository_files must be destroyed by ruby code in order to properly remove carrierwave uploads + has_many :repository_files, inverse_of: :project, class_name: 'Packages::Rpm::RepositoryFile', + dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads has_many :debian_distributions, class_name: 'Packages::Debian::ProjectDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :packages_cleanup_policy, class_name: 'Packages::Cleanup::Policy', inverse_of: :project @@ -262,11 +262,11 @@ class Project < ApplicationRecord has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' has_many :issues has_many :incident_management_issuable_escalation_statuses, through: :issues, inverse_of: :project, class_name: 'IncidentManagement::IssuableEscalationStatus' + has_many :incident_management_timeline_event_tags, inverse_of: :project, class_name: 'IncidentManagement::TimelineEventTag' has_many :labels, class_name: 'ProjectLabel' has_many :integrations has_many :events has_many :milestones - has_many :iterations # Projects with a very large number of notes may time out destroying them # through the foreign key. Additionally, the deprecated attachment uploader @@ -353,6 +353,7 @@ class Project < ApplicationRecord has_many :stages, class_name: 'Ci::Stage', inverse_of: :project has_many :ci_refs, class_name: 'Ci::Ref', inverse_of: :project + has_many :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :project has_many :pending_builds, class_name: 'Ci::PendingBuild' has_many :builds, class_name: 'Ci::Build', inverse_of: :project has_many :processables, class_name: 'Ci::Processable', inverse_of: :project @@ -476,7 +477,8 @@ class Project < ApplicationRecord delegate :dashboard_timezone, to: :metrics_setting, allow_nil: true, prefix: true delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :forward_deployment_enabled, :forward_deployment_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true - delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true + delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci_outbound, allow_nil: true + delegate :inbound_job_token_scope_enabled, :inbound_job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :keep_latest_artifact, :keep_latest_artifact=, to: :ci_cd_settings, allow_nil: true delegate :opt_in_jwt, :opt_in_jwt=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :allow_fork_pipelines_to_run_in_parent_project, :allow_fork_pipelines_to_run_in_parent_project=, to: :ci_cd_settings, prefix: :ci, allow_nil: true @@ -492,12 +494,17 @@ class Project < ApplicationRecord delegate :log_jira_dvcs_integration_usage, :jira_dvcs_server_last_sync_at, :jira_dvcs_cloud_last_sync_at, to: :feature_usage + delegate :maven_package_requests_forwarding, + :pypi_package_requests_forwarding, + :npm_package_requests_forwarding, + to: :namespace + # Validations validates :creator, presence: true, on: :create validates :description, length: { maximum: 2000 }, allow_blank: true validates :ci_config_path, format: { without: %r{(\.{2}|\A/)}, - message: _('cannot include leading slash or directory traversal.') }, + message: N_('cannot include leading slash or directory traversal.') }, length: { maximum: 255 }, allow_blank: true validates :name, @@ -693,13 +700,13 @@ class Project < ApplicationRecord enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, - default: 3600, error_message: _('Maximum job timeout has a value which could not be accepted') + default: 3600, error_message: N_('Maximum job timeout has a value which could not be accepted') validates :build_timeout, allow_nil: true, numericality: { greater_than_or_equal_to: 10.minutes, less_than: MAX_BUILD_TIMEOUT, only_integer: true, - message: _('needs to be between 10 minutes and 1 month') } + message: N_('needs to be between 10 minutes and 1 month') } # Used by Projects::CleanupService to hold a map of rewritten object IDs mount_uploader :bfg_object_map, AttachmentUploader @@ -1280,6 +1287,8 @@ class Project < ApplicationRecord valid?(:import_url) || errors.messages[:import_url].nil? end + # TODO: rename to build_or_assign_import_data as it doesn't save record + # https://gitlab.com/gitlab-org/gitlab/-/issues/377319 def create_or_update_import_data(data: nil, credentials: nil) return if data.nil? && credentials.nil? @@ -2720,6 +2729,7 @@ class Project < ApplicationRecord ci_config_path.blank? || ci_config_path == Gitlab::FileDetector::PATTERNS[:gitlab_ci] end + # DO NOT USE. This method will be deprecated soon def uses_external_project_ci_config? !!(ci_config_path =~ %r{@.+/.+}) end @@ -2844,6 +2854,7 @@ class Project < ApplicationRecord repository.gitlab_ci_yml_for(sha, ci_config_path_or_default) end + # DO NOT USE. This method will be deprecated soon def ci_config_external_project Project.find_by_full_path(ci_config_path.split('@', 2).last) end @@ -2886,12 +2897,18 @@ class Project < ApplicationRecord ci_cd_settings.allow_fork_pipelines_to_run_in_parent_project? end - def ci_job_token_scope_enabled? + def ci_outbound_job_token_scope_enabled? return false unless ci_cd_settings ci_cd_settings.job_token_scope_enabled? end + def ci_inbound_job_token_scope_enabled? + return false unless ci_cd_settings + + ci_cd_settings.inbound_job_token_scope_enabled? + end + def restrict_user_defined_variables? return false unless ci_cd_settings @@ -2939,12 +2956,6 @@ class Project < ApplicationRecord end end - def remove_project_authorizations(user_ids, per_batch = 1000) - user_ids.each_slice(per_batch) do |user_ids_batch| - project_authorizations.where(user_id: user_ids_batch).delete_all - end - end - def enforced_runner_token_expiration_interval all_parent_groups = Gitlab::ObjectHierarchy.new(Group.where(id: group)).base_and_ancestors all_group_settings = NamespaceSetting.where(namespace_id: all_parent_groups) @@ -3023,11 +3034,7 @@ class Project < ApplicationRecord end def packages_policy_subject - if Feature.enabled?(:read_package_policy_rule, group) - ::Packages::Policies::Project.new(self) - else - self - end + ::Packages::Policies::Project.new(self) end def destroy_deployment_by_id(deployment_id) @@ -3040,6 +3047,16 @@ class Project < ApplicationRecord pages_domains.count < Gitlab::CurrentSettings.max_pages_custom_domains_per_project end + # overridden in EE + def can_suggest_reviewers? + false + end + + # overridden in EE + def suggested_reviewers_available? + false + end + private # overridden in EE diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index 5c6fdec16ca..8b43e5e5d63 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true class ProjectAuthorization < ApplicationRecord + BATCH_SIZE = 1000 + SLEEP_DELAY = 0.1 + extend SuppressCompositePrimaryKeyWarning include FromUnion @@ -26,11 +29,45 @@ class ProjectAuthorization < ApplicationRecord super(attributes, unique_by: connection.schema_cache.primary_keys(table_name)) end - def self.insert_all_in_batches(attributes, per_batch = 1000) + def self.insert_all_in_batches(attributes, per_batch = BATCH_SIZE) + add_delay = add_delay_between_batches?(entire_size: attributes.size, batch_size: per_batch) + attributes.each_slice(per_batch) do |attributes_batch| insert_all(attributes_batch) + perform_delay if add_delay + end + end + + def self.delete_all_in_batches_for_project(project:, user_ids:, per_batch: BATCH_SIZE) + add_delay = add_delay_between_batches?(entire_size: user_ids.size, batch_size: per_batch) + + user_ids.each_slice(per_batch) do |user_ids_batch| + project.project_authorizations.where(user_id: user_ids_batch).delete_all + perform_delay if add_delay + end + end + + def self.delete_all_in_batches_for_user(user:, project_ids:, per_batch: BATCH_SIZE) + add_delay = add_delay_between_batches?(entire_size: project_ids.size, batch_size: per_batch) + + project_ids.each_slice(per_batch) do |project_ids_batch| + user.project_authorizations.where(project_id: project_ids_batch).delete_all + perform_delay if add_delay end end + + private_class_method def self.add_delay_between_batches?(entire_size:, batch_size:) + # The reason for adding a delay is to give the replica database enough time to + # catch up with the primary when large batches of records are being added/removed. + # Hance, we add a delay only if the GitLab installation has a replica database configured. + entire_size > batch_size && + !::Gitlab::Database::LoadBalancing.primary_only? && + Feature.enabled?(:enable_minor_delay_during_project_authorizations_refresh) + end + + private_class_method def self.perform_delay + sleep(SLEEP_DELAY) + end end ProjectAuthorization.prepend_mod_with('ProjectAuthorization') diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 38740aa20dd..d7a5d0d9d84 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -22,10 +22,6 @@ class ProjectCiCdSetting < ApplicationRecord chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval - def forward_deployment_enabled? - super && ::Feature.enabled?(:forward_deployment_enabled, project) - end - def keep_latest_artifacts_available? # The project level feature can only be enabled when the feature is enabled instance wide Gitlab::CurrentSettings.current_application_settings.keep_latest_artifact? && keep_latest_artifact? diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index 2ba3c74df5b..9f9447c1de2 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -9,7 +9,7 @@ class ProjectGroupLink < ApplicationRecord validates :project_id, presence: true validates :group, presence: true - validates :group_id, uniqueness: { scope: [:project_id], message: _("already shared with this group") } + validates :group_id, uniqueness: { scope: [:project_id], message: N_("already shared with this group") } validates :group_access, presence: true validates :group_access, inclusion: { in: Gitlab::Access.values }, presence: true validate :different_group diff --git a/app/models/project_label.rb b/app/models/project_label.rb index d0b16cc98b4..dc647901b46 100644 --- a/app/models/project_label.rb +++ b/app/models/project_label.rb @@ -4,6 +4,7 @@ class ProjectLabel < Label MAX_NUMBER_OF_PRIORITIES = 1 belongs_to :project + belongs_to :parent_container, foreign_key: :project_id, class_name: 'Project' validates :project, presence: true diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index f5c346eda30..6d40544fad4 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -21,6 +21,7 @@ class ProjectSetting < ApplicationRecord validates :merge_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH } validates :squash_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH } validates :target_platforms, inclusion: { in: ALLOWED_TARGET_PLATFORMS } + validates :suggested_reviewers_enabled, inclusion: { in: [true, false] } validate :validates_mr_default_target_self diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index a91e0291438..f108e43015e 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -27,6 +27,16 @@ class ProjectStatistics < ApplicationRecord snippets_size: %i[storage_size] }.freeze NAMESPACE_RELATABLE_COLUMNS = [:repository_size, :wiki_size, :lfs_objects_size, :uploads_size, :container_registry_size].freeze + STORAGE_SIZE_COMPONENTS = [ + :repository_size, + :wiki_size, + :lfs_objects_size, + :build_artifacts_size, + :packages_size, + :snippets_size, + :pipeline_artifacts_size, + :uploads_size + ].freeze scope :for_project_ids, ->(project_ids) { where(project_id: project_ids) } @@ -39,17 +49,18 @@ class ProjectStatistics < ApplicationRecord def refresh!(only: []) return if Gitlab::Database.read_only? - COLUMNS_TO_REFRESH.each do |column, generator| - if only.empty? || only.include?(column) - public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend - end + columns_to_update = only.empty? ? COLUMNS_TO_REFRESH : COLUMNS_TO_REFRESH & only + columns_to_update.each do |column| + public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend end if only.empty? || only.any? { |column| NAMESPACE_RELATABLE_COLUMNS.include?(column) } schedule_namespace_aggregation_worker end - save! + detect_race_on_record(log_fields: { caller: __method__, attributes: columns_to_update }) do + save! + end end def update_commit_count @@ -97,21 +108,13 @@ class ProjectStatistics < ApplicationRecord end def update_storage_size - storage_size = repository_size + - wiki_size + - lfs_objects_size + - build_artifacts_size + - packages_size + - snippets_size + - pipeline_artifacts_size + - uploads_size - - self.storage_size = storage_size + self.storage_size = storage_size_components.sum { |component| method(component).call } end def refresh_storage_size! - update_storage_size - save! + detect_race_on_record(log_fields: { caller: __method__, attributes: :storage_size }) do + update!(storage_size: storage_size_sum) + end end # Since this incremental update method does not call update_storage_size above through before_save, @@ -129,35 +132,41 @@ class ProjectStatistics < ApplicationRecord if counter_attribute_enabled?(key) project_statistics.delayed_increment_counter(key, amount) else - legacy_increment_statistic(project, key, amount) + project_statistics.legacy_increment_statistic(key, amount) end end end - def self.legacy_increment_statistic(project, key, amount) - where(project_id: project.id).columns_to_increment(key, amount) + def self.incrementable_attribute?(key) + INCREMENTABLE_COLUMNS.key?(key) || counter_attribute_enabled?(key) + end + + def legacy_increment_statistic(key, amount) + increment_columns!(key, amount) Namespaces::ScheduleAggregationWorker.perform_async( # rubocop: disable CodeReuse/Worker project.namespace_id) end - def self.columns_to_increment(key, amount) - updates = ["#{key} = COALESCE(#{key}, 0) + (#{amount})"] - - if (additional = INCREMENTABLE_COLUMNS[key]) - additional.each do |column| - updates << "#{column} = COALESCE(#{column}, 0) + (#{amount})" - end - end + private - update_all(updates.join(', ')) + def storage_size_components + STORAGE_SIZE_COMPONENTS end - def self.incrementable_attribute?(key) - INCREMENTABLE_COLUMNS.key?(key) || counter_attribute_enabled?(key) + def storage_size_sum + storage_size_components.map { |component| "COALESCE (#{component}, 0)" }.join(' + ').freeze end - private + def increment_columns!(key, amount) + increments = { key => amount } + additional = INCREMENTABLE_COLUMNS.fetch(key, []) + additional.each do |column| + increments[column] = amount + end + + update_counters_with_lease(increments) + end def schedule_namespace_aggregation_worker run_after_commit do diff --git a/app/models/projects/build_artifacts_size_refresh.rb b/app/models/projects/build_artifacts_size_refresh.rb index e66e1d5b42f..2ffc7478178 100644 --- a/app/models/projects/build_artifacts_size_refresh.rb +++ b/app/models/projects/build_artifacts_size_refresh.rb @@ -80,9 +80,7 @@ module Projects end def reset_project_statistics! - statistics = project.statistics - statistics.update!(build_artifacts_size: 0) - statistics.clear_counter!(:build_artifacts_size) + project.statistics.reset_counter!(:build_artifacts_size) end def next_batch(limit:) diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index b3a918d8952..dfd5c315f6e 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -95,6 +95,10 @@ class ProtectedBranch < ApplicationRecord def self.downcase_humanized_name name.underscore.humanize.downcase end + + def default_branch? + name == project.default_branch + end end ProtectedBranch.prepend_mod_with('ProtectedBranch') diff --git a/app/models/protected_branch/merge_access_level.rb b/app/models/protected_branch/merge_access_level.rb index de240e40316..df75c557717 100644 --- a/app/models/protected_branch/merge_access_level.rb +++ b/app/models/protected_branch/merge_access_level.rb @@ -2,4 +2,6 @@ class ProtectedBranch::MergeAccessLevel < ApplicationRecord include ProtectedBranchAccess + # default value for the access_level column + GITLAB_DEFAULT_ACCESS_LEVEL = Gitlab::Access::MAINTAINER end diff --git a/app/models/protected_branch/push_access_level.rb b/app/models/protected_branch/push_access_level.rb index 5248834a2f2..6076fab20b7 100644 --- a/app/models/protected_branch/push_access_level.rb +++ b/app/models/protected_branch/push_access_level.rb @@ -2,6 +2,8 @@ class ProtectedBranch::PushAccessLevel < ApplicationRecord include ProtectedBranchAccess + # default value for the access_level column + GITLAB_DEFAULT_ACCESS_LEVEL = Gitlab::Access::MAINTAINER belongs_to :deploy_key diff --git a/app/models/repository.rb b/app/models/repository.rb index ee1bea0e8d2..3413b3e3424 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -48,22 +48,19 @@ class Repository # For example, for entry `:commit_count` there's a method called `commit_count` which # stores its data in the `commit_count` cache key. CACHED_METHODS = %i(size commit_count readme_path contribution_guide - changelog license_blob license_key gitignore + changelog license_blob license_licensee license_gitaly gitignore gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? root_ref merged_branch_names has_visible_content? issue_template_names_hash merge_request_template_names_hash user_defined_metrics_dashboard_paths xcode_project? has_ambiguous_refs?).freeze - # Methods that use cache_method but only memoize the value - MEMOIZED_CACHED_METHODS = %i(license).freeze - # Certain method caches should be refreshed when certain types of files are # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to # the corresponding methods to call for refreshing caches. METHOD_CACHES_FOR_FILE_TYPES = { readme: %i(readme_path), changelog: :changelog, - license: %i(license_blob license_key license), + license: %i(license_blob license_licensee license_gitaly), contributing: :contribution_guide, gitignore: :gitignore, gitlab_ci: :gitlab_ci_yml, @@ -650,25 +647,30 @@ class Repository cache_method :license_blob def license_key - return unless exists? - - raw_repository.license_short_name + license&.key end - cache_method :license_key def license - return unless license_key + if Feature.enabled?(:license_from_gitaly) + license_gitaly + else + license_licensee + end + end - licensee_object = Licensee::License.new(license_key) + def license_licensee + return unless exists? - return if licensee_object.name.blank? + raw_repository.license(false) + end + cache_method :license_licensee - licensee_object - rescue Licensee::InvalidLicense => e - Gitlab::ErrorTracking.track_exception(e) - nil + def license_gitaly + return unless exists? + + raw_repository.license(true) end - memoize_method :license + cache_method :license_gitaly def gitignore file_on_head(:gitignore) @@ -787,8 +789,8 @@ class Repository Commit.order_by(collection: commits, order_by: order_by, sort: sort) end - def branch_names_contains(sha) - raw_repository.branch_names_contains_sha(sha) + def branch_names_contains(sha, limit: 0) + raw_repository.branch_names_contains_sha(sha, limit: limit) end def tag_names_contains(sha, limit: 0) diff --git a/app/models/resource_label_event.rb b/app/models/resource_label_event.rb index 0a59d9cef9b..a1753df9294 100644 --- a/app/models/resource_label_event.rb +++ b/app/models/resource_label_event.rb @@ -115,7 +115,7 @@ class ResourceLabelEvent < ResourceEvent end def discussion_id_key - [self.class.name, created_at, user_id] + [self.class.name, created_at.to_f, user_id] end end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 9b7c37dd23e..9ec685c5580 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -350,7 +350,7 @@ class Snippet < ApplicationRecord end def can_cache_field?(field) - field != :content || MarkupHelper.gitlab_markdown?(file_name) + field != :content || Gitlab::MarkupHelper.gitlab_markdown?(file_name) end def hexdigest diff --git a/app/models/tree.rb b/app/models/tree.rb index 941d0394b94..c6adf5c263c 100644 --- a/app/models/tree.rb +++ b/app/models/tree.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class Tree - include Gitlab::MarkupHelper include Gitlab::Utils::StrongMemoize attr_accessor :repository, :sha, :path, :entries, :cursor @@ -24,11 +23,11 @@ class Tree end previewable_readmes = available_readmes.select do |blob| - previewable?(blob.name) + Gitlab::MarkupHelper.previewable?(blob.name) end plain_readmes = available_readmes.select do |blob| - plain?(blob.name) + Gitlab::MarkupHelper.plain?(blob.name) end # Prioritize previewable over plain readmes diff --git a/app/models/user.rb b/app/models/user.rb index 3f07e1b1ec0..6d198fc755b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -60,7 +60,7 @@ class User < ApplicationRecord default_value_for :admin, false default_value_for(:external) { Gitlab::CurrentSettings.user_default_external } - default_value_for :can_create_group, gitlab_config.default_can_create_group + default_value_for(:can_create_group) { Gitlab::CurrentSettings.can_create_group } default_value_for :can_create_team, false default_value_for :hide_no_ssh_key, false default_value_for :hide_no_password, false @@ -79,6 +79,7 @@ class User < ApplicationRecord otp_secret_encryption_key: Gitlab::Application.secrets.otp_key_base devise :two_factor_backupable, otp_number_of_backup_codes: 10 + devise :two_factor_backupable_pbkdf2 serialize :otp_backup_codes, JSON # rubocop:disable Cop/ActiveRecordSerialize devise :lockable, :recoverable, :rememberable, :trackable, @@ -168,6 +169,10 @@ class User < ApplicationRecord through: :group_members, source: :group alias_attribute :masters_groups, :maintainers_groups + has_many :developer_maintainer_owned_groups, + -> { where(members: { access_level: [Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, + through: :group_members, + source: :group has_many :reporter_developer_maintainer_owned_groups, -> { where(members: { access_level: [Gitlab::Access::REPORTER, Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, through: :group_members, @@ -193,6 +198,10 @@ class User < ApplicationRecord has_many :snippets, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :issues, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :legacy_assigned_merge_requests, class_name: 'MergeRequest', dependent: :nullify, foreign_key: :assignee_id # rubocop:disable Cop/ActiveRecordDependent + has_many :merged_merge_requests, class_name: 'MergeRequest::Metrics', dependent: :nullify, foreign_key: :merged_by_id # rubocop:disable Cop/ActiveRecordDependent + has_many :closed_merge_requests, class_name: 'MergeRequest::Metrics', dependent: :nullify, foreign_key: :latest_closed_by_id # rubocop:disable Cop/ActiveRecordDependent + has_many :updated_merge_requests, class_name: 'MergeRequest', dependent: :nullify, foreign_key: :updated_by_id # rubocop:disable Cop/ActiveRecordDependent has_many :updated_issues, class_name: 'Issue', dependent: :nullify, foreign_key: :updated_by_id # rubocop:disable Cop/ActiveRecordDependent has_many :closed_issues, class_name: 'Issue', dependent: :nullify, foreign_key: :closed_by_id # rubocop:disable Cop/ActiveRecordDependent has_many :merge_requests, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent @@ -205,14 +214,15 @@ class User < ApplicationRecord has_many :spam_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :builds, class_name: 'Ci::Build' has_many :pipelines, class_name: 'Ci::Pipeline' - has_many :todos + has_many :todos, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :authored_todos, class_name: 'Todo', dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notification_settings has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :triggers, class_name: 'Ci::Trigger', foreign_key: :owner_id has_many :issue_assignees, inverse_of: :assignee - has_many :merge_request_assignees, inverse_of: :assignee - has_many :merge_request_reviewers, inverse_of: :reviewer + has_many :merge_request_assignees, inverse_of: :assignee, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :merge_request_reviewers, inverse_of: :reviewer, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, class_name: "MergeRequest", through: :merge_request_assignees, source: :merge_request has_many :created_custom_emoji, class_name: 'CustomEmoji', inverse_of: :creator @@ -223,7 +233,6 @@ class User < ApplicationRecord has_many :callouts, class_name: 'Users::Callout' has_many :group_callouts, class_name: 'Users::GroupCallout' has_many :project_callouts, class_name: 'Users::ProjectCallout' - has_many :namespace_callouts, class_name: 'Users::NamespaceCallout' has_many :term_agreements belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' @@ -235,6 +244,7 @@ class User < ApplicationRecord has_one :user_highest_role has_one :user_canonical_email has_one :credit_card_validation, class_name: '::Users::CreditCardValidation' + has_one :phone_number_validation, class_name: '::Users::PhoneNumberValidation' has_one :atlassian_identity, class_name: 'Atlassian::Identity' has_one :banned_user, class_name: '::Users::BannedUser' @@ -245,6 +255,8 @@ class User < ApplicationRecord has_many :timelogs has_many :resource_label_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + has_many :resource_state_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + has_many :authored_events, class_name: 'Event', dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent # # Validations @@ -274,10 +286,10 @@ class User < ApplicationRecord validate :check_username_format, if: :username_changed? validates :theme_id, allow_nil: true, inclusion: { in: Gitlab::Themes.valid_ids, - message: _("%{placeholder} is not a valid theme") % { placeholder: '%{value}' } } + message: ->(*) { _("%{placeholder} is not a valid theme") % { placeholder: '%{value}' } } } validates :color_scheme_id, allow_nil: true, inclusion: { in: Gitlab::ColorSchemes.valid_ids, - message: _("%{placeholder} is not a valid color scheme") % { placeholder: '%{value}' } } + message: ->(*) { _("%{placeholder} is not a valid color scheme") % { placeholder: '%{value}' } } } validates :website_url, allow_blank: true, url: true, if: :website_url_changed? @@ -289,6 +301,7 @@ class User < ApplicationRecord before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? } before_validation :ensure_namespace_correct before_save :ensure_namespace_correct # in case validation is skipped + before_save :ensure_user_detail_assigned after_validation :set_username_errors after_update :username_changed_hook, if: :saved_change_to_username? after_destroy :post_destroy_hook @@ -338,8 +351,10 @@ class User < ApplicationRecord :setup_for_company, :setup_for_company=, :render_whitespace_in_code, :render_whitespace_in_code=, :markdown_surround_selection, :markdown_surround_selection=, + :markdown_automatic_lists, :markdown_automatic_lists=, :diffs_deletion_color, :diffs_deletion_color=, :diffs_addition_color, :diffs_addition_color=, + :use_legacy_web_ide, :use_legacy_web_ide=, to: :user_preference delegate :path, to: :namespace, allow_nil: true, prefix: true @@ -934,6 +949,7 @@ class User < ApplicationRecord # that the password is the user's password def valid_password?(password) return false unless password_allowed?(password) + return false if password_automatically_set? return super if Feature.enabled?(:pbkdf2_password_encryption) Devise::Encryptor.compare(self.class, encrypted_password, password) @@ -943,6 +959,22 @@ class User < ApplicationRecord false end + def generate_otp_backup_codes! + if Gitlab::FIPS.enabled? + generate_otp_backup_codes_pbkdf2! + else + super + end + end + + def invalidate_otp_backup_code!(code) + if Gitlab::FIPS.enabled? && pbkdf2? + invalidate_otp_backup_code_pdkdf2!(code) + else + super(code) + end + end + # This method should be removed once the :pbkdf2_password_encryption feature flag is removed. def password=(new_password) if Feature.enabled?(:pbkdf2_password_encryption) && Feature.enabled?(:pbkdf2_password_encryption_write, self) @@ -1129,12 +1161,6 @@ class User < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass - def remove_project_authorizations(project_ids, per_batch = 1000) - project_ids.each_slice(per_batch) do |project_ids_batch| - project_authorizations.where(project_id: project_ids_batch).delete_all - end - end - def authorized_projects(min_access_level = nil) # We're overriding an association, so explicitly call super with no # arguments or it would be passed as `force_reload` to the association @@ -1565,6 +1591,11 @@ class User < ApplicationRecord end end + # Temporary, will be removed when user_detail fields are fully migrated + def ensure_user_detail_assigned + user_detail.assign_changed_fields_from_user if UserDetail.user_fields_changed?(self) + end + def set_username_errors namespace_path_errors = self.errors.delete(:"namespace.path") @@ -1647,8 +1678,9 @@ class User < ApplicationRecord begin followee = Users::UserFollowUser.create(follower_id: self.id, followee_id: user.id) self.followees.reset if followee.persisted? + followee rescue ActiveRecord::RecordNotUnique - false + nil end end @@ -1737,7 +1769,7 @@ class User < ApplicationRecord end def authorized_project_mirrors(level) - projects = Ci::ProjectMirror.by_project_id(ci_project_mirrors_for_project_members(level)) + projects = Ci::ProjectMirror.by_project_id(ci_project_ids_for_project_members(level)) namespace_projects = Ci::ProjectMirror.by_namespace_id(ci_namespace_mirrors_for_group_members(level).select(:namespace_id)) @@ -2075,14 +2107,6 @@ class User < ApplicationRecord callout_dismissed?(callout, ignore_dismissal_earlier_than) end - # Deprecated: do not use. See: https://gitlab.com/gitlab-org/gitlab/-/issues/371017 - def dismissed_callout_for_namespace?(feature_name:, namespace:, ignore_dismissal_earlier_than: nil) - source_feature_name = "#{feature_name}_#{namespace.id}" - callout = namespace_callouts_by_feature_name[source_feature_name] - - callout_dismissed?(callout, ignore_dismissal_earlier_than) - end - def dismissed_callout_for_project?(feature_name:, project:, ignore_dismissal_earlier_than: nil) callout = project_callouts.find_by(feature_name: feature_name, project: project) @@ -2115,11 +2139,6 @@ class User < ApplicationRecord .find_or_initialize_by(feature_name: ::Users::GroupCallout.feature_names[feature_name], group_id: group_id) end - def find_or_initialize_namespace_callout(feature_name, namespace_id) - namespace_callouts - .find_or_initialize_by(feature_name: ::Users::NamespaceCallout.feature_names[feature_name], namespace_id: namespace_id) - end - def find_or_initialize_project_callout(feature_name, project_id) project_callouts .find_or_initialize_by(feature_name: ::Users::ProjectCallout.feature_names[feature_name], project_id: project_id) @@ -2198,6 +2217,12 @@ class User < ApplicationRecord private + def pbkdf2? + return false unless otp_backup_codes&.any? + + otp_backup_codes.first.start_with?("$pbkdf2-sha512$") + end + # To enable JiHu repository to modify the default language options def default_preferred_language 'en' @@ -2209,7 +2234,7 @@ class User < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass - def ci_project_mirrors_for_project_members(level) + def ci_project_ids_for_project_members(level) project_members.where('access_level >= ?', level).pluck(:source_id) end @@ -2246,10 +2271,6 @@ class User < ApplicationRecord @group_callouts_by_feature_name ||= group_callouts.index_by(&:source_feature_name) end - def namespace_callouts_by_feature_name - @namespace_callouts_by_feature_name ||= namespace_callouts.index_by(&:source_feature_name) - end - def authorized_groups_without_shared_membership Group.from_union( [ @@ -2298,7 +2319,7 @@ class User < ApplicationRecord self.projects_limit = 0 else # Only revert these back to the default if they weren't specifically changed in this update. - self.can_create_group = gitlab_config.default_can_create_group unless can_create_group_changed? + self.can_create_group = Gitlab::CurrentSettings.can_create_group unless can_create_group_changed? self.projects_limit = Gitlab::CurrentSettings.default_projects_limit unless projects_limit_changed? end end @@ -2363,7 +2384,7 @@ class User < ApplicationRecord end def ci_owned_project_runners_from_project_members - project_ids = ci_project_mirrors_for_project_members(Gitlab::Access::MAINTAINER) + project_ids = ci_project_ids_for_project_members(Gitlab::Access::MAINTAINER) Ci::Runner .joins(:runner_projects) diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index b9b69d12729..2e662faea6a 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -2,9 +2,6 @@ class UserDetail < ApplicationRecord extend ::Gitlab::Utils::Override - include IgnorableColumns - - ignore_columns :other_role, remove_after: '2022-07-22', remove_with: '15.3' REGISTRATION_OBJECTIVE_PAIRS = { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5, joining_team: 6 }.freeze @@ -15,15 +12,55 @@ class UserDetail < ApplicationRecord validates :job_title, length: { maximum: 200 } validates :bio, length: { maximum: 255 }, allow_blank: true + DEFAULT_FIELD_LENGTH = 500 + + validates :linkedin, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :twitter, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :skype, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :location, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :organization, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :website_url, length: { maximum: DEFAULT_FIELD_LENGTH }, url: true, allow_blank: true + + before_validation :sanitize_attrs before_save :prevent_nil_bio enum registration_objective: REGISTRATION_OBJECTIVE_PAIRS, _suffix: true + def self.user_fields_changed?(user) + (%w[linkedin skype twitter website_url location organization] & user.changed).any? + end + + def sanitize_attrs + %i[linkedin skype twitter website_url].each do |attr| + value = self[attr] + self[attr] = Sanitize.clean(value) if value.present? + end + %i[location organization].each do |attr| + value = self[attr] + self[attr] = Sanitize.clean(value).gsub('&', '&') if value.present? + end + end + + def assign_changed_fields_from_user + self.linkedin = trim_field(user.linkedin) if user.linkedin_changed? + self.twitter = trim_field(user.twitter) if user.twitter_changed? + self.skype = trim_field(user.skype) if user.skype_changed? + self.website_url = trim_field(user.website_url) if user.website_url_changed? + self.location = trim_field(user.location) if user.location_changed? + self.organization = trim_field(user.organization) if user.organization_changed? + end + private def prevent_nil_bio self.bio = '' if bio_changed? && bio.nil? end + + def trim_field(value) + return '' unless value + + value.first(DEFAULT_FIELD_LENGTH) + end end UserDetail.prepend_mod_with('UserDetail') diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index 9b4c0a2527a..c6ebd550daf 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -22,6 +22,7 @@ class UserPreference < ApplicationRecord validates :diffs_deletion_color, :diffs_addition_color, format: { with: ColorsHelper::HEX_COLOR_PATTERN }, allow_blank: true + validates :use_legacy_web_ide, allow_nil: false, inclusion: { in: [true, false] } ignore_columns :experience_level, remove_with: '14.10', remove_after: '2021-03-22' @@ -29,7 +30,6 @@ class UserPreference < ApplicationRecord default_value_for :time_display_relative, value: true, allows_nil: false default_value_for :time_format_in_24h, value: false, allows_nil: false default_value_for :render_whitespace_in_code, value: false, allows_nil: false - default_value_for :markdown_surround_selection, value: true, allows_nil: false class << self def notes_filters diff --git a/app/models/users/banned_user.rb b/app/models/users/banned_user.rb index c52b6d4b728..615668e2b55 100644 --- a/app/models/users/banned_user.rb +++ b/app/models/users/banned_user.rb @@ -7,6 +7,6 @@ module Users belongs_to :user validates :user, presence: true - validates :user_id, uniqueness: { message: _("banned user already exists") } + validates :user_id, uniqueness: { message: N_("banned user already exists") } end end diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 03841ee48fa..ae6950d800c 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -61,7 +61,8 @@ module Users namespace_storage_limit_banner_alert_threshold: 57, # EE-only namespace_storage_limit_banner_error_threshold: 58, # EE-only project_quality_summary_feedback: 59, # EE-only - merge_request_settings_moved_callout: 60 + merge_request_settings_moved_callout: 60, + new_top_level_group_alert: 61 } validates :feature_name, diff --git a/app/models/users/namespace_callout.rb b/app/models/users/namespace_callout.rb deleted file mode 100644 index 4e655a96b57..00000000000 --- a/app/models/users/namespace_callout.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Users - class NamespaceCallout < ApplicationRecord - include Users::Calloutable - - self.table_name = 'user_namespace_callouts' - - belongs_to :namespace - - enum feature_name: { - invite_members_banner: 1, - approaching_seat_count_threshold: 2, # EE-only - storage_enforcement_banner_first_enforcement_threshold: 3, # EE-only - storage_enforcement_banner_second_enforcement_threshold: 4, # EE-only - storage_enforcement_banner_third_enforcement_threshold: 5, # EE-only - storage_enforcement_banner_fourth_enforcement_threshold: 6, # EE-only - preview_user_over_limit_free_plan_alert: 7, # EE-only - user_reached_limit_free_plan_alert: 8, # EE-only - web_hook_disabled: 9 - } - - validates :namespace, presence: true - validates :feature_name, - presence: true, - uniqueness: { scope: [:user_id, :namespace_id] }, - inclusion: { in: NamespaceCallout.feature_names.keys } - - def source_feature_name - "#{feature_name}_#{namespace_id}" - end - end -end diff --git a/app/models/users/phone_number_validation.rb b/app/models/users/phone_number_validation.rb new file mode 100644 index 00000000000..f6123c01fd0 --- /dev/null +++ b/app/models/users/phone_number_validation.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Users + class PhoneNumberValidation < ApplicationRecord + self.primary_key = :user_id + self.table_name = 'user_phone_number_validations' + + belongs_to :user, foreign_key: :user_id + belongs_to :banned_user, class_name: '::Users::BannedUser', foreign_key: :user_id + + validates :country, + presence: true, + length: { maximum: 3 } + + validates :international_dial_code, + presence: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: 1, + less_than_or_equal_to: 999 + } + + validates :phone_number, + presence: true, + format: { + with: /\A\d+\Z/, + message: -> (object, data) { _('can contain only digits') } + }, + length: { maximum: 12 } + + validates :telesign_reference_xid, + length: { maximum: 255 } + + def self.related_to_banned_user?(international_dial_code, phone_number) + joins(:banned_user).where( + international_dial_code: international_dial_code, + phone_number: phone_number + ).exists? + end + end +end diff --git a/app/models/users/project_callout.rb b/app/models/users/project_callout.rb index 98dacbe394a..c73b3a4ee71 100644 --- a/app/models/users/project_callout.rb +++ b/app/models/users/project_callout.rb @@ -11,7 +11,11 @@ module Users enum feature_name: { awaiting_members_banner: 1, # EE-only web_hook_disabled: 2, - ultimate_feature_removal_banner: 3 + ultimate_feature_removal_banner: 3, + storage_enforcement_banner_first_enforcement_threshold: 4, # EE-only + storage_enforcement_banner_second_enforcement_threshold: 5, # EE-only + storage_enforcement_banner_third_enforcement_threshold: 6, # EE-only + storage_enforcement_banner_fourth_enforcement_threshold: 7 # EE-only } validates :project, presence: true diff --git a/app/models/users/user_follow_user.rb b/app/models/users/user_follow_user.rb index a94239a746c..5a82a81364a 100644 --- a/app/models/users/user_follow_user.rb +++ b/app/models/users/user_follow_user.rb @@ -1,7 +1,22 @@ # frozen_string_literal: true module Users class UserFollowUser < ApplicationRecord + MAX_FOLLOWEE_LIMIT = 300 + belongs_to :follower, class_name: 'User' belongs_to :followee, class_name: 'User' + + validate :max_follow_limit + + private + + def max_follow_limit + followee_count = self.class.where(follower_id: follower_id).limit(MAX_FOLLOWEE_LIMIT).count + return if followee_count < MAX_FOLLOWEE_LIMIT + + errors.add(:base, format( + _("You can't follow more than %{limit} users. To follow more users, unfollow some others."), + limit: MAX_FOLLOWEE_LIMIT)) + end end end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index fac79a8194a..b718c3a096f 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -9,6 +9,8 @@ class Wiki extend ActiveModel::Naming + DuplicatePageError = Class.new(StandardError) + MARKUPS = { # rubocop:disable Style/MultilineIfModifier markdown: { name: 'Markdown', @@ -109,11 +111,34 @@ class Wiki end def sluggified_title(title) - title = Gitlab::EncodingHelper.encode_utf8_no_detect(title) - title = File.expand_path(title, '/') + title = Gitlab::EncodingHelper.encode_utf8_no_detect(title.to_s.strip) + title = File.absolute_path(title, '/') title = Pathname.new(title).relative_path_from('/').to_s title.tr(' ', '-') end + + def canonicalize_filename(filename) + ::File.basename(filename, ::File.extname(filename)).tr('-', ' ') + end + + def cname(name, char_white_sub = '-', char_other_sub = '-') + name.to_s.gsub(/\s/, char_white_sub).gsub(/[<>+]/, char_other_sub) + end + + def preview_slug(title, format) + ext = format == :markdown ? "md" : format.to_s + name = cname(title) + '.' + ext + canonical_name = canonicalize_filename(name) + + path = + if name.include?('/') + name.sub(%r{/[^/]+$}, '/') + else + '' + end + + path + cname(canonical_name, '-', '-') + end end def initialize(container, user = nil) @@ -145,14 +170,6 @@ class Wiki container.path + '.wiki' end - # Returns the Gitlab::Git::Wiki object. - def wiki - strong_memoize(:wiki) do - create_wiki_repository - Gitlab::Git::Wiki.new(repository.raw) - end - end - def create_wiki_repository repository.create_if_not_exists(default_branch) @@ -173,7 +190,7 @@ class Wiki end def empty? - !repository_exists? || list_pages(limit: 1).empty? + !repository_exists? || list_page_paths.empty? end def exists? @@ -190,15 +207,9 @@ class Wiki # # Returns an Array of GitLab WikiPage instances or an # empty Array if this Wiki has no pages. - def list_pages(limit: 0, sort: nil, direction: DIRECTION_ASC, load_content: false) - wiki.list_pages( - limit: limit, - sort: sort, - direction_desc: direction == DIRECTION_DESC, - load_content: load_content - ).map do |page| - WikiPage.new(self, page) - end + def list_pages(limit: 0, direction: DIRECTION_ASC, load_content: false) + create_wiki_repository unless repository_exists? + list_pages_with_repository_rpcs(limit: limit, direction: direction, load_content: load_content) end def sidebar_entries(limit: Gitlab::WikiPages::MAX_SIDEBAR_PAGES, **options) @@ -217,19 +228,15 @@ class Wiki # # Returns an initialized WikiPage instance or nil def find_page(title, version = nil, load_content: true) - if find_page_with_repository_rpcs? - create_wiki_repository unless repository_exists? - find_page_with_repository_rpcs(title, version, load_content: load_content) - else - find_page_with_legacy_wiki_service(title, version, load_content: load_content) - end + create_wiki_repository unless repository_exists? + find_page_with_repository_rpcs(title, version, load_content: load_content) end def find_sidebar(version = nil) find_page(SIDEBAR, version) end - def find_file(name, version = 'HEAD', load_content: true) + def find_file(name, version = default_branch, load_content: true) data_limit = load_content ? -1 : 0 blobs = repository.blobs_at([[version, name]], blob_size_limit: data_limit) @@ -256,7 +263,7 @@ class Wiki raise_duplicate_page_error! end end - rescue Gitlab::Git::Wiki::DuplicatePageError => e + rescue DuplicatePageError => e @error_message = _("Duplicate page: %{error_message}" % { error_message: e.message }) false @@ -272,6 +279,7 @@ class Wiki extension = page.format != format.to_sym ? default_extension : File.extname(page.path).downcase[1..] capture_git_error(:updated) do + create_wiki_repository unless repository_exists? repository.update_file( user, sluggified_full_path(title, extension), @@ -290,6 +298,7 @@ class Wiki return unless page capture_git_error(:deleted) do + create_wiki_repository unless repository_exists? repository.delete_file(user, page.path, **multi_commit_options(:deleted, message, page.title)) after_wiki_activity @@ -306,8 +315,10 @@ class Wiki [title, title_array.join("/")] end + # TODO: This method is redundant. Should be replaced by create_wiki_repository def ensure_repository - raise CouldNotCreateWikiError unless wiki.repository_exists? + create_wiki_repository + raise CouldNotCreateWikiError unless repository_exists? end def hook_attrs @@ -343,7 +354,7 @@ class Wiki override :default_branch def default_branch - super || Gitlab::Git::Wiki.default_ref(container) + super || Gitlab::DefaultBranch.value(object: container) end def wiki_base_path @@ -423,11 +434,11 @@ class Wiki escaped_title = Regexp.escape(sluggified_title(title)) regex = Regexp.new("^#{escaped_title}\.#{ALLOWED_EXTENSIONS_REGEX}$", 'i') - repository.ls_files('HEAD').any? { |s| s =~ regex } + repository.ls_files(default_branch).any? { |s| s =~ regex } end def raise_duplicate_page_error! - raise Gitlab::Git::Wiki::DuplicatePageError, _('A page with that title already exists') + raise ::Wiki::DuplicatePageError, _('A page with that title already exists') end def sluggified_full_path(title, extension) @@ -439,27 +450,12 @@ class Wiki end def canonicalize_filename(filename) - Gitlab::Git::Wiki::GollumSlug.canonicalize_filename(filename) - end - - def find_page_with_legacy_wiki_service(title, version, load_content: false) - page_title, page_dir = page_title_and_dir(title) - - if page = wiki.page(title: page_title, version: version, dir: page_dir, load_content: load_content) - WikiPage.new(self, page) - end + self.class.canonicalize_filename(filename) end def find_matched_file(title, version) escaped_path = RE2::Regexp.escape(sluggified_title(title)) - # We could not use ALLOWED_EXTENSIONS_REGEX constant or similar regexp with - # Regexp.union. The result combination complicated modifiers: - # /(?i-mx:md|mkdn?|mdown|markdown)|(?i-mx:rdoc).../ - # Regexp used by Gitaly is Go's Regexp package. It does not support those - # features. So, we have to compose another more-friendly regexp to pass to - # Gitaly side. - extension_regexp = Wiki::MARKUPS.map { |_, format| format[:extension_regex].source }.join("|") - path_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)^#{escaped_path}\\.(#{extension_regexp})$") + path_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)^#{escaped_path}\\.(#{file_extension_regexp})$") matched_files = repository.search_files_by_regexp(path_regexp, version) return if matched_files.blank? @@ -473,11 +469,11 @@ class Wiki end def check_page_historical(path, commit) - repository.last_commit_for_path('HEAD', path).id != commit.id + repository.last_commit_for_path(default_branch, path)&.id != commit&.id end def find_page_with_repository_rpcs(title, version, load_content: true) - version = version.presence || 'HEAD' + version = version.presence || default_branch path = find_matched_file(title, version) return if path.blank? @@ -487,27 +483,81 @@ class Wiki format = find_page_format(path) page = Gitlab::Git::WikiPage.new( - url_path: sluggified_title(path.sub(/\.[^.]+\z/, "")), + url_path: sluggified_title(strip_extension(path)), title: canonicalize_filename(path), format: format, path: sluggified_title(path), raw_data: blob.data, name: canonicalize_filename(path), - historical: version == 'HEAD' ? false : check_page_historical(path, commit), + historical: version == default_branch ? false : check_page_historical(path, commit), version: Gitlab::Git::WikiPageVersion.new(commit, format) ) WikiPage.new(self, page) end - def find_page_with_repository_rpcs? - group = - if container.is_a?(::Group) - container - else - container.group - end + def file_extension_regexp + # We could not use ALLOWED_EXTENSIONS_REGEX constant or similar regexp with + # Regexp.union. The result combination complicated modifiers: + # /(?i-mx:md|mkdn?|mdown|markdown)|(?i-mx:rdoc).../ + # Regexp used by Gitaly is Go's Regexp package. It does not support those + # features. So, we have to compose another more-friendly regexp to pass to + # Gitaly side. + Wiki::MARKUPS.map { |_, format| format[:extension_regex].source }.join("|") + end + + def strip_extension(path) + path.sub(/\.[^.]+\z/, "") + end - Feature.enabled?(:wiki_find_page_with_normal_repository_rpcs, group, type: :development) + def list_page_paths + return [] if repository.empty? + + path_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)\\.(#{file_extension_regexp})$") + repository.search_files_by_regexp(path_regexp, default_branch) + end + + def list_pages_with_repository_rpcs(limit:, direction:, load_content:) + paths = list_page_paths + return [] if paths.empty? + + pages = paths.map do |path| + page = Gitlab::Git::WikiPage.new( + url_path: sluggified_title(strip_extension(path)), + title: canonicalize_filename(path), + format: find_page_format(path), + path: sluggified_title(path), + raw_data: '', + name: canonicalize_filename(path), + historical: false + ) + WikiPage.new(self, page) + end + sort_pages!(pages, direction) + pages = pages.take(limit) if limit > 0 + fetch_pages_content!(pages) if load_content + + pages + end + + # After migrating to normal repository RPCs, it's very expensive to sort the + # pages by created_at. We have to either ListLastCommitsForTree RPC call or + # N+1 LastCommitForPath. Either are efficient for a large repository. + # Therefore, we decide to sort the title only. + def sort_pages!(pages, direction) + # Sort by path to ensure the files inside a sub-folder are grouped and sorted together + pages.sort_by!(&:path) + pages.reverse! if direction == DIRECTION_DESC + end + + def fetch_pages_content!(pages) + blobs = + repository + .blobs_at(pages.map { |page| [default_branch, page.path] } ) + .to_h { |blob| [blob.path, blob.data] } + + pages.each do |page| + page.raw_content = blobs[page.path] + end end end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 63c60f5a89e..24b0b94eeb7 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -73,7 +73,7 @@ class WikiPage # The escaped URL path of this page. def slug - attributes[:slug].presence || wiki.wiki.preview_slug(title, format) + attributes[:slug].presence || ::Wiki.preview_slug(title, format) end alias_method :id, :slug # required to use build_stubbed @@ -99,6 +99,13 @@ class WikiPage attributes[:content] ||= page&.text_data end + def raw_content=(content) + return if page.nil? + + page.raw_data = content + attributes[:content] = page.text_data + end + # The hierarchy of the directory this page is contained in. def directory wiki.page_title_and_dir(slug)&.last.to_s @@ -118,7 +125,7 @@ class WikiPage def version return unless persisted? - @version ||= @page.version + @version ||= @page.version || last_version end def path @@ -138,7 +145,7 @@ class WikiPage default_per_page = Kaminari.config.default_per_page offset = [options[:page].to_i - 1, 0].max * options.fetch(:per_page, default_per_page) - wiki.repository.commits('HEAD', + wiki.repository.commits(wiki.default_branch, path: page.path, limit: options.fetch(:limit, default_per_page), offset: offset) @@ -147,11 +154,11 @@ class WikiPage def count_versions return [] unless persisted? - wiki.wiki.count_page_versions(page.path) + wiki.repository.count_commits(ref: wiki.default_branch, path: page.path) end def last_version - @last_version ||= versions(limit: 1).first + @last_version ||= wiki.repository.last_commit_for_path(wiki.default_branch, page.path) if page end def last_commit_sha |