diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-11-18 13:16:36 +0000 |
| commit | 311b0269b4eb9839fa63f80c8d7a58f32b8138a0 (patch) | |
| tree | 07e7870bca8aed6d61fdcc810731c50d2c40af47 /app/models | |
| parent | 27909cef6c4170ed9205afa7426b8d3de47cbb0c (diff) | |
| download | gitlab-ce-311b0269b4eb9839fa63f80c8d7a58f32b8138a0.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-5-stable-eev14.5.0-rc42
Diffstat (limited to 'app/models')
142 files changed, 1523 insertions, 592 deletions
diff --git a/app/models/analytics/cycle_analytics/issue_stage_event.rb b/app/models/analytics/cycle_analytics/issue_stage_event.rb index 3e6ed86d534..837eb35c839 100644 --- a/app/models/analytics/cycle_analytics/issue_stage_event.rb +++ b/app/models/analytics/cycle_analytics/issue_stage_event.rb @@ -8,9 +8,22 @@ module Analytics validates(*%i[stage_event_hash_id issue_id group_id project_id start_event_timestamp], presence: true) + alias_attribute :state, :state_id + enum state: Issue.available_states, _suffix: true + + scope :assigned_to, ->(user) do + assignees_class = IssueAssignee + condition = assignees_class.where(user_id: user).where(arel_table[:issue_id].eq(assignees_class.arel_table[:issue_id])) + where(condition.arel.exists) + end + def self.issuable_id_column :issue_id end + + def self.issuable_model + ::Issue + end end end end diff --git a/app/models/analytics/cycle_analytics/merge_request_stage_event.rb b/app/models/analytics/cycle_analytics/merge_request_stage_event.rb index d0ec3c4e8b9..0dfa322b2c3 100644 --- a/app/models/analytics/cycle_analytics/merge_request_stage_event.rb +++ b/app/models/analytics/cycle_analytics/merge_request_stage_event.rb @@ -8,9 +8,22 @@ module Analytics validates(*%i[stage_event_hash_id merge_request_id group_id project_id start_event_timestamp], presence: true) + alias_attribute :state, :state_id + enum state: MergeRequest.available_states, _suffix: true + + scope :assigned_to, ->(user) do + assignees_class = MergeRequestAssignee + condition = assignees_class.where(user_id: user).where(arel_table[:merge_request_id].eq(assignees_class.arel_table[:merge_request_id])) + where(condition.arel.exists) + end + def self.issuable_id_column :merge_request_id end + + def self.issuable_model + ::MergeRequest + end end end end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index d2757d8c17d..bcd8bdd6638 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true class ApplicationRecord < ActiveRecord::Base - self.gitlab_schema = :gitlab_main + include DatabaseReflection + include Transactions + include LegacyBulkInsert + self.abstract_class = true alias_method :reset, :reload @@ -92,8 +95,7 @@ class ApplicationRecord < ActiveRecord::Base end def self.declarative_enum(enum_mod) - values = enum_mod.definition.transform_values { |v| v[:value] } - enum(enum_mod.key => values) + enum(enum_mod.key => enum_mod.values) end def self.cached_column_list diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 5a8cbd8d71c..af5796d682f 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -536,6 +536,18 @@ class ApplicationSetting < ApplicationRecord validates :sidekiq_job_limiter_limit_bytes, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :sentry_enabled, + inclusion: { in: [true, false], message: _('must be a boolean value') } + validates :sentry_dsn, + addressable_url: true, presence: true, length: { maximum: 255 }, + if: :sentry_enabled? + validates :sentry_clientside_dsn, + addressable_url: true, allow_blank: true, length: { maximum: 255 }, + if: :sentry_enabled? + validates :sentry_environment, + presence: true, length: { maximum: 255 }, + if: :sentry_enabled? + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 7bdea36bb8a..54ec8b2c3e4 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -146,6 +146,9 @@ module ApplicationSettingImplementation session_expire_delay: Settings.gitlab['session_expire_delay'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], shared_runners_text: nil, + sidekiq_job_limiter_mode: Gitlab::SidekiqMiddleware::SizeLimiter::Validator::COMPRESS_MODE, + sidekiq_job_limiter_compression_threshold_bytes: Gitlab::SidekiqMiddleware::SizeLimiter::Validator::DEFAULT_COMPRESSION_THRESHOLD_BYTES, + sidekiq_job_limiter_limit_bytes: Gitlab::SidekiqMiddleware::SizeLimiter::Validator::DEFAULT_SIZE_LIMIT, sign_in_text: nil, signup_enabled: Settings.gitlab['signup_enabled'], snippet_size_limit: 50.megabytes, @@ -241,11 +244,11 @@ module ApplicationSettingImplementation end def home_page_url_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :home_page_url) + ApplicationSetting.database.cached_column_exists?(:home_page_url) end def help_page_support_url_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :help_page_support_url) + ApplicationSetting.database.cached_column_exists?(:help_page_support_url) end def disabled_oauth_sign_in_sources=(sources) diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index a1c6793607f..1a8bd05c42c 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -30,6 +30,8 @@ class AuditEvent < ApplicationRecord scope :by_entity_type, -> (entity_type) { where(entity_type: entity_type) } scope :by_entity_id, -> (entity_id) { where(entity_id: entity_id) } scope :by_author_id, -> (author_id) { where(author_id: author_id) } + scope :by_entity_username, -> (username) { where(entity_id: find_user_id(username)) } + scope :by_author_username, -> (username) { where(author_id: find_user_id(username)) } after_initialize :initialize_details @@ -106,6 +108,10 @@ class AuditEvent < ApplicationRecord self[name] = self.details[name] = original end end + + def self.find_user_id(username) + User.find_by_username(username)&.id + end end AuditEvent.prepend_mod_with('AuditEvent') diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index c8f6b9aaedb..b665f3d5d8c 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -14,7 +14,7 @@ class AwardEmoji < ApplicationRecord validates :user, presence: true validates :awardable, presence: true, unless: :importing? - validates :name, presence: true, inclusion: { in: Gitlab::Emoji.emojis_names } + validates :name, presence: true, 'gitlab/emoji_name': true validates :name, uniqueness: { scope: [:user, :awardable_type, :awardable_id] }, unless: -> { ghost_user? || importing? } participant :user diff --git a/app/models/blob_viewer/package_json.rb b/app/models/blob_viewer/package_json.rb index 7cae60a74d6..1d10cc82a85 100644 --- a/app/models/blob_viewer/package_json.rb +++ b/app/models/blob_viewer/package_json.rb @@ -7,11 +7,15 @@ module BlobViewer self.file_types = %i(package_json) def manager_name - 'npm' + yarn? ? 'yarn' : 'npm' + end + + def yarn? + json_data['engines'].present? && json_data['engines']['yarn'].present? end def manager_url - 'https://www.npmjs.com/' + yarn? ? 'https://yarnpkg.com/' : 'https://www.npmjs.com/' end def package_name @@ -38,7 +42,11 @@ module BlobViewer end def npm_url - "https://www.npmjs.com/package/#{package_name}" + if yarn? + "https://yarnpkg.com/package/#{package_name}" + else + "https://www.npmjs.com/package/#{package_name}" + end end end end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index ecac4ab95f4..2368be6196c 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -112,6 +112,10 @@ class BulkImports::Entity < ApplicationRecord @export_relations_url_path ||= EXPORT_RELATIONS_URL % { resource: pluralized_name, full_path: encoded_source_full_path } end + def relation_download_url_path(relation) + "#{export_relations_url_path}/download?relation=#{relation}" + end + private def validate_parent_is_a_group diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb index ff3f2663b73..da7312df18b 100644 --- a/app/models/chat_name.rb +++ b/app/models/chat_name.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ChatName < ApplicationRecord + include LooseForeignKey + LAST_USED_AT_INTERVAL = 1.hour belongs_to :integration, foreign_key: :service_id @@ -14,6 +16,8 @@ class ChatName < ApplicationRecord validates :user_id, uniqueness: { scope: [:service_id] } validates :chat_id, uniqueness: { scope: [:service_id, :team_id] } + loose_foreign_key :ci_pipeline_chat_data, :chat_name_id, on_delete: :async_delete + # Updates the "last_used_timestamp" but only if it wasn't already updated # recently. # diff --git a/app/models/ci/application_record.rb b/app/models/ci/application_record.rb index 913e7a62c66..ea7b1104e36 100644 --- a/app/models/ci/application_record.rb +++ b/app/models/ci/application_record.rb @@ -2,9 +2,12 @@ module Ci class ApplicationRecord < ::ApplicationRecord - self.gitlab_schema = :gitlab_ci self.abstract_class = true + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + def self.table_name_prefix 'ci_' end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 990ef71a457..3fdc44bccf3 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -10,7 +10,6 @@ module Ci include Presentable include Importable include Ci::HasRef - include IgnorableColumns BuildArchivedError = Class.new(StandardError) @@ -70,9 +69,6 @@ module Ci delegate :gitlab_deploy_token, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true - ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' - ignore_columns :stage_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' - ## # Since Gitlab 11.5, deployments records started being created right after # `ci_builds` creation. We can look up a relevant `environment` through @@ -175,6 +171,7 @@ module Ci scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) } scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) } scope :finished_before, -> (date) { finished.where('finished_at < ?', date) } + scope :license_management_jobs, -> { where(name: %i(license_management license_scanning)) } # handle license rename https://gitlab.com/gitlab-org/gitlab/issues/8911 scope :with_secure_reports_from_config_options, -> (job_types) do joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) @@ -313,12 +310,6 @@ module Ci end after_transition pending: :running do |build| - unless build.update_deployment_after_transaction_commit? - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - build.deployment&.run - end - end - build.run_after_commit do build.pipeline.persistent_ref.create @@ -339,12 +330,6 @@ module Ci end after_transition any => [:success] do |build| - unless build.update_deployment_after_transaction_commit? - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - build.deployment&.succeed - end - end - build.run_after_commit do BuildSuccessWorker.perform_async(id) PagesWorker.perform_async(:deploy, id) if build.pages_generator? @@ -353,23 +338,6 @@ module Ci after_transition any => [:failed] do |build| next unless build.project - next unless build.deployment - - unless build.update_deployment_after_transaction_commit? - begin - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - build.deployment.drop! - end - rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id) - end - end - - true - end - - after_transition any => [:failed] do |build| - next unless build.project if build.auto_retry_allowed? begin @@ -380,25 +348,12 @@ module Ci end end - after_transition any => [:skipped, :canceled] do |build, transition| - unless build.update_deployment_after_transaction_commit? - Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do - if transition.to_name == :skipped - build.deployment&.skip - else - build.deployment&.cancel - end - end - end - end - # Synchronize Deployment Status # Please note that the data integirty is not assured because we can't use # a database transaction due to DB decomposition. after_transition do |build, transition| next if transition.loopback? next unless build.project - next unless build.update_deployment_after_transaction_commit? build.run_after_commit do build.deployment&.sync_status_with(build) @@ -585,7 +540,6 @@ module Ci .concat(persisted_variables) .concat(dependency_proxy_variables) .concat(job_jwt_variables) - .concat(kubernetes_variables) .concat(scoped_variables) .concat(job_variables) .concat(persisted_environment_variables) @@ -1120,12 +1074,6 @@ module Ci runner&.instance_type? end - def update_deployment_after_transaction_commit? - strong_memoize(:update_deployment_after_transaction_commit) do - Feature.enabled?(:update_deployment_after_transaction_commit, project, default_enabled: :yaml) - end - end - protected def run_status_commit_hooks! @@ -1213,10 +1161,6 @@ module Ci end end - def kubernetes_variables - [] # Overridden in EE - end - def conditionally_allow_failure!(exit_code) return unless exit_code diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 0d6d6f7a6a5..ca68989002c 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -23,6 +23,7 @@ module Ci serialize :config_options, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize serialize :config_variables, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize + serialize :runtime_runner_features, Serializers::SymbolizedJson # rubocop:disable Cop/ActiveRecordSerialize chronic_duration_attr_reader :timeout_human_readable, :timeout @@ -37,8 +38,7 @@ module Ci job_timeout_source: 4 } - ignore_column :build_id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' - ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' + ignore_columns :runner_features, remove_with: '14.7', remove_after: '2021-11-22' def update_timeout_state timeout = timeout_with_highest_precedence @@ -48,6 +48,14 @@ module Ci update(timeout: timeout.value, timeout_source: timeout.source) end + def set_cancel_gracefully + runtime_runner_features.merge!( { cancel_gracefully: true } ) + end + + def cancel_gracefully? + runtime_runner_features[:cancel_gracefully] == true + end + private def set_build_project diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index bf1470ca20f..d4cbbfac4ab 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -3,7 +3,6 @@ module Ci class BuildNeed < Ci::ApplicationRecord include BulkInsertSafe - include IgnorableColumns belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs @@ -13,12 +12,5 @@ module Ci scope :scoped_build, -> { where('ci_builds.id=ci_build_needs.build_id') } scope :artifacts, -> { where(artifacts: true) } - - # TODO: Remove once build_id_convert_to_bigint is not an "ignored" column anymore (see .ignore_columns above) - # There is a database-side trigger to populate this column. This is unexpected in the context - # of cloning an instance, e.g. when retrying the job. Hence we exclude the ignored column explicitly here. - def attributes - super.except('build_id_convert_to_bigint') - end end end diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb index e12c0f82c99..c6dbb5d0a43 100644 --- a/app/models/ci/build_runner_session.rb +++ b/app/models/ci/build_runner_session.rb @@ -4,8 +4,6 @@ module Ci # The purpose of this class is to store Build related runner session. # Data will be removed after transitioning from running to any state. class BuildRunnerSession < Ci::ApplicationRecord - include IgnorableColumns - TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com' DEFAULT_SERVICE_NAME = 'build' DEFAULT_PORT_NAME = 'default_port' diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 6edb5ef4579..221a2284106 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -7,7 +7,6 @@ module Ci include ::Checksummable include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::OptimisticLocking - include IgnorableColumns belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id diff --git a/app/models/ci/ci_database_record.rb b/app/models/ci/ci_database_record.rb deleted file mode 100644 index e2b832a28e7..00000000000 --- a/app/models/ci/ci_database_record.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Ci - # TODO: https://gitlab.com/groups/gitlab-org/-/epics/6168 - # - # Do not use this yet outside of `ci_instance_variables`. - # This class is part of a migration to move all CI classes to a new separate database. - # Initially we are only going to be moving the `Ci::InstanceVariable` model and it will be duplicated in the main and CI tables - # Do not extend this class in any other models. - class CiDatabaseRecord < Ci::ApplicationRecord - self.abstract_class = true - - if Gitlab::Database.has_config?(:ci) - connects_to database: { writing: :ci, reading: :ci } - end - end -end diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb index f4aa935b983..da9d4dea537 100644 --- a/app/models/ci/instance_variable.rb +++ b/app/models/ci/instance_variable.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Ci - class InstanceVariable < Ci::CiDatabaseRecord + class InstanceVariable < Ci::ApplicationRecord extend Gitlab::ProcessMemoryCache::Helper include Ci::NewHasVariable include Ci::Maskable diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index ad3e867f9d5..ec1137920ef 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -10,9 +10,7 @@ module Ci include Artifactable include FileStoreMounter include EachBatch - include IgnorableColumns - - ignore_columns %i[id_convert_to_bigint job_id_convert_to_bigint], remove_with: '14.5', remove_after: '2021-11-22' + include Gitlab::Utils::StrongMemoize TEST_REPORT_FILE_TYPES = %w[junit].freeze COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze @@ -124,6 +122,9 @@ module Ci mount_file_store_uploader JobArtifactUploader + skip_callback :save, :after, :store_file!, if: :store_after_commit? + after_commit :store_file_after_commit!, on: [:create, :update], if: :store_after_commit? + validates :file_format, presence: true, unless: :trace?, on: :create validate :validate_file_format!, unless: :trace?, on: :create before_save :set_size, if: :file_changed? @@ -338,8 +339,23 @@ module Ci } end + def store_after_commit? + strong_memoize(:store_after_commit) do + trace? && + JobArtifactUploader.direct_upload_enabled? && + Feature.enabled?(:ci_store_trace_outside_transaction, project, default_enabled: :yaml) + end + end + private + def store_file_after_commit! + return unless previous_changes.key?(:file) + + store_file! + update_file_store + end + def set_size self.size = file.size end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 0041ec5135c..a29aa756e38 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -71,7 +71,7 @@ module Ci has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' has_many :deployments, through: :builds - has_many :environments, -> { distinct }, through: :deployments + has_many :environments, -> { distinct.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338658') }, through: :deployments has_many :latest_builds, -> { latest.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :downloadable_artifacts, -> do not_expired.or(where_exists(::Ci::Pipeline.artifacts_locked.where('ci_pipelines.id = ci_builds.commit_id'))).downloadable.with_job @@ -652,8 +652,15 @@ module Ci end def batch_lookup_report_artifact_for_file_type(file_type) + batch_lookup_report_artifact_for_file_types([file_type]) + end + + def batch_lookup_report_artifact_for_file_types(file_types) + file_types_to_search = [] + file_types.each { |file_type| file_types_to_search.append(*::Ci::JobArtifact.associated_file_types_for(file_type.to_s)) } + latest_report_artifacts - .values_at(*::Ci::JobArtifact.associated_file_types_for(file_type.to_s)) + .values_at(*file_types_to_search.uniq) .flatten .compact .last @@ -684,7 +691,9 @@ module Ci end def freeze_period? - Ci::FreezePeriodStatus.new(project: project).execute + strong_memoize(:freeze_period) do + Ci::FreezePeriodStatus.new(project: project).execute + end end def has_warnings? @@ -780,6 +789,10 @@ module Ci strong_memoize(:legacy_trigger) { trigger_requests.first } end + def variables_builder + @variables_builder ||= ::Gitlab::Ci::Variables::Builder.new(self) + end + def persisted_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless persisted? @@ -796,20 +809,7 @@ module Ci variables.append(key: 'CI_PIPELINE_CREATED_AT', value: created_at&.iso8601) variables.concat(predefined_commit_variables) - - if merge_request? - variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: merge_request_event_type.to_s) - variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: source_sha.to_s) - variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: target_sha.to_s) - - diff = self.merge_request_diff - if diff.present? - variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: diff.id.to_s) - variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: diff.base_commit_sha) - end - - variables.concat(merge_request.predefined_variables) - end + variables.concat(predefined_merge_request_variables) if open_merge_requests_refs.any? variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: open_merge_requests_refs.join(',')) @@ -825,27 +825,49 @@ module Ci end def predefined_commit_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_COMMIT_SHA', value: sha) - variables.append(key: 'CI_COMMIT_SHORT_SHA', value: short_sha) - variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: before_sha) - variables.append(key: 'CI_COMMIT_REF_NAME', value: source_ref) - variables.append(key: 'CI_COMMIT_REF_SLUG', value: source_ref_slug) - variables.append(key: 'CI_COMMIT_BRANCH', value: ref) if branch? - variables.append(key: 'CI_COMMIT_TAG', value: ref) if tag? - variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) - variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) - variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) - variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s) - variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s) - variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s) - - # legacy variables - variables.append(key: 'CI_BUILD_REF', value: sha) - variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) - variables.append(key: 'CI_BUILD_REF_NAME', value: source_ref) - variables.append(key: 'CI_BUILD_REF_SLUG', value: source_ref_slug) - variables.append(key: 'CI_BUILD_TAG', value: ref) if tag? + strong_memoize(:predefined_commit_variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_COMMIT_SHA', value: sha) + variables.append(key: 'CI_COMMIT_SHORT_SHA', value: short_sha) + variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: before_sha) + variables.append(key: 'CI_COMMIT_REF_NAME', value: source_ref) + variables.append(key: 'CI_COMMIT_REF_SLUG', value: source_ref_slug) + variables.append(key: 'CI_COMMIT_BRANCH', value: ref) if branch? + variables.append(key: 'CI_COMMIT_TAG', value: ref) if tag? + variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) + variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) + variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) + variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s) + variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s) + variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s) + + # legacy variables + variables.append(key: 'CI_BUILD_REF', value: sha) + variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) + variables.append(key: 'CI_BUILD_REF_NAME', value: source_ref) + variables.append(key: 'CI_BUILD_REF_SLUG', value: source_ref_slug) + variables.append(key: 'CI_BUILD_TAG', value: ref) if tag? + end + end + end + + def predefined_merge_request_variables + strong_memoize(:predefined_merge_request_variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless merge_request? + + variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: merge_request_event_type.to_s) + variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: source_sha.to_s) + variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: target_sha.to_s) + + diff = self.merge_request_diff + if diff.present? + variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: diff.id.to_s) + variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: diff.base_commit_sha) + end + + variables.concat(merge_request.predefined_variables) + end end end @@ -1254,6 +1276,18 @@ module Ci self.builds.latest.build_matchers(project) end + def predefined_vars_in_builder_enabled? + strong_memoize(:predefined_vars_in_builder_enabled) do + Feature.enabled?(:ci_predefined_vars_in_builder, project, default_enabled: :yaml) + end + end + + def authorized_cluster_agents + strong_memoize(:authorized_cluster_agents) do + ::Clusters::AgentAuthorizationsFinder.new(project).execute.map(&:agent) + end + end + private def add_message(severity, content) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 2f718ad7582..8a3025e5608 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -12,6 +12,7 @@ module Ci include Gitlab::Utils::StrongMemoize include TaggableQueries include Presentable + include LooseForeignKey add_authentication_token_field :token, encrypted: :optional @@ -82,7 +83,9 @@ module Ci groups = Gitlab::ObjectHierarchy.new(groups).base_and_ancestors end - joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups }) + joins(:runner_namespaces) + .where(ci_runner_namespaces: { namespace_id: groups }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } scope :belonging_to_group_or_project, -> (group_id, project_id) { @@ -94,13 +97,16 @@ module Ci union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql from("(#{union_sql}) #{table_name}") + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } scope :belonging_to_parent_group_of_project, -> (project_id) { project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) hierarchy_groups = Gitlab::ObjectHierarchy.new(project_groups).base_and_ancestors - joins(:groups).where(namespaces: { id: hierarchy_groups }) + joins(:groups) + .where(namespaces: { id: hierarchy_groups }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } scope :owned_or_instance_wide, -> (project_id) do @@ -111,7 +117,7 @@ module Ci instance_type ], remove_duplicates: false - ) + ).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') end scope :assignable_for, ->(project) do @@ -162,6 +168,8 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } + loose_foreign_key :clusters_applications_runners, :runner_id, on_delete: :async_nullify + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. @@ -266,6 +274,14 @@ module Ci end def status + return :not_connected unless contacted_at + + online? ? :online : :offline + end + + # DEPRECATED + # TODO Remove in %15.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + def deprecated_rest_status if contacted_at.nil? :not_connected elsif active? @@ -436,10 +452,8 @@ module Ci end def no_groups - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do - if groups.any? - errors.add(:runner, 'cannot have groups assigned') - end + if runner_namespaces.any? + errors.add(:runner, 'cannot have groups assigned') end end @@ -450,10 +464,8 @@ module Ci end def exactly_one_group - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do - unless groups.one? - errors.add(:runner, 'needs to be assigned to exactly one group') - end + unless runner_namespaces.one? + errors.add(:runner, 'needs to be assigned to exactly one group') end end diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index 95842d944f9..f78caf710a6 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -4,9 +4,6 @@ module Ci module Sources class Pipeline < Ci::ApplicationRecord include Ci::NamespacedModelName - include IgnorableColumns - - ignore_columns 'source_job_id_convert_to_bigint', remove_with: '14.5', remove_after: '2021-11-22' self.table_name = "ci_sources_pipelines" diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 131e18adf62..e2b15497638 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -6,7 +6,6 @@ module Ci include Ci::HasStatus include Gitlab::OptimisticLocking include Presentable - include IgnorableColumns enum status: Ci::HasStatus::STATUSES_ENUM diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 595315f14ab..5bf5ae51ec8 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -3,6 +3,10 @@ module Ci class Trigger < Ci::ApplicationRecord include Presentable + include Limitable + + self.limit_name = 'pipeline_triggers' + self.limit_scope = :project belongs_to :project belongs_to :owner, class_name: "User" diff --git a/app/models/clusters/agents/group_authorization.rb b/app/models/clusters/agents/group_authorization.rb index 28a711aaf17..58ba874ab53 100644 --- a/app/models/clusters/agents/group_authorization.rb +++ b/app/models/clusters/agents/group_authorization.rb @@ -3,6 +3,8 @@ module Clusters module Agents class GroupAuthorization < ApplicationRecord + include ::Clusters::Agents::AuthorizationConfigScopes + self.table_name = 'agent_group_authorizations' belongs_to :agent, class_name: 'Clusters::Agent', optional: false diff --git a/app/models/clusters/agents/project_authorization.rb b/app/models/clusters/agents/project_authorization.rb index f6d19086751..b9b44741936 100644 --- a/app/models/clusters/agents/project_authorization.rb +++ b/app/models/clusters/agents/project_authorization.rb @@ -3,6 +3,8 @@ module Clusters module Agents class ProjectAuthorization < ApplicationRecord + include ::Clusters::Agents::AuthorizationConfigScopes + self.table_name = 'agent_project_authorizations' belongs_to :agent, class_name: 'Clusters::Agent', optional: false diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 7cef92ce81a..59a9251d6b7 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.31.0' + VERSION = '0.34.0' self.table_name = 'clusters_applications_runners' @@ -70,7 +70,7 @@ module Clusters } if cluster.group_type? - attributes[:groups] = [group] + attributes[:runner_namespaces] = [::Ci::RunnerNamespace.new(namespace: group)] elsif cluster.project_type? attributes[:runner_projects] = [::Ci::RunnerProject.new(project: project)] end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index feac7bbc363..87afa9f9491 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -139,8 +139,6 @@ module Clusters scope :with_available_elasticstack, -> { joins(:application_elastic_stack).merge(::Clusters::Applications::ElasticStack.available) } scope :with_available_cilium, -> { joins(:application_cilium).merge(::Clusters::Applications::Cilium.available) } scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct } - scope :preload_elasticstack, -> { preload(:integration_elastic_stack) } - scope :preload_environments, -> { preload(:environments) } scope :managed, -> { where(managed: true) } scope :with_persisted_applications, -> { eager_load(*APPLICATIONS_ASSOCIATIONS) } @@ -150,9 +148,7 @@ module Clusters scope :for_project_namespace, -> (namespace_id) { joins(:projects).where(projects: { namespace_id: namespace_id }) } scope :with_name, -> (name) { where(name: name) } - # with_application_prometheus scope is deprecated, and scheduled for removal - # in %14.0. See https://gitlab.com/groups/gitlab-org/-/epics/4280 - scope :with_application_prometheus, -> { includes(:application_prometheus).joins(:application_prometheus) } + scope :with_integration_prometheus, -> { includes(:integration_prometheus).joins(:integration_prometheus) } scope :with_project_http_integrations, -> (project_ids) do conditions = { projects: :alert_management_http_integrations } includes(conditions).joins(conditions).where(projects: { id: project_ids }) @@ -311,7 +307,7 @@ module Clusters end def kubeclient - platform_kubernetes.kubeclient if kubernetes? + platform_kubernetes&.kubeclient if kubernetes? end def elastic_stack_adapter diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb index d745a49afc1..8b21fa351a3 100644 --- a/app/models/clusters/integrations/prometheus.rb +++ b/app/models/clusters/integrations/prometheus.rb @@ -14,6 +14,13 @@ module Clusters validates :cluster, presence: true validates :enabled, inclusion: { in: [true, false] } + # Periodically checked and kept up to date for Monitor demo projects + enum health_status: { + unknown: 0, + healthy: 1, + unhealthy: 2 + } + attr_encrypted :alert_manager_token, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_32, diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 43427e2ebc7..d75f7984e2c 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -48,7 +48,7 @@ class CommitStatus < Ci::ApplicationRecord scope :ordered, -> { order(:name) } scope :ordered_by_stage, -> { order(stage_idx: :asc) } scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) } - scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } + scope :retried_ordered, -> { retried.order(name: :asc, id: :desc).includes(project: :namespace) } scope :ordered_by_pipeline, -> { order(pipeline_id: :asc) } scope :before_stage, -> (index) { where('stage_idx < ?', index) } scope :for_stage, -> (index) { where(stage_idx: index) } diff --git a/app/models/concerns/alert_event_lifecycle.rb b/app/models/concerns/alert_event_lifecycle.rb index 4d2b717ead2..72fe7757b44 100644 --- a/app/models/concerns/alert_event_lifecycle.rb +++ b/app/models/concerns/alert_event_lifecycle.rb @@ -41,8 +41,6 @@ module AlertEventLifecycle scope :firing, -> { where(status: status_value_for(:firing)) } scope :resolved, -> { where(status: status_value_for(:resolved)) } - scope :count_by_project_id, -> { group(:project_id).count } - def self.status_value_for(name) state_machines[:status].states[name].value end diff --git a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb index 7462e1e828b..324e0fb57cb 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb @@ -5,6 +5,23 @@ module Analytics module StageEventModel extend ActiveSupport::Concern + included do + scope :by_stage_event_hash_id, ->(id) { where(stage_event_hash_id: id) } + scope :by_project_id, ->(id) { where(project_id: id) } + scope :by_group_id, ->(id) { where(group_id: id) } + scope :end_event_timestamp_after, -> (date) { where(arel_table[:end_event_timestamp].gteq(date)) } + scope :end_event_timestamp_before, -> (date) { where(arel_table[:end_event_timestamp].lteq(date)) } + scope :start_event_timestamp_after, -> (date) { where(arel_table[:start_event_timestamp].gteq(date)) } + scope :start_event_timestamp_before, -> (date) { where(arel_table[:start_event_timestamp].lteq(date)) } + scope :authored, ->(user) { where(author_id: user) } + scope :with_milestone_id, ->(milestone_id) { where(milestone_id: milestone_id) } + scope :end_event_is_not_happened_yet, -> { where(end_event_timestamp: nil) } + end + + def issuable_id + attributes[self.class.issuable_id_column.to_s] + end + class_methods do def upsert_data(data) upsert_values = data.map do |row| @@ -13,8 +30,9 @@ module Analytics :issuable_id, :group_id, :project_id, - :author_id, :milestone_id, + :author_id, + :state_id, :start_event_timestamp, :end_event_timestamp ) @@ -31,6 +49,7 @@ module Analytics project_id, milestone_id, author_id, + state_id, start_event_timestamp, end_event_timestamp ) @@ -39,10 +58,11 @@ module Analytics DO UPDATE SET group_id = excluded.group_id, project_id = excluded.project_id, - start_event_timestamp = excluded.start_event_timestamp, - end_event_timestamp = excluded.end_event_timestamp, milestone_id = excluded.milestone_id, - author_id = excluded.author_id + author_id = excluded.author_id, + state_id = excluded.state_id, + start_event_timestamp = excluded.start_event_timestamp, + end_event_timestamp = excluded.end_event_timestamp SQL result = connection.execute(query) diff --git a/app/models/concerns/cascading_namespace_setting_attribute.rb b/app/models/concerns/cascading_namespace_setting_attribute.rb index e58e5ddc966..731729a1ed5 100644 --- a/app/models/concerns/cascading_namespace_setting_attribute.rb +++ b/app/models/concerns/cascading_namespace_setting_attribute.rb @@ -127,7 +127,7 @@ module CascadingNamespaceSettingAttribute end def alias_boolean(attribute) - return unless Gitlab::Database.main.exists? && type_for_attribute(attribute).type == :boolean + return unless database.exists? && type_for_attribute(attribute).type == :boolean alias_method :"#{attribute}?", attribute end @@ -176,10 +176,10 @@ module CascadingNamespaceSettingAttribute private def locked_value(attribute) + return application_setting_value(attribute) if locked_by_application_setting?(attribute) + ancestor = locked_ancestor(attribute) return ancestor.read_attribute(attribute) if ancestor - - Gitlab::CurrentSettings.public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend end def locked_ancestor(attribute) diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 27a704c1de0..a9589cea5e9 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -10,11 +10,14 @@ module Ci # Variables in the environment name scope. # def scoped_variables(environment: expanded_environment_name, dependencies: true) - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.concat(predefined_variables) + track_duration do + variables = pipeline.variables_builder.scoped_variables(self, environment: environment, dependencies: dependencies) + + variables.concat(predefined_variables) unless pipeline.predefined_vars_in_builder_enabled? variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runnable? && runner + variables.concat(kubernetes_variables) variables.concat(deployment_variables(environment: environment)) variables.concat(yaml_variables) variables.concat(user_variables) @@ -25,9 +28,23 @@ module Ci variables.concat(trigger_request.user_variables) if trigger_request variables.concat(pipeline.variables) variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule + + variables end end + def track_duration + start_time = ::Gitlab::Metrics::System.monotonic_time + result = yield + duration = ::Gitlab::Metrics::System.monotonic_time - start_time + + ::Gitlab::Ci::Pipeline::Metrics + .pipeline_builder_scoped_variables_histogram + .observe({}, duration.seconds) + + result + end + ## # Variables that do not depend on the environment name. # @@ -72,6 +89,18 @@ module Ci end end + def kubernetes_variables + ::Gitlab::Ci::Variables::Collection.new.tap do |collection| + # Should get merged with the cluster kubeconfig in deployment_variables, see + # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + template = ::Ci::GenerateKubeconfigService.new(self).execute + + if template.valid? + collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) + end + end + end + def deployment_variables(environment:) return [] unless environment diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 8d715279da8..ccaccec3b6b 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -14,21 +14,8 @@ module Ci PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, - failed: 4, canceled: 5, skipped: 6, manual: 7, - scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze - STATUSES_DESCRIPTION = { - created: 'Pipeline has been created', - waiting_for_resource: 'A resource (for example, a runner) that the pipeline requires to run is unavailable', - preparing: 'Pipeline is preparing to run', - pending: 'Pipeline has not started running yet', - running: 'Pipeline is running', - failed: 'At least one stage of the pipeline failed', - success: 'Pipeline completed successfully', - canceled: 'Pipeline was canceled before completion', - skipped: 'Pipeline was skipped', - manual: 'Pipeline needs to be manually started', - scheduled: 'Pipeline is scheduled to run' - }.freeze + failed: 4, canceled: 5, skipped: 6, manual: 7, + scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze UnknownStatusError = Class.new(StandardError) diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 344f5aa4cd5..611b27c722b 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -20,7 +20,8 @@ module Ci delegate :interruptible, to: :metadata, prefix: false, allow_nil: true delegate :has_exposed_artifacts?, to: :metadata, prefix: false, allow_nil: true delegate :environment_auto_stop_in, to: :metadata, prefix: false, allow_nil: true - delegate :runner_features, to: :metadata, prefix: false, allow_nil: false + delegate :set_cancel_gracefully, to: :metadata, prefix: false, allow_nil: false + delegate :cancel_gracefully?, to: :metadata, prefix: false, allow_nil: false before_create :ensure_metadata end diff --git a/app/models/concerns/clusters/agents/authorization_config_scopes.rb b/app/models/concerns/clusters/agents/authorization_config_scopes.rb new file mode 100644 index 00000000000..0a0406c3389 --- /dev/null +++ b/app/models/concerns/clusters/agents/authorization_config_scopes.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module AuthorizationConfigScopes + extend ActiveSupport::Concern + + included do + scope :with_available_ci_access_fields, ->(project) { + where("config->'access_as' IS NULL") + .or(where("config->'access_as' = '{}'")) + .or(where("config->'access_as' ?| array[:fields]", fields: available_ci_access_fields(project))) + } + end + + class_methods do + def available_ci_access_fields(_project) + %w(agent) + end + end + end + end +end + +Clusters::Agents::AuthorizationConfigScopes.prepend_mod diff --git a/app/models/concerns/database_reflection.rb b/app/models/concerns/database_reflection.rb new file mode 100644 index 00000000000..1842f5bf4ec --- /dev/null +++ b/app/models/concerns/database_reflection.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# A module that makes it easier/less verbose to reflect upon a database +# connection. +# +# Using this module you can write this: +# +# User.database.database_name +# +# Instead of this: +# +# Gitlab::Database::Reflection.new(User).database_name +module DatabaseReflection + extend ActiveSupport::Concern + + class_methods do + def database + @database_reflection ||= ::Gitlab::Database::Reflection.new(self) + end + end +end diff --git a/app/models/concerns/enums/vulnerability.rb b/app/models/concerns/enums/vulnerability.rb index 749d1ad65cd..4b325de61bc 100644 --- a/app/models/concerns/enums/vulnerability.rb +++ b/app/models/concerns/enums/vulnerability.rb @@ -37,6 +37,15 @@ module Enums security_audit: 4 }.with_indifferent_access.freeze + # keep the order of the values in the state enum, it is used in state_order method to properly order vulnerabilities based on state + # remember to recreate index_vulnerabilities_on_state_case_id index when you update or extend this enum + VULNERABILITY_STATES = { + detected: 1, + confirmed: 4, + resolved: 3, + dismissed: 2 + }.with_indifferent_access.freeze + def self.confidence_levels CONFIDENCE_LEVELS end @@ -52,6 +61,10 @@ module Enums def self.detection_methods DETECTION_METHODS end + + def self.vulnerability_states + VULNERABILITY_STATES + end end end diff --git a/app/models/concerns/file_store_mounter.rb b/app/models/concerns/file_store_mounter.rb index 9d4463e5297..bfcf8a1e7b9 100644 --- a/app/models/concerns/file_store_mounter.rb +++ b/app/models/concerns/file_store_mounter.rb @@ -7,15 +7,13 @@ module FileStoreMounter def mount_file_store_uploader(uploader) mount_uploader(:file, uploader) + # This hook is a no-op when the file is uploaded after_commit after_save :update_file_store, if: :saved_change_to_file? end end - private - def update_file_store - # The file.object_store is set during `uploader.store!` - # which happens after object is inserted/updated - self.update_column(:file_store, file.object_store) + # The file.object_store is set during `uploader.store!` and `uploader.migrate!` + update_column(:file_store, file.object_store) end end diff --git a/app/models/concerns/has_integrations.rb b/app/models/concerns/has_integrations.rb deleted file mode 100644 index 76e03d68600..00000000000 --- a/app/models/concerns/has_integrations.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module HasIntegrations - extend ActiveSupport::Concern - - class_methods do - def without_integration(integration) - integrations = Integration - .select('1') - .where("#{Integration.table_name}.project_id = projects.id") - .where(type: integration.type) - - Project - .where('NOT EXISTS (?)', integrations) - .where(pending_delete: false) - .where(archived: false) - end - end -end diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb index 4b4f9c0df84..28ee54afaa9 100644 --- a/app/models/concerns/has_user_type.rb +++ b/app/models/concerns/has_user_type.rb @@ -28,6 +28,7 @@ module HasUserType scope :non_internal, -> { humans.or(where(user_type: NON_INTERNAL_USER_TYPES)) } scope :without_ghosts, -> { humans.or(where.not(user_type: :ghost)) } scope :without_project_bot, -> { humans.or(where.not(user_type: :project_bot)) } + scope :human_or_service_user, -> { humans.or(where(user_type: :service_user)) } enum user_type: USER_TYPES diff --git a/app/models/concerns/integrations/push_data_validations.rb b/app/models/concerns/integrations/push_data_validations.rb new file mode 100644 index 00000000000..966fc94e289 --- /dev/null +++ b/app/models/concerns/integrations/push_data_validations.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# This concern is used by registered integrations such as Integrations::TeamCity and +# Integrations::DroneCi and adds methods to perform validations on the received +# data. +module Integrations + module PushDataValidations + extend ActiveSupport::Concern + + def merge_request_valid?(data) + data.dig(:object_attributes, :state) == 'opened' && merge_request_unchecked?(data) + end + + def push_valid?(data) + data[:total_commits_count] > 0 && + !branch_removed?(data) && + # prefer merge request trigger over push to avoid double builds + !opened_merge_requests?(data) + end + + def tag_push_valid?(data) + data[:total_commits_count] > 0 && !branch_removed?(data) + end + + private + + def branch_removed?(data) + Gitlab::Git.blank_ref?(data[:after]) + end + + def opened_merge_requests?(data) + project.merge_requests + .opened + .from_project(project) + .from_source_branches(Gitlab::Git.ref_name(data[:ref])) + .exists? + end + + def merge_request_unchecked?(data) + MergeRequest.state_machines[:merge_status] + .check_state?(data.dig(:object_attributes, :merge_status)) + end + end +end diff --git a/app/models/concerns/integrations/reactively_cached.rb b/app/models/concerns/integrations/reactively_cached.rb new file mode 100644 index 00000000000..62eff06c8e2 --- /dev/null +++ b/app/models/concerns/integrations/reactively_cached.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Integrations + module ReactivelyCached + extend ActiveSupport::Concern + + included do + include ::ReactiveCaching + + # Default cache key: class name + project_id + self.reactive_cache_key = ->(integration) { [integration.class.model_name.singular, integration.project_id] } + self.reactive_cache_work_type = :external_dependency + end + end +end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 5c307158a9a..4273eb331a1 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -92,7 +92,6 @@ module Issuable scope :recent, -> { reorder(id: :desc) } scope :of_projects, ->(ids) { where(project_id: ids) } scope :opened, -> { with_state(:opened) } - scope :only_opened, -> { with_state(:opened) } scope :closed, -> { with_state(:closed) } # rubocop:disable GitlabSecurity/SqlInjection diff --git a/app/models/concerns/legacy_bulk_insert.rb b/app/models/concerns/legacy_bulk_insert.rb new file mode 100644 index 00000000000..1249dfb70cd --- /dev/null +++ b/app/models/concerns/legacy_bulk_insert.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module LegacyBulkInsert + extend ActiveSupport::Concern + + class_methods do + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # This method is deprecated, and you should use the BulkInsertSafe module + # instead. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + # on_conflict - Defines an upsert. Values can be: :disabled (default) or + # :do_nothing + def legacy_bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) + return if rows.empty? + + keys = rows.first.keys + columns = keys.map { |key| connection.quote_column_name(key) } + + disable_quote = Array(disable_quote).to_set + tuples = rows.map do |row| + keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end + end + + sql = <<-EOF + INSERT INTO #{table} (#{columns.join(', ')}) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + + sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing + + sql = "#{sql} RETURNING id" if return_ids + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end + end + end +end diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb index 848ef63f1c2..98f6ad58434 100644 --- a/app/models/concerns/loaded_in_group_list.rb +++ b/app/models/concerns/loaded_in_group_list.rb @@ -41,9 +41,11 @@ module LoadedInGroupList namespaces = Namespace.arel_table children = namespaces.alias('children') + # TODO 6473: remove the filtering of the Namespaces::ProjectNamespace see https://gitlab.com/groups/gitlab-org/-/epics/6473 namespaces.project(Arel.star.count.as('preloaded_subgroup_count')) .from(children) .where(children[:parent_id].eq(namespaces[:id])) + .where(children[:type].is_distinct_from(Namespaces::ProjectNamespace.sti_name)) end def member_count_sql diff --git a/app/models/concerns/loose_foreign_key.rb b/app/models/concerns/loose_foreign_key.rb index 4e822a04869..102292672b3 100644 --- a/app/models/concerns/loose_foreign_key.rb +++ b/app/models/concerns/loose_foreign_key.rb @@ -7,20 +7,18 @@ module LooseForeignKey # Loose foreign keys allow delayed processing of associated database records # with similar guarantees than a database foreign key. # - # TODO: finalize this later once the async job is in place - # # Prerequisites: # # To start using the concern, you'll need to install a database trigger to the parent # table in a standard DB migration (not post-migration). # - # > add_loose_foreign_key_support(:projects, :gitlab_main) + # > track_record_deletions(:projects) # # Usage: # # > class Ci::Build < ApplicationRecord # > - # > loose_foreign_key :security_scans, :build_id, on_delete: :async_delete, gitlab_schema: :gitlab_main + # > loose_foreign_key :security_scans, :build_id, on_delete: :async_delete # > # > # associations can be still defined, the dependent options is no longer necessary: # > has_many :security_scans, class_name: 'Security::Scan' @@ -32,14 +30,6 @@ module LooseForeignKey # - :async_delete - deletes the children rows via an asynchronous process. # - :async_nullify - sets the foreign key column to null via an asynchronous process. # - # Options for gitlab_schema: - # - # - :gitlab_ci - # - :gitlab_main - # - # The value can be determined by calling `Model.gitlab_schema` where the Model represents - # the model for the child table. - # # How it works: # # When adding loose foreign key support to the table, a DELETE trigger is installed @@ -69,23 +59,17 @@ module LooseForeignKey end on_delete_options = %i[async_delete async_nullify] - gitlab_schema_options = [ApplicationRecord.gitlab_schema, Ci::ApplicationRecord.gitlab_schema] unless on_delete_options.include?(symbolized_options[:on_delete]&.to_sym) raise "Invalid on_delete option given: #{symbolized_options[:on_delete]}. Valid options: #{on_delete_options.join(', ')}" end - unless gitlab_schema_options.include?(symbolized_options[:gitlab_schema]&.to_sym) - raise "Invalid gitlab_schema option given: #{symbolized_options[:gitlab_schema]}. Valid options: #{gitlab_schema_options.join(', ')}" - end - definition = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( table_name.to_s, to_table.to_s, { column: column.to_s, - on_delete: symbolized_options[:on_delete].to_sym, - gitlab_schema: symbolized_options[:gitlab_schema].to_sym + on_delete: symbolized_options[:on_delete].to_sym } ) diff --git a/app/models/concerns/merge_request_reviewer_state.rb b/app/models/concerns/merge_request_reviewer_state.rb new file mode 100644 index 00000000000..216a3a0bd64 --- /dev/null +++ b/app/models/concerns/merge_request_reviewer_state.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module MergeRequestReviewerState + extend ActiveSupport::Concern + + included do + enum state: { + unreviewed: 0, + reviewed: 1, + attention_requested: 2 + } + + validates :state, + presence: true, + inclusion: { in: self.states.keys } + + after_initialize :set_state, unless: :persisted? + + def set_state + if Feature.enabled?(:mr_attention_requests, self.merge_request&.project, default_enabled: :yaml) + self.state = :attention_requested + end + end + end +end diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index c4f810ab9b1..12041b103f6 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -14,13 +14,12 @@ module Milestoneable validate :milestone_is_valid - scope :of_milestones, ->(ids) { where(milestone_id: ids) } scope :any_milestone, -> { where.not(milestone_id: nil) } scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) } scope :without_particular_milestone, ->(title) { left_outer_joins(:milestone).where("milestones.title != ? OR milestone_id IS NULL", title) } scope :any_release, -> { joins_milestone_releases } scope :with_release, -> (tag, project_id) { joins_milestone_releases.where( milestones: { releases: { tag: tag, project_id: project_id } } ) } - scope :without_particular_release, -> (tag, project_id) { joins_milestone_releases.where.not( milestones: { releases: { tag: tag, project_id: project_id } } ) } + scope :without_particular_release, -> (tag, project_id) { joins_milestone_releases.where.not(milestones: { releases: { tag: tag, project_id: project_id } }) } scope :left_joins_milestones, -> { joins("LEFT OUTER JOIN milestones ON #{table_name}.milestone_id = milestones.id") } scope :order_milestone_due_desc, -> { left_joins_milestones.reorder(Arel.sql('milestones.due_date IS NULL, milestones.id IS NULL, milestones.due_date DESC')) } diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index f6d4e5bd27b..ea4fe5b27dc 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -98,6 +98,27 @@ module Noteable .order('MIN(created_at), MIN(id)') end + # This does not consider OutOfContextDiscussions in MRs + # where notes from commits are overriden so that they have + # the same discussion_id + def discussion_root_note_ids(notes_filter:) + relations = [] + + relations << discussion_notes.select( + "'notes' AS table_name", + 'discussion_id', + 'MIN(id) AS id', + 'MIN(created_at) AS created_at' + ).with_notes_filter(notes_filter) + .group(:discussion_id) + + if notes_filter != UserPreference::NOTES_FILTERS[:only_comments] + relations += synthetic_note_ids_relations + end + + Note.from_union(relations, remove_duplicates: false).fresh + end + def capped_notes_count(max) notes.limit(max).count end @@ -179,6 +200,18 @@ module Noteable project_email.sub('@', "-#{iid}@") end + + private + + # Synthetic system notes don't have discussion IDs because these are generated dynamically + # in Ruby. These are always root notes anyway so we don't need to group by discussion ID. + def synthetic_note_ids_relations + [ + resource_label_events.select("'resource_label_events'", "'NULL'", :id, :created_at), + resource_milestone_events.select("'resource_milestone_events'", "'NULL'", :id, :created_at), + resource_state_events.select("'resource_state_events'", "'NULL'", :id, :created_at) + ] + end end Noteable.extend(Noteable::ClassMethods) diff --git a/app/models/concerns/reactive_service.rb b/app/models/concerns/reactive_service.rb deleted file mode 100644 index c444f238944..00000000000 --- a/app/models/concerns/reactive_service.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module ReactiveService - extend ActiveSupport::Concern - - included do - include ReactiveCaching - - # Default cache key: class name + project_id - self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] } - self.reactive_cache_work_type = :external_dependency - end -end diff --git a/app/models/concerns/security/latest_pipeline_information.rb b/app/models/concerns/security/latest_pipeline_information.rb new file mode 100644 index 00000000000..87eae3cac68 --- /dev/null +++ b/app/models/concerns/security/latest_pipeline_information.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Security + module LatestPipelineInformation + private + + def scanner_enabled?(scan_type) + latest_builds_reports.include?(scan_type) + end + + def latest_builds_reports(only_successful_builds: false) + strong_memoize("latest_builds_reports_#{only_successful_builds}") do + builds = latest_security_builds + builds = builds.select { |build| build.status == 'success' } if only_successful_builds + builds.flat_map do |build| + build.options[:artifacts][:reports].keys + end + end + end + + def latest_security_builds + return [] unless latest_default_branch_pipeline + + ::Security::SecurityJobsFinder.new(pipeline: latest_default_branch_pipeline).execute + + ::Security::LicenseComplianceJobsFinder.new(pipeline: latest_default_branch_pipeline).execute + end + + def latest_default_branch_pipeline + strong_memoize(:pipeline) { latest_pipeline } + end + + def auto_devops_source? + latest_default_branch_pipeline&.auto_devops_source? + end + end +end diff --git a/app/models/concerns/service_push_data_validations.rb b/app/models/concerns/service_push_data_validations.rb deleted file mode 100644 index 451804a2c56..00000000000 --- a/app/models/concerns/service_push_data_validations.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -# This concern is used by registered integrations such as Integrations::TeamCity and -# Integrations::DroneCi and adds methods to perform validations on the received -# data. - -module ServicePushDataValidations - extend ActiveSupport::Concern - - def merge_request_valid?(data) - data.dig(:object_attributes, :state) == 'opened' && merge_request_unchecked?(data) - end - - def push_valid?(data) - data[:total_commits_count] > 0 && - !branch_removed?(data) && - # prefer merge request trigger over push to avoid double builds - !opened_merge_requests?(data) - end - - def tag_push_valid?(data) - data[:total_commits_count] > 0 && !branch_removed?(data) - end - - private - - def branch_removed?(data) - Gitlab::Git.blank_ref?(data[:after]) - end - - def opened_merge_requests?(data) - project.merge_requests - .opened - .from_project(project) - .from_source_branches(Gitlab::Git.ref_name(data[:ref])) - .exists? - end - - def merge_request_unchecked?(data) - MergeRequest.state_machines[:merge_status] - .check_state?(data.dig(:object_attributes, :merge_status)) - end -end diff --git a/app/models/concerns/sha256_attribute.rb b/app/models/concerns/sha256_attribute.rb index 17fda6c806c..3c906642b1a 100644 --- a/app/models/concerns/sha256_attribute.rb +++ b/app/models/concerns/sha256_attribute.rb @@ -39,7 +39,7 @@ module Sha256Attribute end def database_exists? - Gitlab::Database.main.exists? + database.exists? end end end diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index 27277bc5296..ba7c6c0cd8b 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -32,7 +32,7 @@ module ShaAttribute end def database_exists? - Gitlab::Database.main.exists? + database.exists? end end end diff --git a/app/models/concerns/strip_attribute.rb b/app/models/concerns/strip_attribute.rb index 1c433a3275e..817a4465f91 100644 --- a/app/models/concerns/strip_attribute.rb +++ b/app/models/concerns/strip_attribute.rb @@ -2,7 +2,8 @@ # == Strip Attribute module # -# Contains functionality to clean attributes before validation +# Contains functionality to remove leading and trailing +# whitespace from the attribute before validation # # Usage: # diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index 79cbe225e5a..3fe9d7f4d71 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -11,9 +11,7 @@ module Timebox include StripAttribute include FromUnion - TimeboxStruct = Struct.new(:title, :name, :id) do - include GlobalID::Identification - + TimeboxStruct = Struct.new(:title, :name, :id, :class_name) do # Ensure these models match the interface required for exporting def serializable_hash(_opts = {}) { title: title, name: name, id: id } @@ -22,6 +20,10 @@ module Timebox def self.declarative_policy_class "TimeboxPolicy" end + + def to_global_id + ::Gitlab::GlobalId.build(self, model_name: class_name, id: id) + end end # Represents a "No Timebox" state used for filtering Issues and Merge @@ -33,10 +35,10 @@ module Timebox included do # Defines the same constants above, but inside the including class. - const_set :None, TimeboxStruct.new("No #{self.name}", "No #{self.name}", 0) - const_set :Any, TimeboxStruct.new("Any #{self.name}", '', -1) - const_set :Upcoming, TimeboxStruct.new('Upcoming', '#upcoming', -2) - const_set :Started, TimeboxStruct.new('Started', '#started', -3) + const_set :None, TimeboxStruct.new("No #{self.name}", "No #{self.name}", 0, self.name) + const_set :Any, TimeboxStruct.new("Any #{self.name}", '', -1, self.name) + const_set :Upcoming, TimeboxStruct.new('Upcoming', '#upcoming', -2, self.name) + const_set :Started, TimeboxStruct.new('Started', '#started', -3, self.name) alias_method :timebox_id, :id diff --git a/app/models/concerns/transactions.rb b/app/models/concerns/transactions.rb new file mode 100644 index 00000000000..a186ebc8475 --- /dev/null +++ b/app/models/concerns/transactions.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Transactions + extend ActiveSupport::Concern + + class_methods do + # inside_transaction? will return true if the caller is running within a + # transaction. Handles special cases when running inside a test environment, + # where tests may be wrapped in transactions + def inside_transaction? + base = Rails.env.test? ? @open_transactions_baseline.to_i : 0 + + connection.open_transactions > base + end + + # These methods that access @open_transactions_baseline are not thread-safe. + # These are fine though because we only call these in RSpec's main thread. + # If we decide to run specs multi-threaded, we would need to use something + # like ThreadGroup to keep track of this value + def set_open_transactions_baseline + @open_transactions_baseline = connection.open_transactions + end + + def reset_open_transactions_baseline + @open_transactions_baseline = 0 + end + end +end diff --git a/app/models/concerns/ttl_expirable.rb b/app/models/concerns/ttl_expirable.rb index 00abe0a06e6..6d89521255c 100644 --- a/app/models/concerns/ttl_expirable.rb +++ b/app/models/concerns/ttl_expirable.rb @@ -5,10 +5,11 @@ module TtlExpirable included do validates :status, presence: true + default_value_for :read_at, Time.zone.now enum status: { default: 0, expired: 1, processing: 2, error: 3 } - scope :updated_before, ->(number_of_days) { where("updated_at <= ?", Time.zone.now - number_of_days.days) } + scope :read_before, ->(number_of_days) { where("read_at <= ?", Time.zone.now - number_of_days.days) } scope :active, -> { where(status: :default) } scope :lock_next_by, ->(sort) do @@ -17,4 +18,8 @@ module TtlExpirable .lock('FOR UPDATE SKIP LOCKED') end end + + def read! + self.update(read_at: Time.zone.now) + end end diff --git a/app/models/concerns/update_highest_role.rb b/app/models/concerns/update_highest_role.rb index 6432cc794a5..2b0ec5c7e21 100644 --- a/app/models/concerns/update_highest_role.rb +++ b/app/models/concerns/update_highest_role.rb @@ -15,7 +15,7 @@ module UpdateHighestRole # Schedule a Sidekiq job to update the highest role for a User # # The job will be called outside of a transaction in order to ensure the changes - # to be commited before attempting to update the highest role. + # to be committed before attempting to update the highest role. # The exlusive lease will not be released after completion to prevent multiple jobs # being executed during the defined timeout. def update_highest_role diff --git a/app/models/concerns/x509_serial_number_attribute.rb b/app/models/concerns/x509_serial_number_attribute.rb index dfb1e151b41..e51ed95bf70 100644 --- a/app/models/concerns/x509_serial_number_attribute.rb +++ b/app/models/concerns/x509_serial_number_attribute.rb @@ -39,7 +39,7 @@ module X509SerialNumberAttribute end def database_exists? - Gitlab::Database.main.exists? + database.exists? end end end diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index ecdac64b31b..173b38b2c63 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -31,7 +31,7 @@ class CustomEmoji < ApplicationRecord private def valid_emoji_name - if Gitlab::Emoji.emoji_exists?(name) + if TanukiEmoji.find_by_alpha_code(name) errors.add(:name, _('%{name} is already being used for another emoji') % { name: self.name }) end end diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index c632f8e2efa..5898bc3412f 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -7,7 +7,8 @@ class CustomerRelations::Contact < ApplicationRecord belongs_to :group, -> { where(type: Group.sti_name) }, foreign_key: 'group_id' belongs_to :organization, optional: true - has_and_belongs_to_many :issues, join_table: :issue_customer_relations_contacts # rubocop: disable Rails/HasAndBelongsToMany + has_many :issue_contacts, inverse_of: :contact + has_many :issues, through: :issue_contacts, inverse_of: :customer_relations_contacts strip_attributes! :phone, :first_name, :last_name diff --git a/app/models/customer_relations/issue_contact.rb b/app/models/customer_relations/issue_contact.rb new file mode 100644 index 00000000000..98faf8d6644 --- /dev/null +++ b/app/models/customer_relations/issue_contact.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class CustomerRelations::IssueContact < ApplicationRecord + self.table_name = "issue_customer_relations_contacts" + + belongs_to :issue, optional: false, inverse_of: :customer_relations_contacts + belongs_to :contact, optional: false, inverse_of: :issue_contacts + + validate :contact_belongs_to_issue_group + + private + + def contact_belongs_to_issue_group + return unless contact&.group_id + return unless issue&.project&.namespace_id + return if contact.group_id == issue.project.namespace_id + + errors.add(:base, _('The contact does not belong to the same group as the issue')) + end +end diff --git a/app/models/data_list.rb b/app/models/data_list.rb index adad8e3013e..e99364b2709 100644 --- a/app/models/data_list.rb +++ b/app/models/data_list.rb @@ -1,22 +1,26 @@ # frozen_string_literal: true class DataList - def initialize(batch, data_fields_hash, klass) + def initialize(batch, data_fields_hash, data_fields_klass) @batch = batch @data_fields_hash = data_fields_hash - @klass = klass + @data_fields_klass = data_fields_klass end def to_array - [klass, columns, values] + [data_fields_klass, columns, values] end private - attr_reader :batch, :data_fields_hash, :klass + attr_reader :batch, :data_fields_hash, :data_fields_klass def columns - data_fields_hash.keys << 'service_id' + data_fields_hash.keys << data_fields_foreign_key + end + + def data_fields_foreign_key + data_fields_klass.reflections['integration'].foreign_key end def values diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb index 7ca15652586..bd5c022e692 100644 --- a/app/models/dependency_proxy/blob.rb +++ b/app/models/dependency_proxy/blob.rb @@ -7,6 +7,8 @@ class DependencyProxy::Blob < ApplicationRecord belongs_to :group + MAX_FILE_SIZE = 5.gigabytes.freeze + validates :group, presence: true validates :file, presence: true validates :file_name, presence: true diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index b83047efe54..64f484942ef 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -7,17 +7,19 @@ class DependencyProxy::Manifest < ApplicationRecord belongs_to :group + MAX_FILE_SIZE = 10.megabytes.freeze + DIGEST_HEADER = 'Docker-Content-Digest' + validates :group, presence: true validates :file, presence: true validates :file_name, presence: true validates :digest, presence: true - mount_file_store_uploader DependencyProxy::FileUploader + scope :order_id_desc, -> { reorder(id: :desc) } - def self.find_or_initialize_by_file_name_or_digest(file_name:, digest:) - result = find_by(file_name: file_name) || find_by(digest: digest) - return result if result + mount_file_store_uploader DependencyProxy::FileUploader - new(file_name: file_name, digest: digest) + def self.find_by_file_name_or_digest(file_name:, digest:) + find_by(file_name: file_name) || find_by(digest: digest) end end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 8f5a713af3f..4ed38f578ee 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -7,12 +7,16 @@ class DeployKey < Key has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :deploy_keys_projects + + has_many :deploy_keys_projects_with_write_access, -> { with_write_access }, class_name: "DeployKeysProject" + has_many :projects_with_write_access, -> { includes(:route) }, class_name: 'Project', through: :deploy_keys_projects_with_write_access, source: :project has_many :protected_branch_push_access_levels, class_name: '::ProtectedBranch::PushAccessLevel' scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where(deploy_keys_projects: { project_id: projects }) } scope :with_write_access, -> { joins(:deploy_keys_projects).merge(DeployKeysProject.with_write_access) } scope :are_public, -> { where(public: true) } scope :with_projects, -> { includes(deploy_keys_projects: { project: [:route, namespace: :route] }) } + scope :including_projects_with_write_access, -> { includes(:projects_with_write_access) } accepts_nested_attributes_for :deploy_keys_projects @@ -52,10 +56,6 @@ class DeployKey < Key end end - def projects_with_write_access - Project.with_route.where(id: deploy_keys_projects.with_write_access.select(:project_id)) - end - def self.with_write_access_for_project(project, deploy_key: nil) query = in_projects(project).with_write_access query = query.where(id: deploy_key) if deploy_key diff --git a/app/models/deployment.rb b/app/models/deployment.rb index f91700f764b..ade19ce02a8 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -8,11 +8,12 @@ class Deployment < ApplicationRecord include Importable include Gitlab::Utils::StrongMemoize include FastDestroyAll - include IgnorableColumns StatusUpdateError = Class.new(StandardError) StatusSyncError = Class.new(StandardError) + ARCHIVABLE_OFFSET = 50_000 + belongs_to :project, required: true belongs_to :environment, required: true belongs_to :cluster, class_name: 'Clusters::Cluster', optional: true @@ -54,6 +55,8 @@ class Deployment < ApplicationRecord scope :finished_after, ->(date) { where('finished_at >= ?', date) } scope :finished_before, ->(date) { where('finished_at < ?', date) } + scope :ordered, -> { order(finished_at: :desc) } + FINISHED_STATUSES = %i[success failed canceled].freeze state_machine :status, initial: :created do @@ -99,6 +102,10 @@ class Deployment < ApplicationRecord deployment.run_after_commit do Deployments::UpdateEnvironmentWorker.perform_async(id) Deployments::LinkMergeRequestWorker.perform_async(id) + + if ::Feature.enabled?(:deployments_archive, deployment.project, default_enabled: :yaml) + Deployments::ArchiveInProjectWorker.perform_async(deployment.project_id) + end end end @@ -132,6 +139,14 @@ class Deployment < ApplicationRecord skipped: 5 } + def self.archivables_in(project, limit:) + start_iid = project.deployments.order(iid: :desc).limit(1) + .select("(iid - #{ARCHIVABLE_OFFSET}) AS start_iid") + + project.deployments.preload(:environment).where('iid <= (?)', start_iid) + .where(archived: false).limit(limit) + end + def self.last_for_environment(environment) ids = self .for_environment(environment) @@ -299,7 +314,7 @@ class Deployment < ApplicationRecord "#{id} as deployment_id", "#{environment_id} as environment_id").to_sql - # We don't use `Gitlab::Database.main.bulk_insert` here so that we don't need to + # We don't use `ApplicationRecord.legacy_bulk_insert` here so that we don't need to # first pluck lots of IDs into memory. # # We also ignore any duplicates so this method can be called multiple times @@ -325,6 +340,7 @@ class Deployment < ApplicationRecord def sync_status_with(build) return false unless ::Deployment.statuses.include?(build.status) + return false if build.created? || build.status == self.status update_status!(build.status) rescue StandardError => e diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb index 6cda03557d1..5819404efb9 100644 --- a/app/models/design_management/version.rb +++ b/app/models/design_management/version.rb @@ -88,7 +88,7 @@ module DesignManagement rows = design_actions.map { |action| action.row_attrs(version) } - Gitlab::Database.main.bulk_insert(::DesignManagement::Action.table_name, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(::DesignManagement::Action.table_name, rows) # rubocop:disable Gitlab/BulkInsert version.designs.reset version.validate! design_actions.each(&:performed) diff --git a/app/models/email.rb b/app/models/email.rb index 0140f784842..676e79406e9 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -29,7 +29,7 @@ class Email < ApplicationRecord end def unique_email - self.errors.add(:email, 'has already been taken') if User.exists?(email: self.email) + self.errors.add(:email, 'has already been taken') if primary_email_of_another_user? end def validate_email_format @@ -40,4 +40,14 @@ class Email < ApplicationRecord def update_invalid_gpg_signatures user.update_invalid_gpg_signatures if confirmed? end + + def user_primary_email? + email.casecmp?(user.email) + end + + private + + def primary_email_of_another_user? + User.where(email: email).where.not(id: user_id).exists? + end end diff --git a/app/models/environment.rb b/app/models/environment.rb index 31ab426728b..2618991c9e5 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -425,6 +425,14 @@ class Environment < ApplicationRecord clear_reactive_cache! end + def should_link_to_merge_requests? + unfoldered? || production? || staging? + end + + def unfoldered? + environment_type.nil? + end + private def rollout_status_available? diff --git a/app/models/error_tracking/error.rb b/app/models/error_tracking/error.rb index 2d6a4694def..efbb6adff70 100644 --- a/app/models/error_tracking/error.rb +++ b/app/models/error_tracking/error.rb @@ -18,9 +18,10 @@ class ErrorTracking::Error < ApplicationRecord scope :for_status, -> (status) { where(status: status) } validates :project, presence: true - validates :name, presence: true - validates :description, presence: true - validates :actor, presence: true + validates :name, presence: true, length: { maximum: 255 } + validates :description, presence: true, length: { maximum: 1024 } + validates :actor, presence: true, length: { maximum: 255 } + validates :platform, length: { maximum: 255 } validates :status, presence: true enum status: { diff --git a/app/models/error_tracking/error_event.rb b/app/models/error_tracking/error_event.rb index 686518a39fb..0b638f65768 100644 --- a/app/models/error_tracking/error_event.rb +++ b/app/models/error_tracking/error_event.rb @@ -6,7 +6,9 @@ class ErrorTracking::ErrorEvent < ApplicationRecord validates :payload, json_schema: { filename: 'error_tracking_event_payload' } validates :error, presence: true - validates :description, presence: true + validates :description, presence: true, length: { maximum: 1024 } + validates :level, length: { maximum: 255 } + validates :environment, length: { maximum: 255 } validates :occurred_at, presence: true def stacktrace @@ -61,9 +63,9 @@ class ErrorTracking::ErrorEvent < ApplicationRecord pre_context = entry['pre_context'] post_context = entry['post_context'] - context += lines_with_position(pre_context, error_line_no - pre_context.size) + context += lines_with_position(pre_context, error_line_no - pre_context.size) if pre_context context += lines_with_position([error_line], error_line_no) - context += lines_with_position(post_context, error_line_no + 1) + context += lines_with_position(post_context, error_line_no + 1) if post_context context.reject(&:blank?) end diff --git a/app/models/event.rb b/app/models/event.rb index d6588699d27..f6174589a84 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -9,9 +9,6 @@ class Event < ApplicationRecord include Gitlab::Utils::StrongMemoize include UsageStatistics include ShaAttribute - include IgnorableColumns - - ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' default_scope { reorder(nil) } # rubocop:disable Cop/DefaultScope diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb index 0c36e51120f..2775b520b2f 100644 --- a/app/models/gpg_signature.rb +++ b/app/models/gpg_signature.rb @@ -12,7 +12,8 @@ class GpgSignature < ApplicationRecord same_user_different_email: 2, other_user: 3, unverified_key: 4, - unknown_key: 5 + unknown_key: 5, + multiple_signatures: 6 } belongs_to :project diff --git a/app/models/group.rb b/app/models/group.rb index c5e119451e3..2dd20300ad2 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -56,6 +56,9 @@ class Group < Namespace has_many :boards has_many :badges, class_name: 'GroupBadge' + has_many :organizations, class_name: 'CustomerRelations::Organization', inverse_of: :group + has_many :contacts, class_name: 'CustomerRelations::Contact', inverse_of: :group + has_many :cluster_groups, class_name: 'Clusters::Group' has_many :clusters, through: :cluster_groups, class_name: 'Clusters::Cluster' @@ -194,13 +197,8 @@ class Group < Namespace def ids_with_disabled_email(groups) inner_groups = Group.where('id = namespaces_with_emails_disabled.id') - inner_ancestors = if Feature.enabled?(:linear_group_ancestor_scopes, default_enabled: :yaml) - inner_groups.self_and_ancestors - else - Gitlab::ObjectHierarchy.new(inner_groups).base_and_ancestors - end - - inner_query = inner_ancestors + inner_query = inner_groups + .self_and_ancestors .where(emails_disabled: true) .select('1') .limit(1) @@ -317,13 +315,15 @@ class Group < Namespace owners.include?(user) end - def add_users(users, access_level, current_user: nil, expires_at: nil) + def add_users(users, access_level, current_user: nil, expires_at: nil, tasks_to_be_done: [], tasks_project_id: nil) Members::Groups::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass self, users, access_level, current_user: current_user, - expires_at: expires_at + expires_at: expires_at, + tasks_to_be_done: tasks_to_be_done, + tasks_project_id: tasks_project_id ) end @@ -760,18 +760,6 @@ class Group < Namespace Timelog.in_group(self) end - def cached_issues_state_count_enabled? - Feature.enabled?(:cached_issues_state_count, self, default_enabled: :yaml) - end - - def organizations - ::CustomerRelations::Organization.where(group_id: self.id) - end - - def contacts - ::CustomerRelations::Contact.where(group_id: self.id) - end - def dependency_proxy_image_ttl_policy super || build_dependency_proxy_image_ttl_policy end diff --git a/app/models/integration.rb b/app/models/integration.rb index 158764bb783..d3059fa6d4a 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -14,7 +14,7 @@ class Integration < ApplicationRecord asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email - pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack + pivotaltracker prometheus pushover redmine shimo slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao ].freeze PROJECT_SPECIFIC_INTEGRATION_NAMES = %w[ @@ -373,7 +373,7 @@ class Integration < ApplicationRecord end def to_data_fields_hash - data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id') + data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id', 'integration_id') end def event_channel_names diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index 1a7cbaa34c7..0774b84b69f 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -3,7 +3,7 @@ module Integrations class Bamboo < BaseCi include ActionView::Helpers::UrlHelper - include ReactiveService + include ReactivelyCached prop_accessor :bamboo_url, :build_key, :username, :password diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index c6335782b5e..ca72de47d30 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -73,7 +73,12 @@ module Integrations { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}", required: true }.freeze, { type: 'text', name: 'username', placeholder: 'GitLab-integration' }.freeze, { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'Do not send notifications for successful pipelines.' }.freeze, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices }.freeze, + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + }.freeze, { type: 'text', name: 'labels_to_be_notified', diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb index 94a37f0c4f2..9fad3a42647 100644 --- a/app/models/integrations/buildkite.rb +++ b/app/models/integrations/buildkite.rb @@ -5,7 +5,7 @@ require "addressable/uri" module Integrations class Buildkite < BaseCi include HasWebHook - include ReactiveService + include ReactivelyCached extend Gitlab::Utils::Override ENDPOINT = "https://buildkite.com" diff --git a/app/models/integrations/discord.rb b/app/models/integrations/discord.rb index 76160a61bc3..21993dd3c43 100644 --- a/app/models/integrations/discord.rb +++ b/app/models/integrations/discord.rb @@ -41,7 +41,12 @@ module Integrations [ { type: "text", name: "webhook", placeholder: "https://discordapp.com/api/webhooks/…", help: "URL to the webhook for the Discord channel." }, { type: "checkbox", name: "notify_only_broken_pipelines" }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + } ] end diff --git a/app/models/integrations/drone_ci.rb b/app/models/integrations/drone_ci.rb index c93ae432fe9..856d14c022d 100644 --- a/app/models/integrations/drone_ci.rb +++ b/app/models/integrations/drone_ci.rb @@ -3,8 +3,8 @@ module Integrations class DroneCi < BaseCi include HasWebHook - include ReactiveService - include ServicePushDataValidations + include PushDataValidations + include ReactivelyCached extend Gitlab::Utils::Override prop_accessor :drone_url, :token diff --git a/app/models/integrations/emails_on_push.rb b/app/models/integrations/emails_on_push.rb index e277633664f..a9cd67550dc 100644 --- a/app/models/integrations/emails_on_push.rb +++ b/app/models/integrations/emails_on_push.rb @@ -76,7 +76,12 @@ module Integrations help: s_("EmailsOnPushService|Send notifications from the committer's email address if the domain matches the domain used by your GitLab instance (such as %{domains}).") % { domains: domains } }, { type: 'checkbox', name: 'disable_diffs', title: s_("EmailsOnPushService|Disable code diffs"), help: s_("EmailsOnPushService|Don't include possibly sensitive code diffs in notification body.") }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices }, + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + }, { type: 'textarea', name: 'recipients', @@ -92,7 +97,7 @@ module Integrations return if recipients.blank? if self.class.valid_recipients(recipients).size > RECIPIENTS_LIMIT - errors.add(:recipients, s_("EmailsOnPushService|can't exceed %{recipients_limit}") % { recipients_limit: RECIPIENTS_LIMIT }) + errors.add(:recipients, s_("Integrations|can't exceed %{recipients_limit}") % { recipients_limit: RECIPIENTS_LIMIT }) end end end diff --git a/app/models/integrations/hangouts_chat.rb b/app/models/integrations/hangouts_chat.rb index d02cfe4ec56..0d6b9fb1019 100644 --- a/app/models/integrations/hangouts_chat.rb +++ b/app/models/integrations/hangouts_chat.rb @@ -40,7 +40,12 @@ module Integrations [ { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + } ] end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index ec6adc87bf4..42c291abf55 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -89,7 +89,6 @@ module Integrations site: URI.join(url, '/').to_s.delete_suffix('/'), # Intended to find the root context_path: (url.path.presence || '/').delete_suffix('/'), auth_type: :basic, - read_timeout: 120, use_cookies: true, additional_cookies: ['OBBasicAuth=fromDialog'], use_ssl: url.scheme == 'https' @@ -303,6 +302,14 @@ module Integrations private + def branch_name(noteable) + if Feature.enabled?(:jira_use_first_ref_by_oid, project, default_enabled: :yaml) + noteable.first_ref_by_oid(project.repository) + else + noteable.ref_names(project.repository).first + end + end + def server_info strong_memoize(:server_info) do client_url.present? ? jira_request { client.ServerInfo.all.attrs } : nil @@ -496,7 +503,7 @@ module Integrations { id: noteable.short_id, description: noteable.safe_message, - branch: noteable.ref_names(project.repository).first + branch: branch_name(noteable) } elsif noteable.is_a?(MergeRequest) { @@ -521,7 +528,9 @@ module Integrations yield rescue StandardError => error @error = error - log_error("Error sending message", client_url: client_url, error: @error.message) + payload = { client_url: client_url } + Gitlab::ExceptionLogFormatter.format!(error, payload) + log_error("Error sending message", payload) nil end diff --git a/app/models/integrations/microsoft_teams.rb b/app/models/integrations/microsoft_teams.rb index 5aad25e8ddc..71cd4ddaf82 100644 --- a/app/models/integrations/microsoft_teams.rb +++ b/app/models/integrations/microsoft_teams.rb @@ -37,7 +37,12 @@ module Integrations [ { type: 'text', name: 'webhook', placeholder: "#{webhook_placeholder}" }, { type: 'checkbox', name: 'notify_only_broken_pipelines', help: 'If selected, successful pipelines do not trigger a notification event.' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + } ] end diff --git a/app/models/integrations/pipelines_email.rb b/app/models/integrations/pipelines_email.rb index efba35cc2a8..6dc41958daa 100644 --- a/app/models/integrations/pipelines_email.rb +++ b/app/models/integrations/pipelines_email.rb @@ -4,9 +4,12 @@ module Integrations class PipelinesEmail < Integration include NotificationBranchSelection + RECIPIENTS_LIMIT = 30 + prop_accessor :recipients, :branches_to_be_notified boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch validates :recipients, presence: true, if: :validate_recipients? + validate :number_of_recipients_within_limit, if: :validate_recipients? def initialize_properties if properties.nil? @@ -49,7 +52,7 @@ module Integrations return unless supported_events.include?(data[:object_kind]) return unless force || should_pipeline_be_notified?(data) - all_recipients = retrieve_recipients(data) + all_recipients = retrieve_recipients return unless all_recipients.any? @@ -71,6 +74,7 @@ module Integrations name: 'notify_only_broken_pipelines' }, { type: 'select', name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), choices: branch_choices } ] end @@ -98,8 +102,18 @@ module Integrations end end - def retrieve_recipients(data) + def retrieve_recipients recipients.to_s.split(/[,\r\n ]+/).reject(&:empty?) end + + private + + def number_of_recipients_within_limit + return if recipients.blank? + + if retrieve_recipients.size > RECIPIENTS_LIMIT + errors.add(:recipients, s_("Integrations|can't exceed %{recipients_limit}") % { recipients_limit: RECIPIENTS_LIMIT }) + end + end end end diff --git a/app/models/integrations/shimo.rb b/app/models/integrations/shimo.rb new file mode 100644 index 00000000000..4f42fda2577 --- /dev/null +++ b/app/models/integrations/shimo.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Integrations + class Shimo < Integration + prop_accessor :external_wiki_url + validates :external_wiki_url, presence: true, public_url: true, if: :activated? + + def render? + valid? && activated? + end + + def title + s_('Shimo|Shimo') + end + + def description + s_('Shimo|Link to a Shimo Workspace from the sidebar.') + end + + def self.to_param + 'shimo' + end + + # support for `test` method + def execute(_data) + response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true, use_read_total_timeout: true) + response.body if response.code == 200 + rescue StandardError + nil + end + + def self.supported_events + %w() + end + + def fields + [ + { + type: 'text', + name: 'external_wiki_url', + title: s_('Shimo|Shimo Workspace URL'), + required: true + } + ] + end + end +end diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb index 3f868b57597..008b591c304 100644 --- a/app/models/integrations/teamcity.rb +++ b/app/models/integrations/teamcity.rb @@ -2,8 +2,8 @@ module Integrations class Teamcity < BaseCi - include ReactiveService - include ServicePushDataValidations + include PushDataValidations + include ReactivelyCached prop_accessor :teamcity_url, :build_type, :username, :password diff --git a/app/models/integrations/unify_circuit.rb b/app/models/integrations/unify_circuit.rb index e3e180ae959..f085423d229 100644 --- a/app/models/integrations/unify_circuit.rb +++ b/app/models/integrations/unify_circuit.rb @@ -34,7 +34,12 @@ module Integrations [ { type: 'text', name: 'webhook', placeholder: "https://yourcircuit.com/rest/v2/webhooks/incoming/…", required: true }, { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + } ] end diff --git a/app/models/integrations/webex_teams.rb b/app/models/integrations/webex_teams.rb index 6fd82a32035..7660eda6f83 100644 --- a/app/models/integrations/webex_teams.rb +++ b/app/models/integrations/webex_teams.rb @@ -36,7 +36,12 @@ module Integrations [ { type: 'text', name: 'webhook', placeholder: "https://api.ciscospark.com/v1/webhooks/incoming/...", required: true }, { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { type: 'select', name: 'branches_to_be_notified', choices: branch_choices } + { + type: 'select', + name: 'branches_to_be_notified', + title: s_('Integrations|Branches for which notifications are to be sent'), + choices: branch_choices + } ] end diff --git a/app/models/integrations/zentao.rb b/app/models/integrations/zentao.rb index 68c02f54c61..493d42cc40b 100644 --- a/app/models/integrations/zentao.rb +++ b/app/models/integrations/zentao.rb @@ -2,6 +2,8 @@ module Integrations class Zentao < Integration + include Gitlab::Routing + data_field :url, :api_url, :api_token, :zentao_product_xid validates :url, public_url: true, presence: true, if: :activated? @@ -9,16 +11,29 @@ module Integrations validates :api_token, presence: true, if: :activated? validates :zentao_product_xid, presence: true, if: :activated? + # License Level: EEP_FEATURES + def self.issues_license_available?(project) + project&.licensed_feature_available?(:zentao_issues_integration) + end + def data_fields zentao_tracker_data || self.build_zentao_tracker_data end def title - self.class.name.demodulize + 'ZenTao' end def description - s_("ZentaoIntegration|Use Zentao as this project's issue tracker.") + s_("ZentaoIntegration|Use ZenTao as this project's issue tracker.") + end + + def help + s_("ZentaoIntegration|Before you enable this integration, you must configure ZenTao. For more details, read the %{link_start}ZenTao integration documentation%{link_end}.") % { + link_start: '<a href="%{url}" target="_blank" rel="noopener noreferrer">' + .html_safe % { url: help_page_url('user/project/integrations/zentao') }, + link_end: '</a>'.html_safe + } end def self.to_param @@ -42,28 +57,29 @@ module Integrations { type: 'text', name: 'url', - title: s_('ZentaoIntegration|Zentao Web URL'), + title: s_('ZentaoIntegration|ZenTao Web URL'), placeholder: 'https://www.zentao.net', - help: s_('ZentaoIntegration|Base URL of the Zentao instance.'), + help: s_('ZentaoIntegration|Base URL of the ZenTao instance.'), required: true }, { type: 'text', name: 'api_url', - title: s_('ZentaoIntegration|Zentao API URL (optional)'), + title: s_('ZentaoIntegration|ZenTao API URL (optional)'), help: s_('ZentaoIntegration|If different from Web URL.') }, { type: 'password', name: 'api_token', - title: s_('ZentaoIntegration|Zentao API token'), - non_empty_password_title: s_('ZentaoIntegration|Enter API token'), + title: s_('ZentaoIntegration|ZenTao API token'), + non_empty_password_title: s_('ZentaoIntegration|Enter new ZenTao API token'), + non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), required: true }, { type: 'text', name: 'zentao_product_xid', - title: s_('ZentaoIntegration|Zentao Product ID'), + title: s_('ZentaoIntegration|ZenTao Product ID'), required: true } ] @@ -76,3 +92,5 @@ module Integrations end end end + +::Integrations::Zentao.prepend_mod diff --git a/app/models/issue.rb b/app/models/issue.rb index 9c568414ec2..47dc084d69c 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -81,7 +81,8 @@ class Issue < ApplicationRecord has_and_belongs_to_many :self_managed_prometheus_alert_events, join_table: :issues_self_managed_prometheus_alert_events # rubocop: disable Rails/HasAndBelongsToMany has_and_belongs_to_many :prometheus_alert_events, join_table: :issues_prometheus_alert_events # rubocop: disable Rails/HasAndBelongsToMany has_many :prometheus_alerts, through: :prometheus_alert_events - has_and_belongs_to_many :customer_relations_contacts, join_table: :issue_customer_relations_contacts, class_name: 'CustomerRelations::Contact' # rubocop: disable Rails/HasAndBelongsToMany + has_many :issue_customer_relations_contacts, class_name: 'CustomerRelations::IssueContact', inverse_of: :issue + has_many :customer_relations_contacts, through: :issue_customer_relations_contacts, source: :contact, class_name: 'CustomerRelations::Contact', inverse_of: :issues accepts_nested_attributes_for :issuable_severity, update_only: true accepts_nested_attributes_for :sentry_issue @@ -203,6 +204,8 @@ class Issue < ApplicationRecord before_transition closed: :opened do |issue| issue.closed_at = nil issue.closed_by = nil + + issue.clear_closure_reason_references end end @@ -378,6 +381,11 @@ class Issue < ApplicationRecord !duplicated_to_id.nil? end + def clear_closure_reason_references + self.moved_to_id = nil + self.duplicated_to_id = nil + end + def can_move?(user, to_project = nil) if to_project return false unless user.can?(:admin_issue, to_project) diff --git a/app/models/key.rb b/app/models/key.rb index 64385953865..a478434538c 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -46,7 +46,7 @@ class Key < ApplicationRecord scope :order_last_used_at_desc, -> { reorder(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } # Date is set specifically in this scope to improve query time. - scope :expired_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') BETWEEN '2000-01-01' AND CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } + scope :expired_today_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') = CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } scope :expiring_soon_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') > CURRENT_DATE AND date(expires_at AT TIME ZONE 'UTC') < ? AND before_expiry_notification_delivered_at IS NULL", DAYS_TO_EXPIRE.days.from_now.to_date]) } def self.regular_keys diff --git a/app/models/label_link.rb b/app/models/label_link.rb index 4fb5fd8c58a..d326b07ad31 100644 --- a/app/models/label_link.rb +++ b/app/models/label_link.rb @@ -11,4 +11,16 @@ class LabelLink < ApplicationRecord validates :label, presence: true, unless: :importing? scope :for_target, -> (target_id, target_type) { where(target_id: target_id, target_type: target_type) } + + # Example: Issues has at least one label within a project + # > Issue.where(project_id: 100) # or any scope on issues + # > .where(LabelLink.by_target_for_exists_query('Issue', Issue.arel_table[:id]).arel.exists) + scope :by_target_for_exists_query, -> (target_type, arel_join_column, label_ids = nil) do + relation = LabelLink + .where(target_type: target_type) + .where(arel_table['target_id'].eq(arel_join_column)) + + relation = relation.where(label_id: label_ids) if label_ids + relation + end end diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index ca5a2800a03..c3b3e76f67b 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -1,5 +1,32 @@ # frozen_string_literal: true class LooseForeignKeys::DeletedRecord < ApplicationRecord - extend SuppressCompositePrimaryKeyWarning + self.primary_key = :id + + scope :for_table, -> (table) { where(fully_qualified_table_name: table) } + scope :consume_order, -> { order(:partition, :consume_after, :id) } + + enum status: { pending: 1, processed: 2 }, _prefix: :status + + def self.load_batch_for_table(table, batch_size) + for_table(table) + .status_pending + .consume_order + .limit(batch_size) + .to_a + end + + def self.mark_records_processed(all_records) + # Run a query for each partition to optimize the row lookup by primary key (partition, id) + update_count = 0 + + all_records.group_by(&:partition).each do |partition, records_within_partition| + update_count += status_pending + .where(partition: partition) + .where(id: records_within_partition.pluck(:id)) + .update_all(status: :processed) + end + + update_count + end end diff --git a/app/models/loose_foreign_keys/modification_tracker.rb b/app/models/loose_foreign_keys/modification_tracker.rb new file mode 100644 index 00000000000..6eb04608cd9 --- /dev/null +++ b/app/models/loose_foreign_keys/modification_tracker.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module LooseForeignKeys + class ModificationTracker + MAX_DELETES = 100_000 + MAX_UPDATES = 50_000 + MAX_RUNTIME = 3.minutes + + delegate :monotonic_time, to: :'Gitlab::Metrics::System' + + def initialize + @delete_count_by_table = Hash.new { |h, k| h[k] = 0 } + @update_count_by_table = Hash.new { |h, k| h[k] = 0 } + @start_time = monotonic_time + @deletes_counter = Gitlab::Metrics.counter( + :loose_foreign_key_deletions, + 'The number of loose foreign key deletions' + ) + @updates_counter = Gitlab::Metrics.counter( + :loose_foreign_key_updates, + 'The number of loose foreign key updates' + ) + end + + def add_deletions(table, count) + @delete_count_by_table[table] += count + @deletes_counter.increment({ table: table }, count) + end + + def add_updates(table, count) + @update_count_by_table[table] += count + @updates_counter.increment({ table: table }, count) + end + + def over_limit? + @delete_count_by_table.values.sum >= MAX_DELETES || + @update_count_by_table.values.sum >= MAX_UPDATES || + monotonic_time - @start_time >= MAX_RUNTIME + end + + def stats + { + over_limit: over_limit?, + delete_count_by_table: @delete_count_by_table, + update_count_by_table: @update_count_by_table, + delete_count: @delete_count_by_table.values.sum, + update_count: @update_count_by_table.values.sum + } + end + end +end diff --git a/app/models/member.rb b/app/models/member.rb index 21fd4aebd7b..11f67a77ee2 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -13,6 +13,7 @@ class Member < ApplicationRecord include FromUnion include UpdateHighestRole include RestrictedSignup + include Gitlab::Experiment::Dsl AVATAR_SIZE = 40 ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 @@ -22,8 +23,10 @@ class Member < ApplicationRecord belongs_to :created_by, class_name: "User" belongs_to :user belongs_to :source, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + has_one :member_task delegate :name, :username, :email, to: :user, prefix: true + delegate :tasks_to_be_done, to: :member_task, allow_nil: true validates :expires_at, allow_blank: true, future_date: true validates :user, presence: true, unless: :invite? @@ -413,6 +416,14 @@ class Member < ApplicationRecord def after_accept_invite post_create_hook + + if experiment(:invite_members_for_task).enabled? + run_after_commit_or_now do + if member_task + TasksToBeDone::CreateWorker.perform_async(member_task.id, created_by_id, [user_id.to_i]) + end + end + end end def after_decline_invite diff --git a/app/models/members/member_task.rb b/app/models/members/member_task.rb new file mode 100644 index 00000000000..f093619ff36 --- /dev/null +++ b/app/models/members/member_task.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class MemberTask < ApplicationRecord + TASKS = { + code: 0, + ci: 1, + issues: 2 + }.freeze + + belongs_to :member + belongs_to :project + + validates :member, :project, presence: true + validates :tasks, inclusion: { in: TASKS.values } + validate :tasks_uniqueness + validate :project_in_member_source + + scope :for_members, -> (members) { joins(:member).where(member: members) } + + def tasks_to_be_done + Array(self[:tasks]).map { |task| TASKS.key(task) } + end + + def tasks_to_be_done=(tasks) + self[:tasks] = Array(tasks).map do |task| + TASKS[task.to_sym] + end.uniq + end + + private + + def tasks_uniqueness + errors.add(:tasks, 'are not unique') unless Array(tasks).length == Array(tasks).uniq.length + end + + def project_in_member_source + if member.is_a?(GroupMember) + errors.add(:project, _('is not in the member group')) unless project.namespace == member.source + elsif member.is_a?(ProjectMember) + errors.add(:project, _('is not the member project')) unless project == member.source + end + end +end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index eec46b3493e..89b72508e84 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -94,7 +94,6 @@ class ProjectMember < Member override :refresh_member_authorized_projects def refresh_member_authorized_projects(blocking:) - return super unless Feature.enabled?(:specialized_service_for_project_member_auth_refresh) return unless user # rubocop:disable CodeReuse/ServiceClass diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 15862fb2bfa..0cd8f12088c 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -268,7 +268,6 @@ class MergeRequest < ApplicationRecord from_fork.where('source_project_id = ? OR target_project_id = ?', project.id, project.id) end scope :merged, -> { with_state(:merged) } - scope :closed_and_merged, -> { with_states(:closed, :merged) } scope :open_and_closed, -> { with_states(:opened, :closed) } scope :drafts, -> { where(draft: true) } scope :from_source_branches, ->(branches) { where(source_branch: branches) } @@ -663,7 +662,7 @@ class MergeRequest < ApplicationRecord # updates `merge_jid` with the MergeWorker#jid. # This helps tracking enqueued and ongoing merge jobs. def merge_async(user_id, params) - jid = MergeWorker.perform_async(id, user_id, params.to_h) + jid = MergeWorker.with_status.perform_async(id, user_id, params.to_h) update_column(:merge_jid, jid) # merge_ongoing? depends on merge_jid @@ -682,7 +681,7 @@ class MergeRequest < ApplicationRecord # attribute is set *and* that the sidekiq job is still running. So a JID # for a completed RebaseWorker is equivalent to a nil JID. jid = Sidekiq::Worker.skipping_transaction_check do - RebaseWorker.perform_async(id, user_id, skip_ci) + RebaseWorker.with_status.perform_async(id, user_id, skip_ci) end update_column(:rebase_jid, jid) @@ -1317,6 +1316,10 @@ class MergeRequest < ApplicationRecord end def default_merge_commit_message(include_description: false) + if self.target_project.merge_commit_template.present? && !include_description + return ::Gitlab::MergeRequests::MergeCommitMessage.new(merge_request: self).message + end + closes_issues_references = visible_closing_issues_for.map do |issue| issue.to_reference(target_project) end @@ -1409,7 +1412,15 @@ class MergeRequest < ApplicationRecord def environments return Environment.none unless actual_head_pipeline&.merge_request? - actual_head_pipeline.environments + build_for_actual_head_pipeline = Ci::Build.latest.where(pipeline: actual_head_pipeline) + + environments = build_for_actual_head_pipeline.joins(:metadata) + .where.not('ci_builds_metadata.expanded_environment_name' => nil) + .distinct('ci_builds_metadata.expanded_environment_name') + .limit(100) + .pluck(:expanded_environment_name) + + Environment.where(project: project, name: environments) end def fetch_ref! @@ -1907,6 +1918,10 @@ class MergeRequest < ApplicationRecord true end + def find_assignee(user) + merge_request_assignees.find_by(user_id: user.id) + end + def find_reviewer(user) merge_request_reviewers.find_by(user_id: user.id) end @@ -1930,6 +1945,10 @@ class MergeRequest < ApplicationRecord end end + def attention_requested_enabled? + Feature.enabled?(:mr_attention_requests, project, default_enabled: :yaml) + end + private def set_draft_status diff --git a/app/models/merge_request_assignee.rb b/app/models/merge_request_assignee.rb index 86bf950ae19..fd8e5860040 100644 --- a/app/models/merge_request_assignee.rb +++ b/app/models/merge_request_assignee.rb @@ -1,10 +1,16 @@ # frozen_string_literal: true class MergeRequestAssignee < ApplicationRecord + include MergeRequestReviewerState + belongs_to :merge_request, touch: true belongs_to :assignee, class_name: "User", foreign_key: :user_id, inverse_of: :merge_request_assignees validates :assignee, uniqueness: { scope: :merge_request_id } scope :in_projects, ->(project_ids) { joins(:merge_request).where(merge_requests: { target_project_id: project_ids }) } + + def cache_key + [model_name.cache_key, id, state, assignee.cache_key] + end end diff --git a/app/models/merge_request_context_commit.rb b/app/models/merge_request_context_commit.rb index 09824ed4468..ebbdecf8aa7 100644 --- a/app/models/merge_request_context_commit.rb +++ b/app/models/merge_request_context_commit.rb @@ -26,7 +26,7 @@ class MergeRequestContextCommit < ApplicationRecord # create MergeRequestContextCommit by given commit sha and it's diff file record def self.bulk_insert(rows, **args) - Gitlab::Database.main.bulk_insert('merge_request_context_commits', rows, **args) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_context_commits', rows, **args) # rubocop:disable Gitlab/BulkInsert end def to_commit diff --git a/app/models/merge_request_context_commit_diff_file.rb b/app/models/merge_request_context_commit_diff_file.rb index b9efebe3af2..fdf57068928 100644 --- a/app/models/merge_request_context_commit_diff_file.rb +++ b/app/models/merge_request_context_commit_diff_file.rb @@ -14,7 +14,7 @@ class MergeRequestContextCommitDiffFile < ApplicationRecord # create MergeRequestContextCommitDiffFile by given diff file record(s) def self.bulk_insert(*args) - Gitlab::Database.main.bulk_insert('merge_request_context_commit_diff_files', *args) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_context_commit_diff_files', *args) # rubocop:disable Gitlab/BulkInsert end def path diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index bd94c0ad30e..2516ff05bda 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -515,7 +515,7 @@ class MergeRequestDiff < ApplicationRecord transaction do MergeRequestDiffFile.where(merge_request_diff_id: id).delete_all - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert save! end @@ -535,7 +535,7 @@ class MergeRequestDiff < ApplicationRecord transaction do MergeRequestDiffFile.where(merge_request_diff_id: id).delete_all - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert update!(stored_externally: false) end @@ -595,7 +595,7 @@ class MergeRequestDiff < ApplicationRecord rows = build_external_merge_request_diff_files(rows) if use_external_diff? # Faster inserts - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert end def build_external_diff_tempfile(rows) @@ -710,7 +710,7 @@ class MergeRequestDiff < ApplicationRecord end CommitCollection - .new(merge_request.source_project, commits, merge_request.source_branch) + .new(merge_request.target_project, commits, merge_request.target_branch) end def save_diffs diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index d9a1784cdda..66f1e45fd49 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -6,6 +6,12 @@ class MergeRequestDiffCommit < ApplicationRecord include BulkInsertSafe include ShaAttribute include CachedCommit + include IgnorableColumns + include FromUnion + + ignore_column %i[author_name author_email committer_name committer_email], + remove_with: '14.6', + remove_after: '2021-11-22' belongs_to :merge_request_diff @@ -51,9 +57,14 @@ class MergeRequestDiffCommit < ApplicationRecord committer = users[[commit_hash[:committer_name], commit_hash[:committer_email]]] + # These fields are only used to determine the author/committer IDs, we + # don't store them in the DB. + commit_hash = commit_hash + .except(:author_name, :author_email, :committer_name, :committer_email) + commit_hash.merge( - commit_author_id: author&.id, - committer_id: committer&.id, + commit_author_id: author.id, + committer_id: committer.id, merge_request_diff_id: merge_request_diff_id, relative_order: index, sha: Gitlab::Database::ShaAttribute.serialize(sha), # rubocop:disable Cop/ActiveRecordSerialize @@ -63,7 +74,7 @@ class MergeRequestDiffCommit < ApplicationRecord ) end - Gitlab::Database.main.bulk_insert(self.table_name, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(self.table_name, rows) # rubocop:disable Gitlab/BulkInsert end def self.prepare_commits_for_bulk_insert(commits) @@ -104,18 +115,18 @@ class MergeRequestDiffCommit < ApplicationRecord end def author_name - commit_author_id ? commit_author.name : super + commit_author&.name end def author_email - commit_author_id ? commit_author.email : super + commit_author&.email end def committer_name - committer_id ? committer.name : super + committer&.name end def committer_email - committer_id ? committer.email : super + committer&.email end end diff --git a/app/models/merge_request_reviewer.rb b/app/models/merge_request_reviewer.rb index 4a1f31a7f39..4abf0fa09f0 100644 --- a/app/models/merge_request_reviewer.rb +++ b/app/models/merge_request_reviewer.rb @@ -1,15 +1,12 @@ # frozen_string_literal: true class MergeRequestReviewer < ApplicationRecord - enum state: { - unreviewed: 0, - reviewed: 1 - } - - validates :state, - presence: true, - inclusion: { in: MergeRequestReviewer.states.keys } + include MergeRequestReviewerState belongs_to :merge_request belongs_to :reviewer, class_name: 'User', foreign_key: :user_id, inverse_of: :merge_request_reviewers + + def cache_key + [model_name.cache_key, id, state, reviewer.cache_key] + end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 07f9bb99952..353a896b3fe 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -16,9 +16,11 @@ class Namespace < ApplicationRecord include Namespaces::Traversal::Linear include EachBatch - ignore_column :delayed_project_removal, remove_with: '14.1', remove_after: '2021-05-22' + # Temporary column used for back-filling project namespaces. + # Remove it once the back-filling of all project namespaces is done. + ignore_column :tmp_project_id, remove_with: '14.7', remove_after: '2022-01-22' - # Tells ActiveRecord not to store the full class name, in order to space some space + # Tells ActiveRecord not to store the full class name, in order to save some space # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69794 self.store_full_sti_class = false self.store_full_class_name = false @@ -54,7 +56,7 @@ class Namespace < ApplicationRecord belongs_to :owner, class_name: "User" belongs_to :parent, class_name: "Namespace" - has_many :children, class_name: "Namespace", foreign_key: :parent_id + has_many :children, -> { where(type: Group.sti_name) }, class_name: "Namespace", foreign_key: :parent_id has_many :custom_emoji, inverse_of: :namespace has_one :chat_team, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :root_storage_statistics, class_name: 'Namespace::RootStorageStatistics' @@ -95,9 +97,11 @@ class Namespace < ApplicationRecord validates :max_artifacts_size, numericality: { only_integer: true, greater_than: 0, allow_nil: true } validate :validate_parent_type, if: -> { Feature.enabled?(:validate_namespace_parent_type, default_enabled: :yaml) } - validate :nesting_level_allowed - validate :changing_shared_runners_enabled_is_allowed - validate :changing_allow_descendants_override_disabled_shared_runners_is_allowed + + # ProjectNamespaces excluded as they are not meant to appear in the group hierarchy at the moment. + validate :nesting_level_allowed, unless: -> { project_namespace? } + validate :changing_shared_runners_enabled_is_allowed, unless: -> { project_namespace? } + validate :changing_allow_descendants_override_disabled_shared_runners_is_allowed, unless: -> { project_namespace? } delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true @@ -123,7 +127,7 @@ class Namespace < ApplicationRecord scope :user_namespaces, -> { where(type: [nil, Namespaces::UserNamespace.sti_name]) } # TODO: this can be simplified with `type != 'Project'` when working on issue # https://gitlab.com/gitlab-org/gitlab/-/issues/341070 - scope :without_project_namespaces, -> { where("type IS DISTINCT FROM ?", Namespaces::ProjectNamespace.sti_name) } + scope :without_project_namespaces, -> { where(Namespace.arel_table[:type].is_distinct_from(Namespaces::ProjectNamespace.sti_name)) } scope :sort_by_type, -> { order(Gitlab::Database.nulls_first_order(:type)) } scope :include_route, -> { includes(:route) } scope :by_parent, -> (parent) { where(parent_id: parent) } @@ -140,6 +144,7 @@ class Namespace < ApplicationRecord 'COALESCE(SUM(ps.snippets_size), 0) AS snippets_size', 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', + 'COALESCE(SUM(ps.pipeline_artifacts_size), 0) AS pipeline_artifacts_size', 'COALESCE(SUM(ps.packages_size), 0) AS packages_size', 'COALESCE(SUM(ps.uploads_size), 0) AS uploads_size' ) @@ -189,9 +194,9 @@ class Namespace < ApplicationRecord # Returns an ActiveRecord::Relation. def search(query, include_parents: false) if include_parents - where(id: Route.for_routable_type(Namespace.name).fuzzy_search(query, [Route.arel_table[:path], Route.arel_table[:name]]).select(:source_id)) + without_project_namespaces.where(id: Route.for_routable_type(Namespace.name).fuzzy_search(query, [Route.arel_table[:path], Route.arel_table[:name]]).select(:source_id)) else - fuzzy_search(query, [:path, :name]) + without_project_namespaces.fuzzy_search(query, [:path, :name]) end end @@ -494,6 +499,10 @@ class Namespace < ApplicationRecord Feature.enabled?(:block_issue_repositioning, self, type: :ops, default_enabled: :yaml) end + def project_namespace_creation_enabled? + Feature.enabled?(:create_project_namespace_on_project_create, self, default_enabled: :yaml) + end + private def expire_child_caches @@ -535,21 +544,23 @@ class Namespace < ApplicationRecord # Until we compare the inconsistency rates of the new specialized worker and # the old approach, we still run AuthorizedProjectsWorker # but with some delay and lower urgency as a safety net. - Group - .joins(project_group_links: :project) - .where(projects: { namespace_id: id }) - .distinct - .find_each do |group| - group.refresh_members_authorized_projects( - blocking: false, - priority: UserProjectAccessChangedService::LOW_PRIORITY - ) - end + enqueue_jobs_for_groups_requiring_authorizations_refresh(priority: UserProjectAccessChangedService::LOW_PRIORITY) else - Group - .joins(project_group_links: :project) - .where(projects: { namespace_id: id }) - .find_each(&:refresh_members_authorized_projects) + enqueue_jobs_for_groups_requiring_authorizations_refresh(priority: UserProjectAccessChangedService::HIGH_PRIORITY) + end + end + + def enqueue_jobs_for_groups_requiring_authorizations_refresh(priority:) + groups_requiring_authorizations_refresh = Group + .joins(project_group_links: :project) + .where(projects: { namespace_id: id }) + .distinct + + groups_requiring_authorizations_refresh.find_each do |group| + group.refresh_members_authorized_projects( + blocking: false, + priority: priority + ) end end @@ -573,7 +584,7 @@ class Namespace < ApplicationRecord end if user_namespace? - errors.add(:parent_id, _('cannot not be used for user namespace')) + errors.add(:parent_id, _('cannot be used for user namespace')) elsif group_namespace? errors.add(:parent_id, _('user namespace cannot be the parent of another namespace')) if parent.user_namespace? end diff --git a/app/models/namespaces/project_namespace.rb b/app/models/namespaces/project_namespace.rb index d1806c1c088..22ec550dee2 100644 --- a/app/models/namespaces/project_namespace.rb +++ b/app/models/namespaces/project_namespace.rb @@ -4,8 +4,6 @@ module Namespaces class ProjectNamespace < Namespace has_one :project, foreign_key: :project_namespace_id, inverse_of: :project_namespace - validates :project, presence: true - def self.sti_name 'Project' end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index d7130322ed1..1736fe82ca5 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -161,7 +161,7 @@ module Namespaces def lineage(top: nil, bottom: nil, hierarchy_order: nil) raise UnboundedSearch, 'Must bound search by either top or bottom' unless top || bottom - skope = self.class.without_sti_condition + skope = self.class if top skope = skope.where("traversal_ids @> ('{?}')", top.id) @@ -181,7 +181,6 @@ module Namespaces # standard SELECT to avoid mismatched attribute errors when trying to # chain future ActiveRelation commands, and retain the ordering. skope = self.class - .without_sti_condition .from(skope, self.class.table_name) .select(skope.arel_table[Arel.star]) .order(depth: hierarchy_order) diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 2da0e48c2da..f5c44171c42 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -15,12 +15,18 @@ module Namespaces select('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id') end + def roots + return super unless use_traversal_ids_roots? + + root_ids = all.select("#{quoted_table_name}.traversal_ids[1]").distinct + unscoped.where(id: root_ids) + end + def self_and_ancestors(include_self: true, hierarchy_order: nil) return super unless use_traversal_ids_for_ancestor_scopes? records = unscoped - .without_sti_condition - .where(id: without_sti_condition.select('unnest(traversal_ids)')) + .where(id: select('unnest(traversal_ids)')) .order_by_depth(hierarchy_order) .normal_select @@ -40,24 +46,24 @@ module Namespaces def self_and_descendants(include_self: true) return super unless use_traversal_ids? - records = self_and_descendants_with_duplicates(include_self: include_self) - - distinct = records.select('DISTINCT on(namespaces.id) namespaces.*') - - distinct.normal_select + if Feature.enabled?(:traversal_ids_btree, default_enabled: :yaml) + self_and_descendants_with_comparison_operators(include_self: include_self) + else + records = self_and_descendants_with_duplicates_with_array_operator(include_self: include_self) + distinct = records.select('DISTINCT on(namespaces.id) namespaces.*') + distinct.normal_select + end end def self_and_descendant_ids(include_self: true) return super unless use_traversal_ids? - self_and_descendants_with_duplicates(include_self: include_self) - .select('DISTINCT namespaces.id') - end - - # Make sure we drop the STI `type = 'Group'` condition for better performance. - # Logically equivalent so long as hierarchies remain homogeneous. - def without_sti_condition - unscope(where: :type) + if Feature.enabled?(:traversal_ids_btree, default_enabled: :yaml) + self_and_descendants_with_comparison_operators(include_self: include_self).as_ids + else + self_and_descendants_with_duplicates_with_array_operator(include_self: include_self) + .select('DISTINCT namespaces.id') + end end def order_by_depth(hierarchy_order) @@ -75,7 +81,7 @@ module Namespaces # When we have queries that break this SELECT * format we can run in to errors. # For example `SELECT DISTINCT on(...)` will fail when we chain a `.count` c def normal_select - unscoped.without_sti_condition.from(all, :namespaces) + unscoped.from(all, :namespaces) end private @@ -84,16 +90,52 @@ module Namespaces Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) end + def use_traversal_ids_roots? + Feature.enabled?(:use_traversal_ids_roots, default_enabled: :yaml) && + use_traversal_ids? + end + def use_traversal_ids_for_ancestor_scopes? Feature.enabled?(:use_traversal_ids_for_ancestor_scopes, default_enabled: :yaml) && use_traversal_ids? end - def self_and_descendants_with_duplicates(include_self: true) + def self_and_descendants_with_comparison_operators(include_self: true) + base = all.select( + :traversal_ids, + 'LEAD (namespaces.traversal_ids, 1) OVER (ORDER BY namespaces.traversal_ids ASC) next_traversal_ids' + ) + cte = Gitlab::SQL::CTE.new(:base_cte, base) + + namespaces = Arel::Table.new(:namespaces) + records = unscoped + .with(cte.to_arel) + .from([cte.table, namespaces]) + + # Bound the search space to ourselves (optional) and descendants. + # + # WHERE (base_cte.next_traversal_ids IS NULL OR base_cte.next_traversal_ids > namespaces.traversal_ids) + # AND next_traversal_ids_sibling(base_cte.traversal_ids) > namespaces.traversal_ids + records = records + .where(cte.table[:next_traversal_ids].eq(nil).or(cte.table[:next_traversal_ids].gt(namespaces[:traversal_ids]))) + .where(next_sibling_func(cte.table[:traversal_ids]).gt(namespaces[:traversal_ids])) + + # AND base_cte.traversal_ids <= namespaces.traversal_ids + if include_self + records.where(cte.table[:traversal_ids].lteq(namespaces[:traversal_ids])) + else + records.where(cte.table[:traversal_ids].lt(namespaces[:traversal_ids])) + end + end + + def next_sibling_func(*args) + Arel::Nodes::NamedFunction.new('next_traversal_ids_sibling', args) + end + + def self_and_descendants_with_duplicates_with_array_operator(include_self: true) base_ids = select(:id) records = unscoped - .without_sti_condition .from("namespaces, (#{base_ids.to_sql}) base") .where('namespaces.traversal_ids @> ARRAY[base.id]') diff --git a/app/models/namespaces/traversal/recursive_scopes.rb b/app/models/namespaces/traversal/recursive_scopes.rb index 6659cefe095..925d9b8bb0c 100644 --- a/app/models/namespaces/traversal/recursive_scopes.rb +++ b/app/models/namespaces/traversal/recursive_scopes.rb @@ -10,6 +10,13 @@ module Namespaces select('id') end + def roots + Gitlab::ObjectHierarchy + .new(all) + .base_and_ancestors + .where(namespaces: { parent_id: nil }) + end + def self_and_ancestors(include_self: true, hierarchy_order: nil) records = Gitlab::ObjectHierarchy.new(all).base_and_ancestors(hierarchy_order: hierarchy_order) diff --git a/app/models/namespaces/user_namespace.rb b/app/models/namespaces/user_namespace.rb index 22b7a0a3b2b..d4d7d352e71 100644 --- a/app/models/namespaces/user_namespace.rb +++ b/app/models/namespaces/user_namespace.rb @@ -3,6 +3,26 @@ # TODO: currently not created/mapped in the database, will be done in another issue # https://gitlab.com/gitlab-org/gitlab/-/issues/341070 module Namespaces + #################################################################### + # PLEASE DO NOT OVERRIDE METHODS IN THIS CLASS! + # + # This class is a placeholder for STI. But we also want to ensure + # tests using `:namespace` factory are still testing the same functionality. + # + # Many legacy tests use `:namespace` which has a slight semantic + # mismatch as it always has been a User (personal) namespace. + # + # If you need to make a change here, please ping the + # Manage/Workspaces group so we can ensure that the + # changes do not break existing functionality. + # + # As Namespaces evolve we may be able to relax this restriction + # but for now, please check in with us <3 + # + # For details, see the discussion in + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74152 + #################################################################### + class UserNamespace < Namespace def self.sti_name 'User' diff --git a/app/models/note.rb b/app/models/note.rb index 37473518892..cb285028203 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -114,6 +114,7 @@ class Note < ApplicationRecord scope :fresh, -> { order_created_asc.with_order_id_asc } scope :updated_after, ->(time) { where('updated_at > ?', time) } scope :with_updated_at, ->(time) { where(updated_at: time) } + scope :with_discussion_ids, ->(discussion_ids) { where(discussion_id: discussion_ids) } scope :with_suggestions, -> { joins(:suggestions) } scope :inc_author, -> { includes(:author) } scope :with_api_entity_associations, -> { preload(:note_diff_file, :author) } diff --git a/app/models/packages/npm.rb b/app/models/packages/npm.rb index e49199d911c..9221187d92a 100644 --- a/app/models/packages/npm.rb +++ b/app/models/packages/npm.rb @@ -9,5 +9,9 @@ module Packages package_name.match(Gitlab::Regex.npm_package_name_regex)&.captures&.first end + + def self.table_name_prefix + 'packages_npm_' + end end end diff --git a/app/models/packages/npm/metadatum.rb b/app/models/packages/npm/metadatum.rb new file mode 100644 index 00000000000..7388c4bdbd2 --- /dev/null +++ b/app/models/packages/npm/metadatum.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class Packages::Npm::Metadatum < ApplicationRecord + belongs_to :package, -> { where(package_type: :npm) }, inverse_of: :npm_metadatum + + validates :package, presence: true + # From https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#abbreviated-version-object + validates :package_json, json_schema: { filename: "npm_package_json" } + validate :ensure_npm_package_type + validate :ensure_package_json_size + + private + + def ensure_npm_package_type + return if package&.npm? + + errors.add(:base, _('Package type must be NPM')) + end + + def ensure_package_json_size + return if package_json.to_s.size < 20000 + + errors.add(:package_json, _('structure is too large')) + end +end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 34eae6ab5dc..962a1057a22 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -39,8 +39,9 @@ class Packages::Package < ApplicationRecord has_one :nuget_metadatum, inverse_of: :package, class_name: 'Packages::Nuget::Metadatum' has_one :composer_metadatum, inverse_of: :package, class_name: 'Packages::Composer::Metadatum' has_one :rubygems_metadatum, inverse_of: :package, class_name: 'Packages::Rubygems::Metadatum' + has_one :npm_metadatum, inverse_of: :package, class_name: 'Packages::Npm::Metadatum' has_many :build_infos, inverse_of: :package - has_many :pipelines, through: :build_infos + has_many :pipelines, through: :build_infos, disable_joins: true has_one :debian_publication, inverse_of: :package, class_name: 'Packages::Debian::Publication' has_one :debian_distribution, through: :debian_publication, source: :distribution, inverse_of: :packages, class_name: 'Packages::Debian::ProjectDistribution' @@ -102,7 +103,6 @@ class Packages::Package < ApplicationRecord scope :with_status, ->(status) { where(status: status) } scope :displayable, -> { with_status(DISPLAYABLE_STATUSES) } scope :installable, -> { with_status(INSTALLABLE_STATUSES) } - scope :including_build_info, -> { includes(pipelines: :user) } scope :including_project_route, -> { includes(project: { namespace: :route }) } scope :including_tags, -> { includes(:tags) } scope :including_dependency_links, -> { includes(dependency_links: :dependency) } @@ -126,11 +126,13 @@ class Packages::Package < ApplicationRecord .where(Packages::Composer::Metadatum.table_name => { target_sha: target }) end scope :preload_composer, -> { preload(:composer_metadatum) } + scope :preload_npm_metadatum, -> { preload(:npm_metadatum) } scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) } scope :has_version, -> { where.not(version: nil) } scope :preload_files, -> { preload(:package_files) } + scope :preload_pipelines, -> { preload(pipelines: :user) } scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) } scope :limit_recent, ->(limit) { order_created_desc.limit(limit) } scope :select_distinct_name, -> { select(:name).distinct } @@ -245,7 +247,7 @@ class Packages::Package < ApplicationRecord def versions project.packages - .including_build_info + .preload_pipelines .including_tags .with_name(name) .where.not(version: version) diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 14701b8a800..87c9f56cc41 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -15,7 +15,7 @@ class Packages::PackageFile < ApplicationRecord has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum' has_many :package_file_build_infos, inverse_of: :package_file, class_name: 'Packages::PackageFileBuildInfo' - has_many :pipelines, through: :package_file_build_infos + has_many :pipelines, through: :package_file_build_infos, disable_joins: true has_one :debian_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Debian::FileMetadatum' has_one :helm_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Helm::FileMetadatum' @@ -38,6 +38,7 @@ class Packages::PackageFile < ApplicationRecord scope :with_format, ->(format) { where(::Packages::PackageFile.arel_table[:file_name].matches("%.#{format}")) } scope :preload_package, -> { preload(:package) } + scope :preload_pipelines, -> { preload(pipelines: :user) } scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) } scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) } scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) } diff --git a/app/models/preloaders/group_policy_preloader.rb b/app/models/preloaders/group_policy_preloader.rb new file mode 100644 index 00000000000..95d6e0b5c1f --- /dev/null +++ b/app/models/preloaders/group_policy_preloader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Preloaders + class GroupPolicyPreloader + def initialize(groups, current_user) + @groups = groups + @current_user = current_user + end + + def execute + Preloaders::UserMaxAccessLevelInGroupsPreloader.new(@groups, @current_user).execute + Preloaders::GroupRootAncestorPreloader.new(@groups, root_ancestor_preloads).execute + end + + private + + def root_ancestor_preloads + [] + end + end +end + +Preloaders::GroupPolicyPreloader.prepend_mod_with('Preloaders::GroupPolicyPreloader') diff --git a/app/models/preloaders/group_root_ancestor_preloader.rb b/app/models/preloaders/group_root_ancestor_preloader.rb new file mode 100644 index 00000000000..3ca713d9635 --- /dev/null +++ b/app/models/preloaders/group_root_ancestor_preloader.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Preloaders + class GroupRootAncestorPreloader + def initialize(groups, root_ancestor_preloads = []) + @groups = groups + @root_ancestor_preloads = root_ancestor_preloads + end + + def execute + return unless ::Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + + # type == 'Group' condition located on subquery to prevent a filter in the query + root_query = Namespace.joins("INNER JOIN (#{join_sql}) as root_query ON root_query.root_id = namespaces.id") + .select('namespaces.*, root_query.id as source_id') + + root_query = root_query.preload(*@root_ancestor_preloads) if @root_ancestor_preloads.any? + + root_ancestors_by_id = root_query.group_by(&:source_id) + + @groups.each do |group| + group.root_ancestor = root_ancestors_by_id[group.id].first + end + end + + private + + def join_sql + Group.select('id, traversal_ids[1] as root_id').where(id: @groups.map(&:id)).to_sql + end + end +end diff --git a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb index 14f1d271572..bdd76d39ec1 100644 --- a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb @@ -3,7 +3,6 @@ module Preloaders # This class preloads the max access level (role) for the user within the given groups and # stores the values in requests store. - # Will only be able to preload max access level for groups where the user is a direct member class UserMaxAccessLevelInGroupsPreloader include BulkMemberAccessLoad @@ -13,8 +12,17 @@ module Preloaders end def execute + if ::Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + preload_with_traversal_ids + else + preload_direct_memberships + end + end + + private + + def preload_direct_memberships group_memberships = GroupMember.active_without_invites_and_requests - .non_minimal_access .where(user: @user, source_id: @groups) .group(:source_id) .maximum(:access_level) @@ -23,5 +31,22 @@ module Preloaders merge_value_to_request_store(User, @user.id, group_id, max_access_level) end end + + def preload_with_traversal_ids + max_access_levels = GroupMember.active_without_invites_and_requests + .where(user: @user) + .joins("INNER JOIN (#{traversal_join_sql}) as hierarchy ON members.source_id = hierarchy.traversal_id") + .group('hierarchy.id') + .maximum(:access_level) + + @groups.each do |group| + max_access_level = max_access_levels[group.id] || Gitlab::Access::NO_ACCESS + merge_value_to_request_store(User, @user.id, group.id, max_access_level) + end + end + + def traversal_join_sql + Namespace.select('id, unnest(traversal_ids) as traversal_id').where(id: @groups.map(&:id)).to_sql + end end end diff --git a/app/models/project.rb b/app/models/project.rb index 2ceba10e86e..2288850553c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -19,7 +19,6 @@ class Project < ApplicationRecord include Presentable include HasRepository include HasWiki - include HasIntegrations include CanMoveRepositoryStorage include Routable include GroupDescendant @@ -98,7 +97,7 @@ class Project < ApplicationRecord before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } before_save :ensure_runners_token - before_save :ensure_project_namespace_in_sync + before_validation :ensure_project_namespace_in_sync after_save :update_project_statistics, if: :saved_change_to_namespace_id? @@ -147,7 +146,7 @@ class Project < ApplicationRecord belongs_to :namespace # Sync deletion via DB Trigger to ensure we do not have # a project without a project_namespace (or vice-versa) - belongs_to :project_namespace, autosave: true, class_name: 'Namespaces::ProjectNamespace', foreign_key: 'project_namespace_id', inverse_of: :project + belongs_to :project_namespace, autosave: true, class_name: 'Namespaces::ProjectNamespace', foreign_key: 'project_namespace_id' alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id @@ -189,6 +188,7 @@ class Project < ApplicationRecord has_one :prometheus_integration, class_name: 'Integrations::Prometheus', inverse_of: :project has_one :pushover_integration, class_name: 'Integrations::Pushover' has_one :redmine_integration, class_name: 'Integrations::Redmine' + has_one :shimo_integration, class_name: 'Integrations::Shimo' has_one :slack_integration, class_name: 'Integrations::Slack' has_one :slack_slash_commands_integration, class_name: 'Integrations::SlackSlashCommands' has_one :teamcity_integration, class_name: 'Integrations::Teamcity' @@ -451,6 +451,7 @@ class Project < ApplicationRecord :allow_merge_on_skipped_pipeline=, :has_confluence?, to: :project_setting delegate :active?, to: :prometheus_integration, allow_nil: true, prefix: true + delegate :merge_commit_template, :merge_commit_template=, to: :project_setting, allow_nil: true delegate :log_jira_dvcs_integration_usage, :jira_dvcs_server_last_sync_at, :jira_dvcs_cloud_last_sync_at, to: :feature_usage @@ -475,6 +476,7 @@ class Project < ApplicationRecord validates :project_feature, presence: true validates :namespace, presence: true + validates :project_namespace, presence: true, if: -> { self.namespace && self.root_namespace.project_namespace_creation_enabled? } validates :name, uniqueness: { scope: :namespace_id } validates :import_url, public_url: { schemes: ->(project) { project.persisted? ? VALID_MIRROR_PROTOCOLS : VALID_IMPORT_PROTOCOLS }, ports: ->(project) { project.persisted? ? VALID_MIRROR_PORTS : VALID_IMPORT_PORTS }, @@ -492,6 +494,7 @@ class Project < ApplicationRecord validates :variables, nested_attributes_duplicates: { scope: :environment_scope } validates :bfg_object_map, file_size: { maximum: :max_attachment_size } validates :max_artifacts_size, numericality: { only_integer: true, greater_than: 0, allow_nil: true } + validates :suggestion_commit_message, length: { maximum: 255 } # Scopes scope :pending_delete, -> { where(pending_delete: true) } @@ -857,6 +860,18 @@ class Project < ApplicationRecord rescue ActionController::RoutingError, URI::InvalidURIError nil end + + def without_integration(integration) + integrations = Integration + .select('1') + .where("#{Integration.table_name}.project_id = projects.id") + .where(type: integration.type) + + Project + .where('NOT EXISTS (?)', integrations) + .where(pending_delete: false) + .where(archived: false) + end end def initialize(attributes = nil) @@ -1453,7 +1468,7 @@ class Project < ApplicationRecord end def disabled_integrations - [:zentao] + [:shimo] end def find_or_initialize_integration(name) @@ -1777,10 +1792,12 @@ class Project < ApplicationRecord def all_runners Ci::Runner.from_union([runners, group_runners, shared_runners]) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') end def all_available_runners Ci::Runner.from_union([runners, group_runners, available_shared_runners]) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') end # Once issue 339937 is fixed, please search for all mentioned of @@ -2051,14 +2068,16 @@ class Project < ApplicationRecord end def predefined_variables - Gitlab::Ci::Variables::Collection.new - .concat(predefined_ci_server_variables) - .concat(predefined_project_variables) - .concat(pages_variables) - .concat(container_registry_variables) - .concat(dependency_proxy_variables) - .concat(auto_devops_variables) - .concat(api_variables) + strong_memoize(:predefined_variables) do + Gitlab::Ci::Variables::Collection.new + .concat(predefined_ci_server_variables) + .concat(predefined_project_variables) + .concat(pages_variables) + .concat(container_registry_variables) + .concat(dependency_proxy_variables) + .concat(auto_devops_variables) + .concat(api_variables) + end end def predefined_project_variables @@ -2579,18 +2598,21 @@ class Project < ApplicationRecord config = Gitlab.config.incoming_email wildcard = Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER - config.address&.gsub(wildcard, "#{full_path_slug}-#{id}-issue-") + config.address&.gsub(wildcard, "#{full_path_slug}-#{default_service_desk_suffix}") end def service_desk_custom_address return unless Gitlab::ServiceDeskEmail.enabled? - key = service_desk_setting&.project_key - return unless key.present? + key = service_desk_setting&.project_key || default_service_desk_suffix Gitlab::ServiceDeskEmail.address_for_key("#{full_path_slug}-#{key}") end + def default_service_desk_suffix + "#{id}-issue-" + end + def root_namespace if namespace.has_parent? namespace.root_ancestor @@ -2911,12 +2933,28 @@ class Project < ApplicationRecord end def ensure_project_namespace_in_sync - if changes.keys & [:name, :path, :namespace_id, :visibility_level] && project_namespace.present? - project_namespace.name = name - project_namespace.path = path - project_namespace.parent = namespace - project_namespace.visibility_level = visibility_level - end + # create project_namespace when project is created if create_project_namespace_on_project_create FF is enabled + build_project_namespace if project_namespace_creation_enabled? + + # regardless of create_project_namespace_on_project_create FF we need + # to keep project and project namespace in sync if there is one + sync_attributes(project_namespace) if sync_project_namespace? + end + + def project_namespace_creation_enabled? + new_record? && !project_namespace && self.namespace && self.root_namespace.project_namespace_creation_enabled? + end + + def sync_project_namespace? + (changes.keys & %w(name path namespace_id namespace visibility_level shared_runners_enabled)).any? && project_namespace.present? + end + + def sync_attributes(project_namespace) + project_namespace.name = name + project_namespace.path = path + project_namespace.parent = namespace + project_namespace.shared_runners_enabled = shared_runners_enabled + project_namespace.visibility_level = visibility_level end end diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index 64e768007ee..fed19a37a16 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -24,8 +24,9 @@ class ProjectAuthorization < ApplicationRecord end connection.execute <<-EOF.strip_heredoc - INSERT INTO project_authorizations (user_id, project_id, access_level) - VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + INSERT INTO project_authorizations (user_id, project_id, access_level) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + ON CONFLICT DO NOTHING EOF end end diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index 24d892290a6..6c8d2226bc9 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -12,6 +12,8 @@ class ProjectSetting < ApplicationRecord self.primary_key = :project_id + validates :merge_commit_template, length: { maximum: 500 } + def squash_enabled_by_default? %w[always default_on].include?(squash_option) end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 774d81156b7..94904e9792f 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -41,13 +41,15 @@ class ProjectTeam member end - def add_users(users, access_level, current_user: nil, expires_at: nil) + def add_users(users, access_level, current_user: nil, expires_at: nil, tasks_to_be_done: [], tasks_project_id: nil) Members::Projects::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass project, users, access_level, current_user: current_user, - expires_at: expires_at + expires_at: expires_at, + tasks_to_be_done: tasks_to_be_done, + tasks_project_id: tasks_project_id ) end diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb index f3352ecc5ee..8d6f8c3a9ca 100644 --- a/app/models/projects/topic.rb +++ b/app/models/projects/topic.rb @@ -28,3 +28,5 @@ module Projects end end end + +::Projects::Topic.prepend_mod_with('Projects::Topic') diff --git a/app/models/push_event_payload.rb b/app/models/push_event_payload.rb index 441b94e1855..8358be35470 100644 --- a/app/models/push_event_payload.rb +++ b/app/models/push_event_payload.rb @@ -2,9 +2,6 @@ class PushEventPayload < ApplicationRecord extend SuppressCompositePrimaryKeyWarning - include IgnorableColumns - - ignore_columns :event_id_convert_to_bigint, remove_with: '14.4', remove_after: '2021-10-22' include ShaAttribute diff --git a/app/models/release.rb b/app/models/release.rb index eac6346cc60..0fda6940249 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -34,6 +34,7 @@ class Release < ApplicationRecord project: [:project_feature, :route, { namespace: :route }]) } scope :with_milestones, -> { joins(:milestone_releases) } + scope :with_group_milestones, -> { joins(:milestones).where.not(milestones: { group_id: nil }) } scope :recent, -> { sorted.limit(MAX_NUMBER_TO_DISPLAY) } scope :without_evidence, -> { left_joins(:evidences).where(::Releases::Evidence.arel_table[:id].eq(nil)) } scope :released_within_2hrs, -> { where(released_at: Time.zone.now - 1.hour..Time.zone.now + 1.hour) } diff --git a/app/models/repository.rb b/app/models/repository.rb index 119d874a6e1..47482f04bca 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -731,10 +731,8 @@ class Repository raw_repository.local_branches(sort_by: sort_by, pagination_params: pagination_params) end - def tags_sorted_by(value) - return raw_repository.tags(sort_by: value) if Feature.enabled?(:tags_finder_gitaly, project, default_enabled: :yaml) - - tags_ruby_sort(value) + def tags_sorted_by(value, pagination_params = nil) + raw_repository.tags(sort_by: value, pagination_params: pagination_params) end # Params: @@ -1091,6 +1089,13 @@ class Repository after_create true + rescue Gitlab::Git::Repository::RepositoryExists + # We do not want to call `#after_create` given that we didn't create the + # repo, but we obviously have a mismatch between what's in our exists cache + # and actual on-disk state as seen by Gitaly. Let's thus expire our caches. + expire_status_cache + + nil end def create_from_bundle(bundle_path) @@ -1163,34 +1168,6 @@ class Repository @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore) end - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/339741 - def tags_ruby_sort(value) - case value - when 'name_asc' - VersionSorter.sort(tags) { |tag| tag.name } - when 'name_desc' - VersionSorter.rsort(tags) { |tag| tag.name } - when 'updated_desc' - tags_sorted_by_committed_date.reverse - when 'updated_asc' - tags_sorted_by_committed_date - else - tags - end - end - - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/339741 - def tags_sorted_by_committed_date - # Annotated tags can point to any object (e.g. a blob), but generally - # tags point to a commit. If we don't have a commit, then just default - # to putting the tag at the end of the list. - default = Time.current - - tags.sort_by do |tag| - tag.dereferenced_target&.committed_date || default - end - end - def repository_event(event, tags = {}) Gitlab::Metrics.add_event(event, tags) end diff --git a/app/models/suggestion.rb b/app/models/suggestion.rb index ff564d87449..f1ca5c23997 100644 --- a/app/models/suggestion.rb +++ b/app/models/suggestion.rb @@ -50,6 +50,7 @@ class Suggestion < ApplicationRecord next _("Can't apply as the source branch was deleted.") unless noteable.source_branch_exists? next outdated_reason if outdated?(cached: cached) || !note.active? next _("This suggestion already matches its content.") unless different_content? + next _("This file was modified for readability, and can't accept suggestions. Edit it directly.") if file_path.end_with? "ipynb" end end diff --git a/app/models/todo.rb b/app/models/todo.rb index 94a99603848..742b8fd2a9d 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -18,6 +18,7 @@ class Todo < ApplicationRecord DIRECTLY_ADDRESSED = 7 MERGE_TRAIN_REMOVED = 8 # This is an EE-only feature REVIEW_REQUESTED = 9 + ATTENTION_REQUESTED = 10 ACTION_NAMES = { ASSIGNED => :assigned, @@ -28,7 +29,8 @@ class Todo < ApplicationRecord APPROVAL_REQUIRED => :approval_required, UNMERGEABLE => :unmergeable, DIRECTLY_ADDRESSED => :directly_addressed, - MERGE_TRAIN_REMOVED => :merge_train_removed + MERGE_TRAIN_REMOVED => :merge_train_removed, + ATTENTION_REQUESTED => :attention_requested }.freeze belongs_to :author, class_name: "User" @@ -189,6 +191,10 @@ class Todo < ApplicationRecord action == REVIEW_REQUESTED end + def attention_requested? + action == ATTENTION_REQUESTED + end + def merge_train_removed? action == MERGE_TRAIN_REMOVED end diff --git a/app/models/upload.rb b/app/models/upload.rb index c1a3df82457..ac7ebb31abc 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -2,6 +2,7 @@ class Upload < ApplicationRecord include Checksummable + # Upper limit for foreground checksum processing CHECKSUM_THRESHOLD = 100.megabytes @@ -51,9 +52,9 @@ class Upload < ApplicationRecord ## # FastDestroyAll concerns - def finalize_fast_destroy(keys) - keys.each do |store_class, paths| - store_class.new.delete_keys_async(paths) + def finalize_fast_destroy(items_to_remove) + items_to_remove.each do |store_class, keys| + store_class.new.delete_keys_async(keys) end end end @@ -65,6 +66,10 @@ class Upload < ApplicationRecord uploader_class.absolute_path(self) end + def relative_path + uploader_class.relative_path(self) + end + def calculate_checksum! self.checksum = nil return unless needs_checksum? diff --git a/app/models/uploads/fog.rb b/app/models/uploads/fog.rb index b44e273e9ab..5d57b644dbe 100644 --- a/app/models/uploads/fog.rb +++ b/app/models/uploads/fog.rb @@ -15,13 +15,21 @@ module Uploads end def delete_keys(keys) - keys.each do |key| - connection.delete_object(bucket_name, key) - end + keys.each { |key| delete_object(key) } end private + def delete_object(key) + connection.delete_object(bucket_name, key) + + # So far, only GoogleCloudStorage raises an exception when the file is not found. + # Other providers support idempotent requests and does not raise an error + # when the file is missing. + rescue ::Google::Apis::ClientError => e + Gitlab::ErrorTracking.log_exception(e) + end + def object_store Gitlab.config.uploads.object_store end diff --git a/app/models/uploads/local.rb b/app/models/uploads/local.rb index bd295a66838..9df69998991 100644 --- a/app/models/uploads/local.rb +++ b/app/models/uploads/local.rb @@ -55,3 +55,5 @@ module Uploads end end end + +Uploads::Local.prepend_mod diff --git a/app/models/user.rb b/app/models/user.rb index 0e19e6e4a79..3ab5b7ee364 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -123,7 +123,7 @@ class User < ApplicationRecord # Profile has_many :keys, -> { regular_keys }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :expired_and_unnotified_keys, -> { expired_and_not_notified }, class_name: 'Key' + has_many :expired_today_and_unnotified_keys, -> { expired_today_and_not_notified }, class_name: 'Key' has_many :expiring_soon_and_unnotified_keys, -> { expiring_soon_and_not_notified }, class_name: 'Key' has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :group_deploy_keys @@ -274,14 +274,21 @@ class User < ApplicationRecord after_update :username_changed_hook, if: :saved_change_to_username? after_destroy :post_destroy_hook after_destroy :remove_key_cache + after_create :add_primary_email_to_emails!, if: :confirmed? after_commit(on: :update) do if previous_changes.key?('email') - # Grab previous_email here since previous_changes changes after - # #update_emails_with_primary_email and #update_notification_email are called + # Add the old primary email to Emails if not added already - this should be removed + # after the background migration for MR https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70872/ has completed, + # as the primary email is now added to Emails upon confirmation + # Issue to remove that: https://gitlab.com/gitlab-org/gitlab/-/issues/344134 previous_confirmed_at = previous_changes.key?('confirmed_at') ? previous_changes['confirmed_at'][0] : confirmed_at previous_email = previous_changes[:email][0] + if previous_confirmed_at && !emails.exists?(email: previous_email) + # rubocop: disable CodeReuse/ServiceClass + Emails::CreateService.new(self, user: self, email: previous_email).execute(confirmed_at: previous_confirmed_at) + # rubocop: enable CodeReuse/ServiceClass + end - update_emails_with_primary_email(previous_confirmed_at, previous_email) update_invalid_gpg_signatures end end @@ -454,8 +461,8 @@ class User < ApplicationRecord scope :order_recent_last_activity, -> { reorder(Gitlab::Database.nulls_last_order('last_activity_on', 'DESC')) } scope :order_oldest_last_activity, -> { reorder(Gitlab::Database.nulls_first_order('last_activity_on', 'ASC')) } scope :by_id_and_login, ->(id, login) { where(id: id).where('username = LOWER(:login) OR email = LOWER(:login)', login: login) } - scope :dormant, -> { active.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } - scope :with_no_activity, -> { active.where(last_activity_on: nil) } + scope :dormant, -> { with_state(:active).human_or_service_user.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } + scope :with_no_activity, -> { with_state(:active).human_or_service_user.where(last_activity_on: nil) } scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } scope :get_ids_by_username, -> (username) { where(username: username).pluck(:id) } @@ -466,7 +473,11 @@ class User < ApplicationRecord end def active_for_authentication? - super && can?(:log_in) + return false unless super + + check_ldap_if_ldap_blocked! + + can?(:log_in) end # The messages for these keys are defined in `devise.en.yml` @@ -935,6 +946,8 @@ class User < ApplicationRecord end def unique_email + return if errors.added?(:email, _('has already been taken')) + if !emails.exists?(email: email) && Email.exists?(email: email) errors.add(:email, _('has already been taken')) end @@ -963,24 +976,6 @@ class User < ApplicationRecord skip_reconfirmation! if emails.confirmed.where(email: self.email).any? end - # Note: the use of the Emails services will cause `saves` on the user object, running - # through the callbacks again and can have side effects, such as the `previous_changes` - # hash and `_was` variables getting munged. - # By using an `after_commit` instead of `after_update`, we avoid the recursive callback - # scenario, though it then requires us to use the `previous_changes` hash - # rubocop: disable CodeReuse/ServiceClass - def update_emails_with_primary_email(previous_confirmed_at, previous_email) - primary_email_record = emails.find_by(email: email) - Emails::DestroyService.new(self, user: self).execute(primary_email_record) if primary_email_record - - # the original primary email was confirmed, and we want that to carry over. We don't - # have access to the original confirmation values at this point, so just set confirmed_at - Emails::CreateService.new(self, user: self, email: previous_email).execute(confirmed_at: previous_confirmed_at) - - update_columns(confirmed_at: primary_email_record.confirmed_at) if primary_email_record&.confirmed_at - end - # rubocop: enable CodeReuse/ServiceClass - def update_invalid_gpg_signatures gpg_keys.each(&:update_invalid_gpg_signatures) end @@ -1025,8 +1020,10 @@ class User < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass - def remove_project_authorizations(project_ids) - project_authorizations.where(project_id: project_ids).delete_all + def remove_project_authorizations(project_ids, per_batch = 1000) + project_ids.each_slice(per_batch) do |project_ids_batch| + project_authorizations.where(project_id: project_ids_batch).delete_all + end end def authorized_projects(min_access_level = nil) @@ -1389,7 +1386,7 @@ class User < ApplicationRecord all_emails << email unless temp_oauth_email? all_emails << private_commit_email if include_private_email all_emails.concat(emails.map(&:email)) - all_emails + all_emails.uniq end def verified_emails(include_private_email: true) @@ -1397,7 +1394,7 @@ class User < ApplicationRecord verified_emails << email if primary_email_verified? verified_emails << private_commit_email if include_private_email verified_emails.concat(emails.confirmed.pluck(:email)) - verified_emails + verified_emails.uniq end def public_verified_emails @@ -1610,8 +1607,6 @@ class User < ApplicationRecord true end - # TODO Please check all callers and remove allow_cross_joins_across_databases, - # when https://gitlab.com/gitlab-org/gitlab/-/issues/336436 is done. def ci_owned_runners @ci_owned_runners ||= begin project_runners = Ci::RunnerProject @@ -1624,7 +1619,7 @@ class User < ApplicationRecord .joins(:runner) .select('ci_runners.*') - Ci::Runner.from_union([project_runners, group_runners]) + Ci::Runner.from_union([project_runners, group_runners]).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') end end @@ -1980,6 +1975,37 @@ class User < ApplicationRecord ci_job_token_scope.present? end + # override from Devise::Confirmable + # + # Add the primary email to user.emails (or confirm it if it was already + # present) when the primary email is confirmed. + def confirm(*args) + saved = super(*args) + return false unless saved + + email_to_confirm = self.emails.find_by(email: self.email) + + if email_to_confirm.present? + email_to_confirm.confirm(*args) + else + add_primary_email_to_emails! + end + + saved + end + + def user_project + strong_memoize(:user_project) do + personal_projects.find_by(path: username, visibility_level: Gitlab::VisibilityLevel::PUBLIC) + end + end + + def user_readme + strong_memoize(:user_readme) do + user_project&.repository&.readme + end + end + protected # override, from Devise::Validatable @@ -2020,6 +2046,12 @@ class User < ApplicationRecord 'en' end + # rubocop: disable CodeReuse/ServiceClass + def add_primary_email_to_emails! + Emails::CreateService.new(self, user: self, email: self.email).execute(confirmed_at: self.confirmed_at) + end + # rubocop: enable CodeReuse/ServiceClass + def notification_email_verified return if notification_email.blank? || temp_oauth_email? @@ -2153,6 +2185,13 @@ class User < ApplicationRecord def ci_job_token_scope_cache_key "users:#{id}:ci:job_token_scope" end + + # An `ldap_blocked` user will be unblocked if LDAP indicates they are allowed. + def check_ldap_if_ldap_blocked! + return unless ::Gitlab::Auth::Ldap::Config.enabled? && ldap_blocked? + + ::Gitlab::Auth::Ldap::Access.allowed?(self) + end end User.prepend_mod_with('User') diff --git a/app/models/user_status.rb b/app/models/user_status.rb index 1c8634e47c3..7a803e8f1f6 100644 --- a/app/models/user_status.rb +++ b/app/models/user_status.rb @@ -22,7 +22,7 @@ class UserStatus < ApplicationRecord enum availability: { not_set: 0, busy: 1 } validates :user, presence: true - validates :emoji, inclusion: { in: Gitlab::Emoji.emojis_names } + validates :emoji, 'gitlab/emoji_name': true validates :message, length: { maximum: 100 }, allow_blank: true scope :scheduled_for_cleanup, -> { where(arel_table[:clear_status_at].lteq(Time.current)) } @@ -33,3 +33,5 @@ class UserStatus < ApplicationRecord self.clear_status_at = CLEAR_STATUS_QUICK_OPTIONS[value]&.from_now end end + +UserStatus.prepend_mod_with('UserStatus') diff --git a/app/models/users/credit_card_validation.rb b/app/models/users/credit_card_validation.rb index a4cc43d1f13..556ee03605d 100644 --- a/app/models/users/credit_card_validation.rb +++ b/app/models/users/credit_card_validation.rb @@ -9,6 +9,7 @@ module Users belongs_to :user validates :holder_name, length: { maximum: 26 } + validates :network, length: { maximum: 32 } validates :last_digits, allow_nil: true, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 9999 } @@ -17,7 +18,7 @@ module Users self.class.where( expiration_date: expiration_date, last_digits: last_digits, - holder_name: holder_name + network: network ).order(credit_card_validated_at: :desc).includes(:user) end end diff --git a/app/models/users/in_product_marketing_email.rb b/app/models/users/in_product_marketing_email.rb index 8fe52ac7ecc..1f1eaacfe5c 100644 --- a/app/models/users/in_product_marketing_email.rb +++ b/app/models/users/in_product_marketing_email.rb @@ -22,7 +22,8 @@ module Users experience: 4, team_short: 5, trial_short: 6, - admin_verify: 7 + admin_verify: 7, + invite_team: 8 }, _suffix: true scope :without_track_and_series, -> (track, series) do diff --git a/app/models/users_statistics.rb b/app/models/users_statistics.rb index a903541f69a..a314ae8920b 100644 --- a/app/models/users_statistics.rb +++ b/app/models/users_statistics.rb @@ -3,12 +3,6 @@ class UsersStatistics < ApplicationRecord scope :order_created_at_desc, -> { order(created_at: :desc) } - class << self - def latest - order_created_at_desc.first - end - end - def active [ without_groups_and_projects, @@ -26,30 +20,26 @@ class UsersStatistics < ApplicationRecord end class << self - def create_current_stats! - stats_by_role = highest_role_stats + def latest + order_created_at_desc.first + end - create!( - without_groups_and_projects: without_groups_and_projects_stats, - with_highest_role_guest: stats_by_role[:guest], - with_highest_role_reporter: stats_by_role[:reporter], - with_highest_role_developer: stats_by_role[:developer], - with_highest_role_maintainer: stats_by_role[:maintainer], - with_highest_role_owner: stats_by_role[:owner], - bots: bot_stats, - blocked: blocked_stats - ) + def create_current_stats! + create!(highest_role_stats) end private def highest_role_stats { - owner: batch_count_for_access_level(Gitlab::Access::OWNER), - maintainer: batch_count_for_access_level(Gitlab::Access::MAINTAINER), - developer: batch_count_for_access_level(Gitlab::Access::DEVELOPER), - reporter: batch_count_for_access_level(Gitlab::Access::REPORTER), - guest: batch_count_for_access_level(Gitlab::Access::GUEST) + without_groups_and_projects: without_groups_and_projects_stats, + with_highest_role_guest: batch_count_for_access_level(Gitlab::Access::GUEST), + with_highest_role_reporter: batch_count_for_access_level(Gitlab::Access::REPORTER), + with_highest_role_developer: batch_count_for_access_level(Gitlab::Access::DEVELOPER), + with_highest_role_maintainer: batch_count_for_access_level(Gitlab::Access::MAINTAINER), + with_highest_role_owner: batch_count_for_access_level(Gitlab::Access::OWNER), + bots: bot_stats, + blocked: blocked_stats } end diff --git a/app/models/webauthn_registration.rb b/app/models/webauthn_registration.rb index 76f8faa11c7..71b50192e29 100644 --- a/app/models/webauthn_registration.rb +++ b/app/models/webauthn_registration.rb @@ -5,7 +5,8 @@ class WebauthnRegistration < ApplicationRecord belongs_to :user - validates :credential_xid, :public_key, :name, :counter, presence: true + validates :credential_xid, :public_key, :counter, presence: true + validates :name, length: { minimum: 0, allow_nil: false } validates :counter, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2**32 - 1 } end |