diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-02-20 13:49:51 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-02-20 13:49:51 +0000 |
| commit | 71786ddc8e28fbd3cb3fcc4b3ff15e5962a1c82e (patch) | |
| tree | 6a2d93ef3fb2d353bb7739e4b57e6541f51cdd71 /app/models | |
| parent | a7253423e3403b8c08f8a161e5937e1488f5f407 (diff) | |
| download | gitlab-ce-71786ddc8e28fbd3cb3fcc4b3ff15e5962a1c82e.tar.gz | |
Add latest changes from gitlab-org/gitlab@15-9-stable-eev15.9.0-rc42
Diffstat (limited to 'app/models')
169 files changed, 2057 insertions, 928 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index b15143c8c9c..eb645bcd653 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -29,7 +29,7 @@ class Ability # A list of users that can read confidential notes in a project def users_that_can_read_internal_notes(users, note_parent) DeclarativePolicy.subject_scope do - users.select { |u| allowed?(u, :reporter_access, note_parent) } + users.select { |u| allowed?(u, :read_internal_note, note_parent) } end end diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index ee0c23ef31e..dbcdfa5e946 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -4,6 +4,8 @@ class AbuseReport < ApplicationRecord include CacheMarkdownField include Sortable + MAX_CHAR_LIMIT_URL = 512 + cache_markdown_field :message, pipeline: :single_line belongs_to :reporter, class_name: 'User' @@ -23,13 +25,23 @@ class AbuseReport < ApplicationRecord validates :reported_from_url, allow_blank: true, - length: { maximum: 512 }, + length: { maximum: MAX_CHAR_LIMIT_URL }, addressable_url: { dns_rebind_protection: true, blocked_message: 'is an invalid URL. You can try reporting the abuse again, ' \ 'or contact a GitLab administrator for help.' } + validates :links_to_spam, + allow_blank: true, + length: { + maximum: 20, + message: N_("exceeds the limit of %{count} links") + } + + before_validation :filter_empty_strings_from_links_to_spam + validate :links_to_spam_contains_valid_urls + scope :by_user, ->(user) { where(user_id: user) } scope :with_users, -> { includes(:reporter, :user) } @@ -60,8 +72,38 @@ class AbuseReport < ApplicationRecord end def notify - return unless self.persisted? + return unless persisted? + + AbuseReportMailer.notify(id).deliver_later + end + + private + + def filter_empty_strings_from_links_to_spam + return if links_to_spam.blank? + + links_to_spam.reject!(&:empty?) + end + + def links_to_spam_contains_valid_urls + return if links_to_spam.blank? + + links_to_spam.each do |link| + Gitlab::UrlBlocker.validate!( + link, + schemes: %w[http https], + allow_localhost: true, + dns_rebind_protection: true + ) + + next unless link.length > MAX_CHAR_LIMIT_URL - AbuseReportMailer.notify(self.id).deliver_later + errors.add( + :links_to_spam, + format(_('contains URLs that exceed the %{limit} character limit'), limit: MAX_CHAR_LIMIT_URL) + ) + end + rescue ::Gitlab::UrlBlocker::BlockedUrlError + errors.add(:links_to_spam, _('only supports valid HTTP(S) URLs')) end end diff --git a/app/models/achievements/achievement.rb b/app/models/achievements/achievement.rb index a436e32b35b..95606e50ad4 100644 --- a/app/models/achievements/achievement.rb +++ b/app/models/achievements/achievement.rb @@ -4,6 +4,9 @@ module Achievements class Achievement < ApplicationRecord include Avatarable include StripAttribute + include IgnorableColumns + + ignore_column :revokable, remove_with: '15.11', remove_after: '2023-04-22' belongs_to :namespace, inverse_of: :achievements, optional: false diff --git a/app/models/airflow.rb b/app/models/airflow.rb new file mode 100644 index 00000000000..2e5642a2639 --- /dev/null +++ b/app/models/airflow.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module Airflow + def self.table_name_prefix + 'airflow_' + end +end diff --git a/app/models/airflow/dags.rb b/app/models/airflow/dags.rb new file mode 100644 index 00000000000..d17d4a4f3db --- /dev/null +++ b/app/models/airflow/dags.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Airflow + class Dags < ApplicationRecord + belongs_to :project + + validates :project, presence: true + validates :dag_name, length: { maximum: 255 }, presence: true + validates :schedule, length: { maximum: 255 } + validates :fileloc, length: { maximum: 255 } + + scope :by_project_id, ->(project_id) { where(project_id: project_id).order(id: :asc) } + end +end diff --git a/app/models/analytics/cycle_analytics/aggregation.rb b/app/models/analytics/cycle_analytics/aggregation.rb index b432955ad88..fa165ae9600 100644 --- a/app/models/analytics/cycle_analytics/aggregation.rb +++ b/app/models/analytics/cycle_analytics/aggregation.rb @@ -63,10 +63,13 @@ class Analytics::CycleAnalytics::Aggregation < ApplicationRecord group = group_or_project_namespace.is_a?(Group) ? group_or_project_namespace : group_or_project_namespace.parent top_level_group = group.root_ancestor aggregation = find_by(group_id: top_level_group.id) - return aggregation if aggregation.present? + return aggregation if aggregation&.enabled? - insert({ group_id: top_level_group.id }, unique_by: :group_id) - find_by(group_id: top_level_group.id) + # At this point we're sure that the group is licensed, we can always enable the aggregation. + # This re-enables the aggregation in case the group downgraded and later upgraded the license. + upsert({ group_id: top_level_group.id, enabled: true }) + + find(top_level_group.id) end private diff --git a/app/models/analytics/cycle_analytics/project_level.rb b/app/models/analytics/cycle_analytics/project_level.rb index d43793f60c9..813263fe833 100644 --- a/app/models/analytics/cycle_analytics/project_level.rb +++ b/app/models/analytics/cycle_analytics/project_level.rb @@ -33,8 +33,8 @@ module Analytics private def build_stage(stage_name) - stage_params = stage_params_by_name(stage_name).merge(project: project) - Analytics::CycleAnalytics::ProjectStage.new(stage_params) + stage_params = stage_params_by_name(stage_name).merge(namespace: project.project_namespace) + Analytics::CycleAnalytics::Stage.new(stage_params) end def stage_params_by_name(name) diff --git a/app/models/analytics/cycle_analytics/project_stage.rb b/app/models/analytics/cycle_analytics/project_stage.rb deleted file mode 100644 index 8a80514333f..00000000000 --- a/app/models/analytics/cycle_analytics/project_stage.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Analytics - module CycleAnalytics - class ProjectStage < ApplicationRecord - include Analytics::CycleAnalytics::Stageable - - belongs_to :project, optional: false - belongs_to :value_stream, class_name: 'Analytics::CycleAnalytics::ProjectValueStream', foreign_key: :project_value_stream_id - - alias_attribute :parent, :project - alias_attribute :parent_id, :project_id - - alias_attribute :value_stream_id, :project_value_stream_id - - delegate :group, to: :project - - validate :validate_project_group_for_label_events, if: -> { start_event_label_based? || end_event_label_based? } - - def self.relative_positioning_query_base(stage) - where(project_id: stage.project_id) - end - - def self.relative_positioning_parent_column - :project_id - end - - def self.distinct_stages_within_hierarchy(group) - with_preloaded_labels - .where(project_id: group.all_projects.select(:id)) - .select("DISTINCT ON(stage_event_hash_id) #{quoted_table_name}.*") - end - - private - - # Project should belong to a group when the stage has Label based events since only GroupLabels are allowed. - def validate_project_group_for_label_events - errors.add(:project, s_('CycleAnalyticsStage|should be under a group')) unless project.group - end - end - end -end diff --git a/app/models/analytics/cycle_analytics/project_value_stream.rb b/app/models/analytics/cycle_analytics/project_value_stream.rb deleted file mode 100644 index 3eba7e87b17..00000000000 --- a/app/models/analytics/cycle_analytics/project_value_stream.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -class Analytics::CycleAnalytics::ProjectValueStream < ApplicationRecord - belongs_to :project - - has_many :stages, class_name: 'Analytics::CycleAnalytics::ProjectStage' - - validates :project, :name, presence: true - validates :name, length: { minimum: 3, maximum: 100, allow_nil: false }, uniqueness: { scope: :project_id } - - def custom? - false - end - - def stages - [] - end - - def self.build_default_value_stream(project) - new(name: Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME, project: project) - end -end diff --git a/app/models/analytics/cycle_analytics/stage.rb b/app/models/analytics/cycle_analytics/stage.rb new file mode 100644 index 00000000000..7e9a89975a3 --- /dev/null +++ b/app/models/analytics/cycle_analytics/stage.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class Stage < ApplicationRecord + self.table_name = :analytics_cycle_analytics_group_stages + + include DatabaseEventTracking + include Analytics::CycleAnalytics::Stageable + include Analytics::CycleAnalytics::Parentable + + validates :name, uniqueness: { scope: [:group_id, :group_value_stream_id] } + belongs_to :value_stream, class_name: 'Analytics::CycleAnalytics::ValueStream', +foreign_key: :group_value_stream_id, inverse_of: :stages + + alias_attribute :parent, :namespace + alias_attribute :parent_id, :group_id + alias_attribute :value_stream_id, :group_value_stream_id + + def self.distinct_stages_within_hierarchy(namespace) + # Looking up the whole hierarchy including all kinds (type) of Namespace records. + # We're doing a custom traversal_ids query because: + # - The traversal_ids based `self_and_descendants` doesn't include the ProjectNamespace records. + # - The default recursive lookup also excludes the ProjectNamespace records. + # + # Related issue: https://gitlab.com/gitlab-org/gitlab/-/issues/386124 + all_namespace_ids = + Namespace + .select(Arel.sql('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)]').as('id')) + .where("traversal_ids @> ('{?}')", namespace.id) + + with_preloaded_labels + .where(parent_id: all_namespace_ids) + .select("DISTINCT ON(stage_event_hash_id) #{quoted_table_name}.*") + end + + SNOWPLOW_ATTRIBUTES = %i[ + id + created_at + updated_at + relative_position + start_event_identifier + end_event_identifier + group_id + start_event_label_id + end_event_label_id + hidden + custom + name + group_value_stream_id + ].freeze + end + end +end diff --git a/app/models/analytics/cycle_analytics/stage_event_hash.rb b/app/models/analytics/cycle_analytics/stage_event_hash.rb index 0e1e9b3ef67..6443a970945 100644 --- a/app/models/analytics/cycle_analytics/stage_event_hash.rb +++ b/app/models/analytics/cycle_analytics/stage_event_hash.rb @@ -3,7 +3,7 @@ module Analytics module CycleAnalytics class StageEventHash < ApplicationRecord - has_many :cycle_analytics_project_stages, class_name: 'Analytics::CycleAnalytics::ProjectStage', inverse_of: :stage_event_hash + has_many :cycle_analytics_stages, class_name: 'Analytics::CycleAnalytics::Stage', inverse_of: :stage_event_hash validates :hash_sha256, presence: true @@ -33,10 +33,10 @@ module Analytics end def self.unused_hashes_for(id) - exists_query = Analytics::CycleAnalytics::ProjectStage.where(stage_event_hash_id: id).select('1').limit(1) - where.not('EXISTS (?)', exists_query) + stage_exists_query = ::Analytics::CycleAnalytics::Stage.where(stage_event_hash_id: id).select('1').limit(1) + + where.not('EXISTS (?)', stage_exists_query) end end end end -Analytics::CycleAnalytics::StageEventHash.prepend_mod_with('Analytics::CycleAnalytics::StageEventHash') diff --git a/app/models/analytics/cycle_analytics/value_stream.rb b/app/models/analytics/cycle_analytics/value_stream.rb new file mode 100644 index 00000000000..3d8a0a53f5e --- /dev/null +++ b/app/models/analytics/cycle_analytics/value_stream.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class ValueStream < ApplicationRecord + self.table_name = :analytics_cycle_analytics_group_value_streams + + include Analytics::CycleAnalytics::Parentable + + has_many :stages, -> { ordered }, + class_name: 'Analytics::CycleAnalytics::Stage', + foreign_key: :group_value_stream_id, + index_errors: true, + inverse_of: :value_stream + + validates :name, presence: true + validates :name, length: { minimum: 3, maximum: 100, allow_nil: false }, uniqueness: { scope: :group_id } + + accepts_nested_attributes_for :stages, allow_destroy: true + + scope :preload_associated_models, -> { + includes(:namespace, + stages: [ + :namespace, + :end_event_label, + :start_event_label + ]) + } + + after_save :ensure_aggregation_record_presence + + def custom? + persisted? || name != Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME + end + + def self.build_default_value_stream(namespace) + new(name: Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME, namespace: namespace) + end + + private + + def ensure_aggregation_record_presence + Analytics::CycleAnalytics::Aggregation.safe_create_for_namespace(namespace) + end + end + end +end diff --git a/app/models/analytics/usage_trends/measurement.rb b/app/models/analytics/usage_trends/measurement.rb index c1245d8dce7..ddadaf78c8f 100644 --- a/app/models/analytics/usage_trends/measurement.rb +++ b/app/models/analytics/usage_trends/measurement.rb @@ -24,8 +24,8 @@ module Analytics scope :order_by_latest, -> { order(recorded_at: :desc) } scope :with_identifier, ->(identifier) { where(identifier: identifier) } - scope :recorded_after, ->(date) { where(self.model.arel_table[:recorded_at].gteq(date)) if date.present? } - scope :recorded_before, ->(date) { where(self.model.arel_table[:recorded_at].lteq(date)) if date.present? } + scope :recorded_after, ->(date) { where(model.arel_table[:recorded_at].gteq(date)) if date.present? } + scope :recorded_before, ->(date) { where(model.arel_table[:recorded_at].lteq(date)) if date.present? } def self.identifier_query_mapping { diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 3a5e06e9a1c..b926c6abedc 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -5,9 +5,13 @@ class Appearance < ApplicationRecord include CacheMarkdownField include WithUploads + ALLOWED_PWA_ICON_SCALER_WIDTHS = [192, 512].freeze + attribute :title, default: '' - attribute :pwa_short_name, default: '' attribute :description, default: '' + attribute :pwa_name, default: '' + attribute :pwa_short_name, default: '' + attribute :pwa_description, default: '' attribute :new_project_guidelines, default: '' attribute :profile_image_guidelines, default: '' attribute :header_message, default: '' @@ -22,6 +26,24 @@ class Appearance < ApplicationRecord cache_markdown_field :header_message, pipeline: :broadcast_message cache_markdown_field :footer_message, pipeline: :broadcast_message + validates :pwa_name, + length: { maximum: 255, too_long: ->(object, data) { + N_("is too long (maximum is %{count} characters)") + } }, + allow_blank: true + + validates :pwa_short_name, + length: { maximum: 255, too_long: ->(object, data) { + N_("is too long (maximum is %{count} characters)") + } }, + allow_blank: true + + validates :pwa_description, + length: { maximum: 2048, too_long: ->(object, data) { + N_("is too long (maximum is %{count} characters)") + } }, + allow_blank: true + validates :logo, file_size: { maximum: 1.megabyte } validates :pwa_icon, file_size: { maximum: 1.megabyte } validates :header_logo, file_size: { maximum: 1.megabyte } @@ -47,6 +69,12 @@ class Appearance < ApplicationRecord end end + def pwa_icon_path_scaled(width) + return unless pwa_icon_path.present? + + pwa_icon_path + "?width=#{width}" + end + def logo_path logo_system_path(logo, 'logo') end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 198a3653cd3..291375f647c 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -36,7 +36,7 @@ class ApplicationRecord < ActiveRecord::Base end def self.pluck_primary_key - where(nil).pluck(self.primary_key) + where(nil).pluck(primary_key) end def self.safe_ensure_unique(retries: 0) @@ -95,7 +95,7 @@ class ApplicationRecord < ActiveRecord::Base end def self.underscore - Gitlab::SafeRequestStore.fetch("model:#{self}:underscore") { self.to_s.underscore } + Gitlab::SafeRequestStore.fetch("model:#{self}:underscore") { to_s.underscore } end def self.where_exists(query) @@ -111,7 +111,7 @@ class ApplicationRecord < ActiveRecord::Base end def self.cached_column_list - self.column_names.map { |column_name| self.arel_table[column_name] } + column_names.map { |column_name| arel_table[column_name] } end def self.default_select_columns diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 59ad0650eb3..98adbd3ab06 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ApplicationSetting < ApplicationRecord +class ApplicationSetting < MainClusterwide::ApplicationRecord include CacheableAttributes include CacheMarkdownField include TokenAuthenticatable @@ -12,6 +12,8 @@ class ApplicationSetting < ApplicationRecord ignore_columns %i[static_objects_external_storage_auth_token], remove_with: '14.9', remove_after: '2022-03-22' ignore_column :user_email_lookup_limit, remove_with: '15.0', remove_after: '2022-04-18' ignore_column :send_user_confirmation_email, remove_with: '15.8', remove_after: '2022-12-18' + ignore_column :web_ide_clientside_preview_enabled, remove_with: '15.11', remove_after: '2023-04-22' + ignore_column :clickhouse_connection_string, remove_with: '15.11', remove_after: '2023-04-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -415,6 +417,10 @@ class ApplicationSetting < ApplicationRecord numericality: { only_integer: true, greater_than_or_equal_to: 90, message: N_("'%{value}' days of inactivity must be greater than or equal to 90") }, if: :deactivate_dormant_users? + validates :allow_possible_spam, + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + Gitlab::SSHPublicKey.supported_types.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -676,6 +682,7 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :jitsu_administrator_password, encryption_options_base_32_aes_256_gcm attr_encrypted :telesign_customer_xid, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :telesign_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :product_analytics_clickhouse_connection_string, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) validates :disable_feed_token, inclusion: { in: [true, false], message: N_('must be a boolean value') } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 8ef7e9a92a8..a5f262f2e1e 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -33,7 +33,7 @@ module ApplicationSettingImplementation DEFAULT_MINIMUM_PASSWORD_LENGTH = 8 class_methods do - def defaults + def defaults # rubocop:disable Metrics/AbcSize { admin_mode: false, after_sign_up_text: nil, @@ -41,6 +41,7 @@ module ApplicationSettingImplementation akismet_api_key: nil, allow_local_requests_from_system_hooks: true, allow_local_requests_from_web_hooks_and_services: false, + allow_possible_spam: false, asset_proxy_enabled: false, authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand commit_email_hostname: default_commit_email_hostname, @@ -105,6 +106,7 @@ module ApplicationSettingImplementation invisible_captcha_enabled: false, issues_create_limit: 300, jira_connect_application_key: nil, + jira_connect_public_key_storage_enabled: false, jira_connect_proxy_url: nil, local_markdown_version: 0, login_recaptcha_protection_enabled: false, @@ -248,7 +250,13 @@ module ApplicationSettingImplementation bulk_import_enabled: false, allow_runner_registration_token: true, user_defaults_to_private_profile: false - } + }.tap do |hsh| + hsh.merge!(non_production_defaults) unless Rails.env.production? + end + end + + def non_production_defaults + {} end def default_commit_email_hostname @@ -296,11 +304,11 @@ module ApplicationSettingImplementation end def domain_allowlist_raw - array_to_string(self.domain_allowlist) + array_to_string(domain_allowlist) end def domain_denylist_raw - array_to_string(self.domain_denylist) + array_to_string(domain_denylist) end def domain_allowlist_raw=(values) @@ -316,7 +324,7 @@ module ApplicationSettingImplementation end def outbound_local_requests_allowlist_raw - array_to_string(self.outbound_local_requests_whitelist) + array_to_string(outbound_local_requests_whitelist) end def outbound_local_requests_allowlist_raw=(values) @@ -349,7 +357,7 @@ module ApplicationSettingImplementation end def protected_paths_raw - array_to_string(self.protected_paths) + array_to_string(protected_paths) end def protected_paths_raw=(values) @@ -357,7 +365,7 @@ module ApplicationSettingImplementation end def notes_create_limit_allowlist_raw - array_to_string(self.notes_create_limit_allowlist) + array_to_string(notes_create_limit_allowlist) end def notes_create_limit_allowlist_raw=(values) @@ -365,7 +373,7 @@ module ApplicationSettingImplementation end def users_get_by_id_limit_allowlist_raw - array_to_string(self.users_get_by_id_limit_allowlist) + array_to_string(users_get_by_id_limit_allowlist) end def users_get_by_id_limit_allowlist_raw=(values) @@ -516,12 +524,6 @@ module ApplicationSettingImplementation static_objects_external_storage_url.present? end - # This will eventually be configurable - # https://gitlab.com/gitlab-org/gitlab/issues/208161 - def web_ide_clientside_preview_bundler_url - 'https://sandbox-prod.gitlab-static.net' - end - def ensure_key_restrictions! return if Gitlab::Database.read_only? return unless Gitlab::FIPS.enabled? @@ -535,7 +537,7 @@ module ApplicationSettingImplementation def set_max_key_restriction!(key_type) attr_name = "#{key_type}_key_restriction" - current = self.attributes[attr_name].to_i + current = attributes[attr_name].to_i return if current == KeyRestrictionValidator::FORBIDDEN @@ -548,7 +550,7 @@ module ApplicationSettingImplementation [min_size, current].max end - self.assign_attributes({ attr_name => new_value }) + assign_attributes({ attr_name => new_value }) end def separate_allowlists(string_array) diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 5cc87be388f..3312216932b 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -55,7 +55,7 @@ class AuditEvent < ApplicationRecord end def initialize_details - return unless self.has_attribute?(:details) + return unless has_attribute?(:details) self.details = {} if details&.nil? end @@ -65,7 +65,9 @@ class AuditEvent < ApplicationRecord end def formatted_details - details.merge(details.slice(:from, :to).transform_values(&:to_s)) + details + .merge(details.slice(:from, :to).transform_values(&:to_s)) + .merge(author_email: author.try(:email)) end def author @@ -74,7 +76,7 @@ class AuditEvent < ApplicationRecord def lazy_author BatchLoader.for(author_id).batch do |author_ids, loader| - User.select(:id, :name, :username).where(id: author_ids).find_each do |user| + User.select(:id, :name, :username, :email).where(id: author_ids).find_each do |user| loader.call(user.id, user) end end @@ -82,7 +84,7 @@ class AuditEvent < ApplicationRecord def as_json(options = {}) super(options).tap do |json| - json['ip_address'] = self.ip_address.to_s + json['ip_address'] = ip_address.to_s end end @@ -114,10 +116,10 @@ class AuditEvent < ApplicationRecord def parallel_persist PARALLEL_PERSISTENCE_COLUMNS.each do |name| - original = self[name] || self.details[name] + original = self[name] || details[name] next unless original - self[name] = self.details[name] = original + self[name] = details[name] = original end end diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index f41f0a8be84..dbc5c7a584e 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -55,11 +55,11 @@ class AwardEmoji < ApplicationRecord end def downvote? - self.name == DOWNVOTE_NAME + name == DOWNVOTE_NAME end def upvote? - self.name == UPVOTE_NAME + name == UPVOTE_NAME end def url diff --git a/app/models/board.rb b/app/models/board.rb index 8a7330e7320..2181b2f0545 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -18,7 +18,7 @@ class Board < ApplicationRecord # Sort by case-insensitive name, then ascending ids. This ensures that we will always # get the same list/first board no matter how many other boards are named the same scope :order_by_name_asc, -> { order(arel_table[:name].lower.asc).order(id: :asc) } - scope :first_board, -> { where(id: self.order_by_name_asc.limit(1).select(:id)) } + scope :first_board, -> { where(id: order_by_name_asc.limit(1).select(:id)) } def project_needed? !group diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index ebca5e90313..6fc24c77f1d 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -39,9 +39,28 @@ class BulkImports::Entity < ApplicationRecord validates :project, absence: true, if: :group validates :group, absence: true, if: :project - validates :source_type, :source_full_path, :destination_name, presence: true - validates :destination_namespace, exclusion: [nil], if: :group - validates :destination_namespace, presence: true, if: :project + validates :source_type, presence: true + validates :source_full_path, + presence: true, + format: { with: Gitlab::Regex.bulk_import_source_full_path_regex, + message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message } + + validates :destination_name, + presence: true, + format: { with: Gitlab::Regex.group_path_regex, + message: Gitlab::Regex.group_path_regex_message } + + validates :destination_namespace, + exclusion: [nil], + format: { with: Gitlab::Regex.bulk_import_destination_namespace_path_regex, + message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message }, + if: :group + + validates :destination_namespace, + presence: true, + format: { with: Gitlab::Regex.bulk_import_destination_namespace_path_regex, + message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message }, + if: :project validate :validate_parent_is_a_group, if: :parent validate :validate_imported_entity_type @@ -57,6 +76,10 @@ class BulkImports::Entity < ApplicationRecord alias_attribute :destination_slug, :destination_name + delegate :default_project_visibility, + :default_group_visibility, + to: :'Gitlab::CurrentSettings.current_application_settings' + state_machine :status, initial: :created do state :created, value: 0 state :started, value: 1 @@ -156,6 +179,12 @@ class BulkImports::Entity < ApplicationRecord project? ? project&.full_path : group&.full_path end + def default_visibility_level + return default_group_visibility if group? + + default_project_visibility + end + private def validate_parent_is_a_group diff --git a/app/models/ci/application_record.rb b/app/models/ci/application_record.rb index ea7b1104e36..52f02bfb2fd 100644 --- a/app/models/ci/application_record.rb +++ b/app/models/ci/application_record.rb @@ -13,7 +13,7 @@ module Ci end def self.model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize) + @model_name ||= ActiveModel::Name.new(self, nil, name.demodulize) end end end diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 4af31fd37f2..697f06fbffd 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -55,7 +55,11 @@ module Ci end def retryable? - false + return false unless Feature.enabled?(:ci_recreate_downstream_pipeline, project) + + return false if failed? && (pipeline_loop_detected? || reached_max_descendant_pipelines_depth?) + + super end def self.with_preloads @@ -76,9 +80,9 @@ module Ci def inherit_status_from_downstream!(pipeline) case pipeline.status when 'success' - self.success! + success! when 'failed', 'canceled', 'skipped' - self.drop! + drop! else false end @@ -186,6 +190,10 @@ module Ci def persisted_environment end + def deployment_job? + false + end + def execute_hooks raise NotImplementedError end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 0139b025d98..f8b3777841d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -34,11 +34,11 @@ module Ci DEPLOYMENT_NAMES = %w[deploy release rollout].freeze has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable - has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build + has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build - has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build + has_many :report_results, class_name: 'Ci::BuildReportResult', foreign_key: :build_id, inverse_of: :build has_one :namespace, through: :project # Projects::DestroyService destroys Ci::Pipelines, which use_fast_destroy on :job_artifacts @@ -49,16 +49,18 @@ module Ci has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id, inverse_of: :job has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id - has_many :pages_deployments, inverse_of: :ci_build + has_many :pages_deployments, foreign_key: :ci_build_id, inverse_of: :ci_build Ci::JobArtifact.file_types.each do |key, value| - has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', foreign_key: :job_id, inverse_of: :job end - has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build - has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build + has_one :runner_machine, through: :metadata, class_name: 'Ci::RunnerMachine' - has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', inverse_of: :build, foreign_key: :ci_build_id + has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, foreign_key: :build_id, inverse_of: :build + has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', foreign_key: :build_id, inverse_of: :build + + has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', foreign_key: :ci_build_id, inverse_of: :build accepts_nested_attributes_for :runner_session, update_only: true accepts_nested_attributes_for :job_variables @@ -88,6 +90,12 @@ module Ci scope :unstarted, -> { where(runner_id: nil) } + scope :with_any_artifacts, -> do + where('EXISTS (?)', + Ci::JobArtifact.select(1).where("#{Ci::Build.quoted_table_name}.id = #{Ci::JobArtifact.quoted_table_name}.job_id") + ) + end + scope :with_downloadable_artifacts, -> do where('EXISTS (?)', Ci::JobArtifact.select(1) @@ -179,6 +187,8 @@ module Ci run_after_commit { build.execute_hooks } end + after_commit :track_ci_secrets_management_id_tokens_usage, on: :create, if: :id_tokens? + class << self # This is needed for url_for to work, # as the controller is JobsController @@ -382,21 +392,21 @@ module Ci def detailed_status(current_user) Gitlab::Ci::Status::Build::Factory - .new(self.present, current_user) + .new(present, current_user) .fabricate! end def other_manual_actions - pipeline.manual_actions.reject { |action| action.name == self.name } + pipeline.manual_actions.reject { |action| action.name == name } end def other_scheduled_actions - pipeline.scheduled_actions.reject { |action| action.name == self.name } + pipeline.scheduled_actions.reject { |action| action.name == name } end def pages_generator? Gitlab.config.pages.enabled && - self.name == 'pages' + name == 'pages' end def runnable? @@ -452,7 +462,7 @@ module Ci end def retries_count - pipeline.builds.retried.where(name: self.name).count + pipeline.builds.retried.where(name: name).count end override :all_met_to_become_pending? @@ -525,19 +535,19 @@ module Ci end def deployment_job? - has_environment_keyword? && self.environment_action == 'start' + has_environment_keyword? && environment_action == 'start' end def stops_environment? - has_environment_keyword? && self.environment_action == 'stop' + has_environment_keyword? && environment_action == 'stop' end def environment_action - self.options.fetch(:environment, {}).fetch(:action, 'start') if self.options + options.fetch(:environment, {}).fetch(:action, 'start') if options end def environment_tier_from_options - self.options.dig(:environment, :deployment_tier) if self.options + options.dig(:environment, :deployment_tier) if options end def environment_tier @@ -827,7 +837,7 @@ module Ci end def erased? - !self.erased_at.nil? + !erased_at.nil? end def artifacts_expired? @@ -860,14 +870,14 @@ module Ci end def keep_artifacts! - self.update(artifacts_expire_at: nil) - self.job_artifacts.update_all(expire_at: nil) + update(artifacts_expire_at: nil) + job_artifacts.update_all(expire_at: nil) end - def artifacts_file_for_type(type) + def artifact_for_type(type) file_types = Ci::JobArtifact.associated_file_types_for(type) file_types_ids = file_types&.map { |file_type| Ci::JobArtifact.file_types[file_type] } - job_artifacts.find_by(file_type: file_types_ids)&.file + job_artifacts.find_by(file_type: file_types_ids) end def steps @@ -1092,11 +1102,11 @@ module Ci # without actually loading data. # def all_queuing_entries - ::Ci::PendingBuild.where(build_id: self.id) + ::Ci::PendingBuild.where(build_id: id) end def all_runtime_metadata - ::Ci::RunningBuild.where(build_id: self.id) + ::Ci::RunningBuild.where(build_id: id) end def shared_runner_build? @@ -1281,6 +1291,23 @@ module Ci .increment(status: status) end end + + def track_ci_secrets_management_id_tokens_usage + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('i_ci_secrets_management_id_tokens_build_created', values: user_id) + + Gitlab::Tracking.event( + self.class.to_s, + 'create_id_tokens', + namespace: namespace, + user: user, + label: 'redis_hll_counters.ci_secrets_management.i_ci_secrets_management_id_tokens_build_created_monthly', + ultimate_namespace_id: namespace.root_ancestor.id, + context: [Gitlab::Tracking::ServicePingContext.new( + data_source: :redis_hll, + event: 'i_ci_secrets_management_id_tokens_build_created' + ).to_context] + ) + end end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 1dcb9190f11..b294afd405d 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -18,6 +18,7 @@ module Ci belongs_to :build, class_name: 'CommitStatus' belongs_to :project + belongs_to :runner_machine, class_name: 'Ci::RunnerMachine' before_create :set_build_project @@ -67,7 +68,7 @@ module Ci private def set_build_project - self.project_id ||= self.build.project_id + self.project_id ||= build.project_id end def timeout_with_highest_precedence diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index 3fa17d6d286..03d1bd14bfb 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -4,6 +4,9 @@ module Ci class BuildNeed < Ci::ApplicationRecord include Ci::Partitionable include BulkInsertSafe + include IgnorableColumns + + ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-04-22' belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb index 20c0b04e228..5773b6132be 100644 --- a/app/models/ci/build_runner_session.rb +++ b/app/models/ci/build_runner_session.rb @@ -20,7 +20,7 @@ module Ci validates :url, public_url: { schemes: %w(https) } def terminal_specification - wss_url = Gitlab::UrlHelpers.as_wss(Addressable::URI.escape(self.url)) + wss_url = Gitlab::UrlHelpers.as_wss(Addressable::URI.escape(url)) return {} unless wss_url.present? parsed_wss_url = URI.parse(wss_url) @@ -33,7 +33,7 @@ module Ci port = port.presence || DEFAULT_PORT_NAME service = service.presence || DEFAULT_SERVICE_NAME - parsed_url = URI.parse(Addressable::URI.escape(self.url)) + parsed_url = URI.parse(Addressable::URI.escape(url)) parsed_url.path += "/proxy/#{service}/#{port}/#{path}" subprotocols = subprotocols.presence || ::Ci::BuildRunnerSession::TERMINAL_SUBPROTOCOL diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index c5f6e54c336..541a8b5bffa 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -108,7 +108,7 @@ module Ci raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 return if offset == size # Skip the following process as it doesn't affect anything - self.append(+"", offset) + append(+"", offset) end def append(new_data, offset) @@ -166,7 +166,7 @@ module Ci raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save? self.class.with_read_consistency(build) do - self.reset.then(&:unsafe_persist_data!) + reset.then(&:unsafe_persist_data!) end end rescue FailedToObtainLockError @@ -205,9 +205,9 @@ module Ci end def <=>(other) - return unless self.build_id == other.build_id + return unless build_id == other.build_id - self.chunk_index <=> other.chunk_index + chunk_index <=> other.chunk_index end protected diff --git a/app/models/ci/deleted_object.rb b/app/models/ci/deleted_object.rb index d36646aba66..2b5452c803a 100644 --- a/app/models/ci/deleted_object.rb +++ b/app/models/ci/deleted_object.rb @@ -21,7 +21,7 @@ module Ci accumulator << record if record[:store_dir] && record[:file] end - self.insert_all(attributes) if attributes.any? + insert_all(attributes) if attributes.any? end def delete_file_from_storage diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 508aaa5a63c..b03c46a164f 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -3,9 +3,11 @@ module Ci class GroupVariable < Ci::ApplicationRecord include Ci::HasVariable - include Presentable include Ci::Maskable include Ci::RawVariable + include Limitable + include Presentable + prepend HasEnvironmentScope belongs_to :group, class_name: "::Group" @@ -21,6 +23,9 @@ module Ci scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } scope :for_groups, ->(group_ids) { where(group_id: group_ids) } + self.limit_name = 'group_ci_variables' + self.limit_scope = :group + def audit_details key end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 0dca5b18a24..89a3d269a43 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -134,15 +134,17 @@ module Ci belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id - # We will start using this column once we complete https://gitlab.com/gitlab-org/gitlab/-/issues/285597 - ignore_column :original_filename, remove_with: '14.7', remove_after: '2022-11-22' - mount_file_store_uploader JobArtifactUploader, skip_store_file: true before_save :set_size, if: :file_changed? after_save :store_file_in_transaction!, unless: :store_after_commit? + + after_create_commit :log_create + after_commit :store_file_after_transaction!, on: [:create, :update], if: :store_after_commit? + after_destroy_commit :log_destroy + validates :job, presence: true validates :file_format, presence: true, unless: :trace?, on: :create validate :validate_file_format!, unless: :trace?, on: :create @@ -384,6 +386,14 @@ module Ci # Use job.project to avoid extra DB query for project job.project.pending_delete? end + + def log_create + Gitlab::Ci::Artifacts::Logger.log_created(self) + end + + def log_destroy + Gitlab::Ci::Artifacts::Logger.log_deleted(self, __method__) + end end end diff --git a/app/models/ci/job_token/allowlist.rb b/app/models/ci/job_token/allowlist.rb index 9e9a0a68ebd..618dc2da05c 100644 --- a/app/models/ci/job_token/allowlist.rb +++ b/app/models/ci/job_token/allowlist.rb @@ -17,6 +17,15 @@ module Ci Project.from_union(target_projects, remove_duplicates: false) end + def add!(target_project, user:) + Ci::JobToken::ProjectScopeLink.create!( + source_project: @source_project, + direction: @direction, + target_project: target_project, + added_by: user + ) + end + private def source_links diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index b784f93651a..96e370bba1e 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -1,24 +1,31 @@ # frozen_string_literal: true -# The connection between a source project (which defines the job token scope) -# and a target project which is the one allowed to be accessed by the job token. +# The connection between a source project (which the job token scope's allowlist applies too) +# and a target project which is added to the scope's allowlist. module Ci module JobToken class ProjectScopeLink < Ci::ApplicationRecord self.table_name = 'ci_job_token_project_scope_links' + PROJECT_LINK_DIRECTIONAL_LIMIT = 100 + belongs_to :source_project, class_name: 'Project' + # the project added to the scope's allowlist belongs_to :target_project, class_name: 'Project' belongs_to :added_by, class_name: 'User' - scope :with_source, ->(project) { where(source_project: project) } - scope :with_target, ->(project) { where(target_project: project) } + scope :with_access_direction, ->(direction) { where(direction: direction) } + scope :with_source, ->(project) { where(source_project: project) } + scope :with_target, ->(project) { where(target_project: project) } validates :source_project, presence: true validates :target_project, presence: true validate :not_self_referential_link + validate :source_project_under_link_limit, on: :create + # When outbound the target project is allowed to be accessed by the source job token. + # When inbound the source project is allowed to be accessed by the target job token. enum direction: { outbound: 0, inbound: 1 @@ -37,6 +44,16 @@ module Ci self.errors.add(:target_project, _("can't be the same as the source project")) end end + + def source_project_under_link_limit + return unless source_project + + existing_links_count = self.class.with_source(source_project).with_access_direction(direction).count + + if existing_links_count >= PROJECT_LINK_DIRECTIONAL_LIMIT + errors.add(:source_project, "exceeds the allowable number of project links in this direction") + end + end end end end diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb index e320c0f92d1..20775077bd8 100644 --- a/app/models/ci/job_token/scope.rb +++ b/app/models/ci/job_token/scope.rb @@ -2,18 +2,17 @@ # This model represents the scope of access for a CI_JOB_TOKEN. # -# A scope is initialized with a project. +# A scope is initialized with a current project. # # Projects can be added to the scope by adding ScopeLinks to # create an allowlist of projects in either access direction (inbound, outbound). # -# Currently, projects in the outbound allowlist can be accessed via the token -# in the source project. +# Projects in the outbound allowlist can be accessed via the current project's job token. # -# TODO(Issue #346298) Projects in the inbound allowlist can use their token to access -# the source project. +# Projects in the inbound allowlist can use their project's job token to +# access the current project. # -# CI_JOB_TOKEN should be considered untrusted without these features enabled. +# CI_JOB_TOKEN should be considered untrusted without a scope enabled. # module Ci @@ -25,34 +24,70 @@ module Ci @current_project = current_project end - def allows?(accessed_project) - self_referential?(accessed_project) || outbound_allows?(accessed_project) + def accessible?(accessed_project) + self_referential?(accessed_project) || ( + outbound_accessible?(accessed_project) && + inbound_accessible?(accessed_project) + ) end def outbound_projects outbound_allowlist.projects end - # Deprecated: use outbound_projects, TODO(Issue #346298) remove references to all_project - def all_projects - outbound_projects + def inbound_projects + inbound_allowlist.projects + end + + def add!(added_project, user:, direction:) + case direction + when :inbound + inbound_allowlist.add!(added_project, user: user) + when :outbound + outbound_allowlist.add!(added_project, user: user) + end end private - def outbound_allows?(accessed_project) + def outbound_accessible?(accessed_project) # if the setting is disabled any project is considered to be in scope. - return true unless @current_project.ci_outbound_job_token_scope_enabled? + return true unless current_project.ci_outbound_job_token_scope_enabled? outbound_allowlist.includes?(accessed_project) end + def inbound_accessible?(accessed_project) + # if the flag or setting is disabled any project is considered to be in scope. + return true unless Feature.enabled?(:ci_inbound_job_token_scope, accessed_project) + return true unless accessed_project.ci_inbound_job_token_scope_enabled? + + inbound_linked_as_accessible?(accessed_project) + end + + # We don't check the inbound allowlist here. That is because + # the access check starts from the current project but the inbound + # allowlist contains projects that can access the current project. + def inbound_linked_as_accessible?(accessed_project) + inbound_accessible_projects(accessed_project).includes?(current_project) + end + + def inbound_accessible_projects(accessed_project) + Ci::JobToken::Allowlist.new(accessed_project, direction: :inbound) + end + + # User created list of projects allowed to access the current project + def inbound_allowlist + Ci::JobToken::Allowlist.new(current_project, direction: :inbound) + end + + # User created list of projects that can be accessed from the current project def outbound_allowlist - Ci::JobToken::Allowlist.new(@current_project, direction: :outbound) + Ci::JobToken::Allowlist.new(current_project, direction: :outbound) end def self_referential?(accessed_project) - @current_project.id == accessed_project.id + current_project.id == accessed_project.id end end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index eab2ab69e44..bd426e02b9c 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -355,7 +355,7 @@ module Ci scope :for_name, -> (name) do name_column = Ci::PipelineMetadata.arel_table[:name] - joins(:pipeline_metadata).where(name_column.lower.eq(name.downcase)) + joins(:pipeline_metadata).where(name_column.eq(name)) end scope :created_after, -> (time) { where(arel_table[:created_at].gt(time)) } scope :created_before_id, -> (id) { where(arel_table[:id].lt(id)) } @@ -498,6 +498,10 @@ module Ci 100 end + def self.object_hierarchy(relation, options = {}) + ::Gitlab::Ci::PipelineObjectHierarchy.new(relation, options: options) + end + def uses_needs? processables.where(scheduling_type: :dag).any? end @@ -841,97 +845,6 @@ module Ci end end - def predefined_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_PIPELINE_IID', value: iid.to_s) - variables.append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) - variables.append(key: 'CI_PIPELINE_CREATED_AT', value: created_at&.iso8601) - - variables.concat(predefined_commit_variables) - variables.concat(predefined_merge_request_variables) - - if open_merge_requests_refs.any? - variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: open_merge_requests_refs.join(',')) - end - - variables.append(key: 'CI_GITLAB_FIPS_MODE', value: 'true') if Gitlab::FIPS.enabled? - - variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if has_kubernetes_active? - variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if freeze_period? - - if external_pull_request_event? && external_pull_request - variables.concat(external_pull_request.predefined_variables) - end - end - end - - def predefined_commit_variables - strong_memoize(:predefined_commit_variables) do - Gitlab::Ci::Variables::Collection.new.tap do |variables| - next variables unless sha.present? - - variables.append(key: 'CI_COMMIT_SHA', value: sha) - variables.append(key: 'CI_COMMIT_SHORT_SHA', value: short_sha) - variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: before_sha) - variables.append(key: 'CI_COMMIT_REF_NAME', value: source_ref) - variables.append(key: 'CI_COMMIT_REF_SLUG', value: source_ref_slug) - variables.append(key: 'CI_COMMIT_BRANCH', value: ref) if branch? - variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) - variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) - variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) - variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s) - variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s) - variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s) - - # legacy variables - variables.append(key: 'CI_BUILD_REF', value: sha) - variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) - variables.append(key: 'CI_BUILD_REF_NAME', value: source_ref) - variables.append(key: 'CI_BUILD_REF_SLUG', value: source_ref_slug) - - variables.concat(predefined_commit_tag_variables) - end - end - end - - def predefined_merge_request_variables - strong_memoize(:predefined_merge_request_variables) do - Gitlab::Ci::Variables::Collection.new.tap do |variables| - next variables unless merge_request? - - variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: merge_request_event_type.to_s) - variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: source_sha.to_s) - variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: target_sha.to_s) - - diff = self.merge_request_diff - if diff.present? - variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: diff.id.to_s) - variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: diff.base_commit_sha) - end - - variables.concat(merge_request.predefined_variables) - end - end - end - - def predefined_commit_tag_variables - strong_memoize(:predefined_commit_ref_variables) do - Gitlab::Ci::Variables::Collection.new.tap do |variables| - next variables unless tag? - - git_tag = project.repository.find_tag(ref) - - next variables unless git_tag - - variables.append(key: 'CI_COMMIT_TAG', value: ref) - variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) - - # legacy variable - variables.append(key: 'CI_BUILD_TAG', value: ref) - end - end - end - def queued_duration return unless started_at @@ -1403,6 +1316,12 @@ module Ci (Time.current - created_at).ceil / 60 end + def merge_request_diff + return unless merge_request? + + merge_request.merge_request_diff_for(merge_request_diff_sha) + end + private def cancel_jobs(jobs, retries: 1, auto_canceled_by_pipeline_id: nil) @@ -1455,12 +1374,6 @@ module Ci end end - def merge_request_diff - return unless merge_request? - - merge_request.merge_request_diff_for(merge_request_diff_sha) - end - def push_details strong_memoize(:push_details) do Gitlab::Git::Push.new(project, before_sha, sha, git_ref) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index bac85b6095e..09ac0fa69e7 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -15,6 +15,8 @@ module Ci include EachBatch include Ci::HasRunnerExecutor + extend ::Gitlab::Utils::Override + add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration enum access_level: { @@ -28,6 +30,14 @@ module Ci project_type: 3 } + enum registration_type: { + registration_token: 0, + authenticated_user: 1 + }, _suffix: true + + # Prefix assigned to runners created from the UI, instead of registered via the command line + CREATED_RUNNER_TOKEN_PREFIX = 'glrt-' + # This `ONLINE_CONTACT_TIMEOUT` needs to be larger than # `RUNNER_QUEUE_EXPIRY_TIME+UPDATE_CONTACT_COLUMN_EVERY` # @@ -179,6 +189,7 @@ module Ci validate :tag_constraints validates :access_level, presence: true validates :runner_type, presence: true + validates :registration_type, presence: true validate :no_projects, unless: :project_type? validate :no_groups, unless: :group_type? @@ -373,7 +384,10 @@ module Ci end def short_sha - token[0...8] if token + return unless token + + start_index = authenticated_user_registration_type? ? CREATED_RUNNER_TOKEN_PREFIX.length : 0 + token[start_index..start_index + 8] end def tag_list @@ -474,6 +488,17 @@ module Ci end end + override :format_token + def format_token(token) + return token if registration_token_registration_type? + + "#{CREATED_RUNNER_TOKEN_PREFIX}#{token}" + end + + def ensure_machine(system_xid, &blk) + RunnerMachine.safe_find_or_create_by!(runner_id: id, system_xid: system_xid.to_s, &blk) # rubocop: disable Performance/ActiveRecordSubtransactionMethods + end + private scope :with_upgrade_status, ->(upgrade_status) do @@ -566,6 +591,9 @@ module Ci end end + # TODO Remove in 16.0 when runners are known to send a system_id + # For now, heartbeats with version updates might result in two Sidekiq jobs being queued if a runner has a system_id + # This is not a problem since the jobs are deduplicated on the version def schedule_runner_version_update return unless version diff --git a/app/models/ci/runner_machine.rb b/app/models/ci/runner_machine.rb index 1dd997a8ee1..e52659a011f 100644 --- a/app/models/ci/runner_machine.rb +++ b/app/models/ci/runner_machine.rb @@ -3,12 +3,24 @@ module Ci class RunnerMachine < Ci::ApplicationRecord include FromUnion + include RedisCacheable include Ci::HasRunnerExecutor + include IgnorableColumns + + ignore_column :machine_xid, remove_with: '15.11', remove_after: '2022-03-22' + + # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner Machine DB entry can be updated + UPDATE_CONTACT_COLUMN_EVERY = 40.minutes..55.minutes belongs_to :runner + has_many :build_metadata, class_name: 'Ci::BuildMetadata' + has_many :builds, through: :build_metadata, class_name: 'Ci::Build' + belongs_to :runner_version, inverse_of: :runner_machines, primary_key: :version, foreign_key: :version, + class_name: 'Ci::RunnerVersion' + validates :runner, presence: true - validates :machine_xid, presence: true, length: { maximum: 64 } + validates :system_xid, presence: true, length: { maximum: 64 } validates :version, length: { maximum: 2048 } validates :revision, length: { maximum: 255 } validates :platform, length: { maximum: 255 } @@ -16,6 +28,8 @@ module Ci validates :ip_address, length: { maximum: 1024 } validates :config, json_schema: { filename: 'ci_runner_config' } + cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type + # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner machine # will be considered stale STALE_TIMEOUT = 7.days @@ -29,5 +43,46 @@ module Ci where(contacted_some_time_ago), remove_duplicates: false).where(created_some_time_ago) end + + def heartbeat(values) + ## + # We can safely ignore writes performed by a runner heartbeat. We do + # not want to upgrade database connection proxy to use the primary + # database after heartbeat write happens. + # + ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do + values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} + values[:contacted_at] = Time.current + if values.include?(:executor) + values[:executor_type] = Ci::Runner::EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) + end + + version_changed = values.include?(:version) && values[:version] != version + + cache_attributes(values) + + schedule_runner_version_update if version_changed + + # We save data without validation, it will always change due to `contacted_at` + update_columns(values) if persist_cached_data? + end + end + + private + + def persist_cached_data? + # Use a random threshold to prevent beating DB updates. + contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY) + + real_contacted_at = read_attribute(:contacted_at) + real_contacted_at.nil? || + (Time.current - real_contacted_at) >= contacted_at_max_age + end + + def schedule_runner_version_update + return unless version + + Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) + end end end diff --git a/app/models/ci/runner_version.rb b/app/models/ci/runner_version.rb index bbde98ee591..ec42f46b165 100644 --- a/app/models/ci/runner_version.rb +++ b/app/models/ci/runner_version.rb @@ -8,24 +8,23 @@ module Ci enum_with_nil status: { not_processed: nil, invalid_version: -1, - not_available: 1, + unavailable: 1, available: 2, recommended: 3 } STATUS_DESCRIPTIONS = { invalid_version: 'Runner version is not valid.', - not_available: 'Upgrade is not available for the runner.', + unavailable: 'Upgrade is not available for the runner.', available: 'Upgrade is available for the runner.', recommended: 'Upgrade is available and recommended for the runner.' }.freeze - # Override auto generated negative scope (from available) so the scope has expected behavior - scope :not_available, -> { where(status: :not_available) } + has_many :runner_machines, inverse_of: :runner_version, foreign_key: :version, class_name: 'Ci::RunnerMachine' # This scope returns all versions that might need recalculating. For instance, once a version is considered # :recommended, it normally doesn't change status even if the instance is upgraded - scope :potentially_outdated, -> { where(status: [nil, :not_available, :available]) } + scope :potentially_outdated, -> { where(status: [nil, :unavailable, :available]) } validates :version, length: { maximum: 2048 } end diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 1e6c48bbef5..5e273e0fd4b 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -35,7 +35,7 @@ module Ci end def file_extension - File.extname(name).delete_prefix('.') + File.extname(name).delete_prefix('.').presence end def metadata_parsable? diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 1092b9c9564..1b2a7dc3fe4 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -21,8 +21,18 @@ module Ci validates :token, presence: true, uniqueness: true validates :owner, presence: true + attr_encrypted :encrypted_token_tmp, + attribute: :encrypted_token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_vi: false + before_validation :set_default_values + before_save :copy_token_to_encrypted_token + def set_default_values self.token = "#{TRIGGER_TOKEN_PREFIX}#{SecureRandom.hex(20)}" if self.token.blank? end @@ -42,6 +52,12 @@ module Ci def can_access_project? Ability.allowed?(self.owner, :create_build, project) end + + private + + def copy_token_to_encrypted_token + self.encrypted_token_tmp = token + end end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index f4e17b5d812..23fe89c38df 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -3,9 +3,11 @@ module Ci class Variable < Ci::ApplicationRecord include Ci::HasVariable - include Presentable include Ci::Maskable include Ci::RawVariable + include Limitable + include Presentable + prepend HasEnvironmentScope belongs_to :project @@ -20,6 +22,9 @@ module Ci scope :unprotected, -> { where(protected: false) } scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } + self.limit_name = 'project_ci_variables' + self.limit_scope = :project + def audit_details key end diff --git a/app/models/clusters/applications/cert_manager.rb b/app/models/clusters/applications/cert_manager.rb deleted file mode 100644 index 11f84940c38..00000000000 --- a/app/models/clusters/applications/cert_manager.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - # DEPRECATED for removal in %14.0 - # See https://gitlab.com/groups/gitlab-org/-/epics/4280 - class CertManager < ApplicationRecord - VERSION = 'v0.10.1' - CRD_VERSION = '0.10' - - self.table_name = 'clusters_applications_cert_managers' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - - attribute :version, default: VERSION - after_initialize :set_default_email, if: :new_record? - - validates :email, presence: true - - def chart - 'certmanager/cert-manager' - end - - def repository - 'https://charts.jetstack.io' - end - - def install_command - helm_command_module::InstallCommand.new( - name: 'certmanager', - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files.merge(cluster_issuer_file), - preinstall: pre_install_script, - postinstall: post_install_script - ) - end - - def uninstall_command - helm_command_module::DeleteCommand.new( - name: 'certmanager', - rbac: cluster.platform_kubernetes_rbac?, - files: files, - postdelete: post_delete_script - ) - end - - private - - def set_default_email - self.email ||= self.cluster&.user&.email - end - - def pre_install_script - [ - apply_file("https://raw.githubusercontent.com/jetstack/cert-manager/release-#{CRD_VERSION}/deploy/manifests/00-crds.yaml"), - "kubectl label --overwrite namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} certmanager.k8s.io/disable-validation=true" - ] - end - - def post_install_script - [retry_command(apply_file('/data/helm/certmanager/config/cluster_issuer.yaml'))] - end - - def retry_command(command) - Gitlab::Kubernetes::PodCmd.retry_command(command, times: 90) - end - - def post_delete_script - [ - delete_private_key, - delete_crd('certificates.certmanager.k8s.io'), - delete_crd('certificaterequests.certmanager.k8s.io'), - delete_crd('challenges.certmanager.k8s.io'), - delete_crd('clusterissuers.certmanager.k8s.io'), - delete_crd('issuers.certmanager.k8s.io'), - delete_crd('orders.certmanager.k8s.io') - ].compact - end - - def private_key_name - @private_key_name ||= cluster_issuer_content.dig('spec', 'acme', 'privateKeySecretRef', 'name') - end - - def delete_private_key - return unless private_key_name.present? - - args = %W(secret -n #{Gitlab::Kubernetes::Helm::NAMESPACE} #{private_key_name} --ignore-not-found) - - Gitlab::Kubernetes::KubectlCmd.delete(*args) - end - - def delete_crd(definition) - Gitlab::Kubernetes::KubectlCmd.delete("crd", definition, "--ignore-not-found") - end - - def apply_file(filename) - Gitlab::Kubernetes::KubectlCmd.apply_file(filename) - end - - def cluster_issuer_file - { - 'cluster_issuer.yaml': cluster_issuer_yaml_content - } - end - - def cluster_issuer_yaml_content - YAML.dump(cluster_issuer_content.deep_merge(cluster_issue_overlay)) - end - - def cluster_issuer_content - YAML.safe_load(File.read(cluster_issuer_file_path)) - end - - def cluster_issue_overlay - { "spec" => { "acme" => { "email" => self.email } } } - end - - def cluster_issuer_file_path - Rails.root.join('vendor', 'cert_manager', 'cluster_issuer.yaml') - end - end - end -end diff --git a/app/models/clusters/applications/cilium.rb b/app/models/clusters/applications/cilium.rb deleted file mode 100644 index 7936b0b18de..00000000000 --- a/app/models/clusters/applications/cilium.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - class Cilium < ApplicationRecord - self.table_name = 'clusters_applications_cilium' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - - # Cilium can only be installed and uninstalled through the - # cluster-applications project by triggering CI pipeline for a - # management project. UI operations are not available for such - # applications. More information: - # https://docs.gitlab.com/ee/user/clusters/management_project.html - def allowed_to_uninstall? - false - end - end - end -end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 25d41d68b9e..a35ea6ddb46 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -14,13 +14,11 @@ module Clusters APPLICATIONS = { Clusters::Applications::Helm.application_name => Clusters::Applications::Helm, Clusters::Applications::Ingress.application_name => Clusters::Applications::Ingress, - Clusters::Applications::CertManager.application_name => Clusters::Applications::CertManager, Clusters::Applications::Crossplane.application_name => Clusters::Applications::Crossplane, Clusters::Applications::Prometheus.application_name => Clusters::Applications::Prometheus, Clusters::Applications::Runner.application_name => Clusters::Applications::Runner, Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter, - Clusters::Applications::Knative.application_name => Clusters::Applications::Knative, - Clusters::Applications::Cilium.application_name => Clusters::Applications::Cilium + Clusters::Applications::Knative.application_name => Clusters::Applications::Knative }.freeze DEFAULT_ENVIRONMENT = '*' KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN' @@ -58,13 +56,11 @@ module Clusters has_one_cluster_application :helm has_one_cluster_application :ingress - has_one_cluster_application :cert_manager has_one_cluster_application :crossplane has_one_cluster_application :prometheus has_one_cluster_application :runner has_one_cluster_application :jupyter has_one_cluster_application :knative - has_one_cluster_application :cilium has_many :kubernetes_namespaces has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :cluster @@ -91,15 +87,7 @@ module Clusters delegate :status, to: :provider, allow_nil: true delegate :status_reason, to: :provider, allow_nil: true - delegate :on_creation?, to: :provider, allow_nil: true - delegate :knative_pre_installed?, to: :provider, allow_nil: true - - delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true - delegate :rbac?, to: :platform_kubernetes, prefix: true, allow_nil: true - delegate :available?, to: :application_helm, prefix: true, allow_nil: true - delegate :available?, to: :application_ingress, prefix: true, allow_nil: true - delegate :available?, to: :application_knative, prefix: true, allow_nil: true - delegate :available?, to: :integration_prometheus, prefix: true, allow_nil: true + delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true @@ -245,7 +233,7 @@ module Clusters end def persisted_applications - APPLICATIONS_ASSOCIATIONS.map(&method(:public_send)).compact + APPLICATIONS_ASSOCIATIONS.filter_map { |association_name| public_send(association_name) } # rubocop:disable GitlabSecurity/PublicSend end def applications @@ -266,6 +254,38 @@ module Clusters integration_prometheus || build_integration_prometheus end + def on_creation? + !!provider&.on_creation? + end + + def knative_pre_installed? + !!provider&.knative_pre_installed? + end + + def platform_kubernetes_active? + !!platform_kubernetes&.active? + end + + def platform_kubernetes_rbac? + !!platform_kubernetes&.rbac? + end + + def application_helm_available? + !!application_helm&.available? + end + + def application_ingress_available? + !!application_ingress&.available? + end + + def application_knative_available? + !!application_knative&.available? + end + + def integration_prometheus_available? + !!integration_prometheus&.available? + end + def provider if gcp? provider_gcp diff --git a/app/models/commit.rb b/app/models/commit.rb index a95ab756600..4517b3ef216 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -501,8 +501,8 @@ class Commit end end - def raw_diffs(*args) - raw.diffs(*args) + def raw_diffs(...) + raw.diffs(...) end def raw_deltas diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 64e585bae14..333a176b8f3 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -108,6 +108,8 @@ class CommitStatus < Ci::ApplicationRecord # These are pages deployments and external statuses. # before_create unless: :importing? do + next if Feature.enabled?(:ci_remove_ensure_stage_service, project) + # rubocop: disable CodeReuse/ServiceClass Ci::EnsureStageService.new(project, user).execute(self) do |stage| self.run_after_commit { StageUpdateWorker.perform_async(stage.id) } diff --git a/app/models/commit_user_mention.rb b/app/models/commit_user_mention.rb index 680d20b61cf..4d464f353ee 100644 --- a/app/models/commit_user_mention.rb +++ b/app/models/commit_user_mention.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true class CommitUserMention < UserMention + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' + belongs_to :note end diff --git a/app/models/compare.rb b/app/models/compare.rb index f594a796987..f03390334f4 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -79,8 +79,8 @@ class Compare commit&.sha end - def raw_diffs(*args) - @compare.diffs(*args) + def raw_diffs(...) + @compare.diffs(...) end def diffs(diff_options = nil) diff --git a/app/models/concerns/analytics/cycle_analytics/stageable.rb b/app/models/concerns/analytics/cycle_analytics/stageable.rb index d1f948d1366..caac4f31e1a 100644 --- a/app/models/concerns/analytics/cycle_analytics/stageable.rb +++ b/app/models/concerns/analytics/cycle_analytics/stageable.rb @@ -4,7 +4,6 @@ module Analytics module CycleAnalytics module Stageable extend ActiveSupport::Concern - include RelativePositioning include Gitlab::Utils::StrongMemoize included do @@ -92,10 +91,6 @@ module Analytics end_event_identifier.to_s.eql?(stage_params[:end_event_identifier].to_s) end - def find_with_same_parent!(id) - parent.cycle_analytics_stages.find(id) - end - private def validate_stage_event_pairs diff --git a/app/models/concerns/ci/has_variable.rb b/app/models/concerns/ci/has_variable.rb index 3b437fbba16..77e7e5035a0 100644 --- a/app/models/concerns/ci/has_variable.rb +++ b/app/models/concerns/ci/has_variable.rb @@ -18,6 +18,7 @@ module Ci scope :by_key, -> (key) { where(key: key) } scope :order_key_asc, -> { reorder(key: :asc) } + scope :order_key_desc, -> { reorder(key: :desc) } attr_encrypted :value, mode: :per_attribute_iv_and_salt, @@ -30,6 +31,16 @@ module Ci end end + class_methods do + def order_by(method) + case method.to_s + when 'key_asc' then order_key_asc + when 'key_desc' then order_key_desc + else all + end + end + end + def to_runner_variable var_cache_key = to_runner_variable_cache_key diff --git a/app/models/concerns/ci/maskable.rb b/app/models/concerns/ci/maskable.rb index 62be0150ee0..e2cef0981d1 100644 --- a/app/models/concerns/ci/maskable.rb +++ b/app/models/concerns/ci/maskable.rb @@ -12,10 +12,28 @@ module Ci # * Characters must be from the Base64 alphabet (RFC4648) with the addition of '@', ':', '.', and '~' # * Absolutely no fun is allowed REGEX = %r{\A[a-zA-Z0-9_+=/@:.~-]{8,}\z}.freeze + # * Single line + # * No spaces + # * Minimal length of 8 characters + # * Some fun is allowed + MASK_AND_RAW_REGEX = %r{\A\S{8,}\z}.freeze included do validates :masked, inclusion: { in: [true, false] } - validates :value, format: { with: REGEX }, if: :masked? + validates :value, format: { with: REGEX }, if: :masked_and_expanded? + validates :value, format: { with: MASK_AND_RAW_REGEX }, if: :masked_and_raw? + end + + def masked_and_raw? + return false unless self.class.method_defined?(:raw) + + masked? && raw? + end + + def masked_and_expanded? + return masked? unless self.class.method_defined?(:raw) + + masked? && !raw? end def to_runner_variable diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index d93f4a150d5..d91f33452a0 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -22,7 +22,7 @@ module Ci delegate :set_cancel_gracefully, to: :metadata, prefix: false, allow_nil: false delegate :id_tokens, to: :metadata, allow_nil: true - before_create :ensure_metadata + before_validation :ensure_metadata, on: :create end def has_exposed_artifacts? @@ -34,7 +34,7 @@ module Ci end def ensure_metadata - metadata || build_metadata(project: project, partition_id: partition_id) + metadata || build_metadata(project: project) end def degenerated? diff --git a/app/models/concerns/commit_signature.rb b/app/models/concerns/commit_signature.rb index 7f1fbbefd94..5dac3c7833a 100644 --- a/app/models/concerns/commit_signature.rb +++ b/app/models/concerns/commit_signature.rb @@ -4,6 +4,7 @@ module CommitSignature included do include ShaAttribute + include EachBatch sha_attribute :commit_sha @@ -14,7 +15,8 @@ module CommitSignature other_user: 3, unverified_key: 4, unknown_key: 5, - multiple_signatures: 6 + multiple_signatures: 6, + revoked_key: 7 } belongs_to :project, class_name: 'Project', foreign_key: 'project_id', optional: false diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 784afd1f231..58ea57962c5 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -93,7 +93,7 @@ module CounterAttribute run_after_commit_or_now do new_value = counter(attribute).increment(increment) - log_increment_counter(attribute, increment.amount, new_value) + log_increment_counter(attribute, increment, new_value) end end @@ -101,7 +101,7 @@ module CounterAttribute run_after_commit_or_now do new_value = counter(attribute).bulk_increment(increments) - log_increment_counter(attribute, increments.sum(&:amount), new_value) + log_bulk_increment_counter(attribute, increments, new_value) end end @@ -198,7 +198,8 @@ module CounterAttribute message: 'Increment counter attribute', attribute: attribute, project_id: project_id, - increment: increment, + increment: increment.amount, + ref: increment.ref, new_counter_value: new_value, current_db_value: read_attribute(attribute) ) @@ -206,6 +207,16 @@ module CounterAttribute Gitlab::AppLogger.info(payload) end + def log_bulk_increment_counter(attribute, increments, new_value) + if Feature.enabled?(:split_log_bulk_increment_counter, type: :ops) + increments.each do |increment| + log_increment_counter(attribute, increment, new_value) + end + else + log_increment_counter(attribute, Gitlab::Counters::Increment.new(amount: increments.sum(&:amount)), new_value) + end + end + def log_clear_counter(attribute) payload = Gitlab::ApplicationContext.current.merge( message: 'Clear counter attribute', diff --git a/app/models/concerns/cross_database_modification.rb b/app/models/concerns/cross_database_modification.rb index 273d5f35e76..df4f4f0bfe1 100644 --- a/app/models/concerns/cross_database_modification.rb +++ b/app/models/concerns/cross_database_modification.rb @@ -102,6 +102,10 @@ module CrossDatabaseModification :gitlab_main when 'Ci::ApplicationRecord' :gitlab_ci + when 'MainClusterwide::ApplicationRecord' + :gitlab_main_clusterwide + when 'PackageMetadata::ApplicationRecord' + :gitlab_pm else Gitlab::Database::GitlabSchema.table_schema(table_name) if table_name end diff --git a/app/models/concerns/enums/package_metadata.rb b/app/models/concerns/enums/package_metadata.rb new file mode 100644 index 00000000000..e15fe758e69 --- /dev/null +++ b/app/models/concerns/enums/package_metadata.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Enums + class PackageMetadata + PURL_TYPES = { + composer: 1, + conan: 2, + gem: 3, + golang: 4, + maven: 5, + npm: 6, + nuget: 7, + pypi: 8 + }.with_indifferent_access.freeze + + def self.purl_types + PURL_TYPES + end + end +end diff --git a/app/models/concerns/exportable.rb b/app/models/concerns/exportable.rb new file mode 100644 index 00000000000..066a44912be --- /dev/null +++ b/app/models/concerns/exportable.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Exportable + extend ActiveSupport::Concern + + def readable_records(association, current_user: nil) + association_records = try(association) + return unless association_records.present? + + if has_many_association?(association) + DeclarativePolicy.user_scope do + association_records.select { |record| readable_record?(record, current_user) } + end + else + readable_record?(association_records, current_user) ? association_records : nil + end + end + + def exportable_association?(association, current_user: nil) + return false unless respond_to?(association) + return true if has_many_association?(association) + + readable = try(association) + return true if readable.nil? + + readable_record?(readable, current_user) + end + + def restricted_associations(keys) + exportable_restricted_associations & keys + end + + def has_many_association?(association_name) + self.class.reflect_on_association(association_name)&.macro == :has_many + end + + private + + def exportable_restricted_associations + [] + end + + def readable_record?(record, user) + if record.respond_to?(:exportable_record?) + record.exportable_record?(user) + else + record.readable_by?(user) + end + end +end diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb index b376537a418..224ac8930b5 100644 --- a/app/models/concerns/group_descendant.rb +++ b/app/models/concerns/group_descendant.rb @@ -21,7 +21,7 @@ module GroupDescendant descendants = Array.wrap(descendants).uniq return [] if descendants.empty? - unless descendants.all? { |hierarchy| hierarchy.is_a?(GroupDescendant) } + unless descendants.all?(GroupDescendant) raise ArgumentError, _('element is not a hierarchy') end diff --git a/app/models/concerns/id_in_ordered.rb b/app/models/concerns/id_in_ordered.rb index b89409e6841..39067574520 100644 --- a/app/models/concerns/id_in_ordered.rb +++ b/app/models/concerns/id_in_ordered.rb @@ -5,7 +5,7 @@ module IdInOrdered included do scope :id_in_ordered, -> (ids) do - raise ArgumentError, "ids must be an array of integers" unless ids.is_a?(Enumerable) && ids.all? { |id| id.is_a?(Integer) } + raise ArgumentError, "ids must be an array of integers" unless ids.is_a?(Enumerable) && ids.all?(Integer) # No need to sort if no more than 1 and the sorting code doesn't work # with an empty array diff --git a/app/models/concerns/integrations/has_web_hook.rb b/app/models/concerns/integrations/has_web_hook.rb index e622faf4a51..dcf14a4c7dc 100644 --- a/app/models/concerns/integrations/has_web_hook.rb +++ b/app/models/concerns/integrations/has_web_hook.rb @@ -42,9 +42,9 @@ module Integrations end # Execute the webhook, creating it if necessary. - def execute_web_hook!(*args) + def execute_web_hook!(...) update_web_hook! - service_hook.execute(*args) + service_hook.execute(...) end end end diff --git a/app/models/concerns/issuable_link.rb b/app/models/concerns/issuable_link.rb index c319d685362..7f29083d6c6 100644 --- a/app/models/concerns/issuable_link.rb +++ b/app/models/concerns/issuable_link.rb @@ -20,6 +20,12 @@ module IssuableLink def issuable_type raise NotImplementedError end + + # Used to get the available types for the API + # overriden in EE + def available_link_types + [TYPE_RELATES_TO] + end end included do diff --git a/app/models/concerns/issue_parent.rb b/app/models/concerns/issue_parent.rb new file mode 100644 index 00000000000..c1fcbdcfc12 --- /dev/null +++ b/app/models/concerns/issue_parent.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# == IssuParent +# +# Used as a common ancestor for Group and Project so we can allow a polymorphic +# Types::GlobalIDType[::IssueParent] in the GraphQL API +# +# Used by Project, Group +# +module IssueParent +end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index eed396f785b..7addcf9e2ec 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -106,9 +106,9 @@ module Noteable relations << discussion_notes.select( "'notes' AS table_name", - 'discussion_id', 'MIN(id) AS id', - 'MIN(created_at) AS created_at' + 'MIN(created_at) AS created_at', + 'ARRAY_AGG(id) AS ids' ).with_notes_filter(notes_filter) .group(:discussion_id) @@ -116,17 +116,19 @@ module Noteable relations += synthetic_note_ids_relations end - Note.from_union(relations, remove_duplicates: false).fresh + Note.from_union(relations, remove_duplicates: false) + .select(:table_name, :id, :created_at, :ids) + .fresh end def capped_notes_count(max) notes.limit(max).count end - def grouped_diff_discussions(*args) + def grouped_diff_discussions(...) # Doesn't use `discussion_notes`, because this may include commit diff notes # besides MR diff notes, that we do not want to display on the MR Changes tab. - notes.inc_relations_for_view(self).grouped_diff_discussions(*args) + notes.inc_relations_for_view(self).grouped_diff_discussions(...) end # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -223,15 +225,16 @@ module Noteable # currently multiple models include Noteable concern, but not all of them support # all resource events, so we check if given model supports given resource event. if respond_to?(:resource_label_events) - relations << resource_label_events.select("'resource_label_events'", "'NULL'", :id, :created_at) + relations << resource_label_events.select("'resource_label_events'", 'MIN(id)', :created_at, 'ARRAY_AGG(id)') + .group(:created_at, :user_id) end if respond_to?(:resource_state_events) - relations << resource_state_events.select("'resource_state_events'", "'NULL'", :id, :created_at) + relations << resource_state_events.select("'resource_state_events'", :id, :created_at, 'ARRAY_FILL(id, ARRAY[1])') end if respond_to?(:resource_milestone_events) - relations << resource_milestone_events.select("'resource_milestone_events'", "'NULL'", :id, :created_at) + relations << resource_milestone_events.select("'resource_milestone_events'", :id, :created_at, 'ARRAY_FILL(id, ARRAY[1])') end relations diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb index df297017119..b85ac9ad4a6 100644 --- a/app/models/concerns/prometheus_adapter.rb +++ b/app/models/concerns/prometheus_adapter.rb @@ -49,7 +49,9 @@ module PrometheusAdapter query_class = query_klass_for(query_name) query_args = build_query_args(*args) - with_reactive_cache(query_class.name, *query_args, &query_class.method(:transform_reactive_result)) + with_reactive_cache(query_class.name, *query_args) do |result| + query_class.transform_reactive_result(result) + end end # Cache metrics for specific environment diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb index 9ed2070d11c..aa0fced99c4 100644 --- a/app/models/concerns/reactive_caching.rb +++ b/app/models/concerns/reactive_caching.rb @@ -122,8 +122,8 @@ module ReactiveCaching worker_class.perform_async(self.class, id, *args) end - def keep_alive_reactive_cache!(*args) - Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime) + def keep_alive_reactive_cache!(...) + Rails.cache.write(alive_reactive_cache_key(...), true, expires_in: self.class.reactive_cache_lifetime) end def full_reactive_cache_key(*qualifiers) @@ -145,8 +145,8 @@ module ReactiveCaching Gitlab::ExclusiveLease.cancel(full_reactive_cache_key(*args), uuid) end - def within_reactive_cache_lifetime?(*args) - Rails.cache.exist?(alive_reactive_cache_key(*args)) + def within_reactive_cache_lifetime?(...) + Rails.cache.exist?(alive_reactive_cache_key(...)) end def enqueuing_update(*args) diff --git a/app/models/concerns/require_email_verification.rb b/app/models/concerns/require_email_verification.rb index cf6a31e6ebd..5ff4f520d24 100644 --- a/app/models/concerns/require_email_verification.rb +++ b/app/models/concerns/require_email_verification.rb @@ -45,8 +45,9 @@ module RequireEmailVerification private def override_devise_lockable? - strong_memoize(:override_devise_lockable) do - Feature.enabled?(:require_email_verification, self) && !two_factor_enabled? - end + Feature.enabled?(:require_email_verification, self) && + !two_factor_enabled? && + Feature.disabled?(:skip_require_email_verification, self, type: :ops) end + strong_memoize_attr :override_devise_lockable? end diff --git a/app/models/concerns/sensitive_serializable_hash.rb b/app/models/concerns/sensitive_serializable_hash.rb index 794748483e4..5a9b75d4db8 100644 --- a/app/models/concerns/sensitive_serializable_hash.rb +++ b/app/models/concerns/sensitive_serializable_hash.rb @@ -24,12 +24,12 @@ module SensitiveSerializableHash options[:except].concat self.class.attributes_exempt_from_serializable_hash - if self.class.respond_to?(:encrypted_attributes) - options[:except].concat self.class.encrypted_attributes.keys + if self.class.respond_to?(:attr_encrypted_attributes) + options[:except].concat self.class.attr_encrypted_attributes.keys # Per https://github.com/attr-encrypted/attr_encrypted/blob/a96693e9a2a25f4f910bf915e29b0f364f277032/lib/attr_encrypted.rb#L413 - options[:except].concat self.class.encrypted_attributes.values.map { |v| v[:attribute] } - options[:except].concat self.class.encrypted_attributes.values.map { |v| "#{v[:attribute]}_iv" } + options[:except].concat self.class.attr_encrypted_attributes.values.map { |v| v[:attribute] } + options[:except].concat self.class.attr_encrypted_attributes.values.map { |v| "#{v[:attribute]}_iv" } end super(options) diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index 701d2fda5c5..35c48c15fb2 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -22,7 +22,7 @@ module ShaAttribute class_methods do def sha_attribute(name) - return if ENV['STATIC_VERIFICATION'] + return if Gitlab::Environment.static_verification? sha_attribute_fields << name @@ -34,7 +34,7 @@ module ShaAttribute end def sha256_attribute(name) - return if ENV['STATIC_VERIFICATION'] + return if Gitlab::Environment.static_verification? sha256_attribute_fields << name diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index d27b451892a..fba923e843a 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -123,6 +123,6 @@ module Spammable # Override in Spammable if differs def allow_possible_spam? - Feature.enabled?(:allow_possible_spam, project) + Gitlab::CurrentSettings.allow_possible_spam end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index 05addcf83d2..f9eba4cc2fe 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -24,10 +24,28 @@ module Taskable (\s.+) # followed by whitespace and some text. }x.freeze + # ignore tasks in code or html comment blocks. HTML blocks + # are ok as we allow tasks inside <detail> blocks + REGEX = %r{ + #{::Gitlab::Regex.markdown_code_or_html_comments} + | + (?<task_item> + #{ITEM_PATTERN} + ) + }mx.freeze + def self.get_tasks(content) - content.to_s.scan(ITEM_PATTERN).map do |prefix, checkbox, label| - TaskList::Item.new("#{prefix} #{checkbox}", label.strip) + items = [] + + content.to_s.scan(REGEX) do + next unless $~[:task_item] + + $~[:task_item].scan(ITEM_PATTERN) do |prefix, checkbox, label| + items << TaskList::Item.new("#{prefix.strip} #{checkbox}", label.strip) + end end + + items end def self.get_updated_tasks(old_content:, new_content:) @@ -67,10 +85,10 @@ module Taskable checklist_item_noun = n_('checklist item', 'checklist items', sum.item_count) if short format(s_('Tasks|%{complete_count}/%{total_count} %{checklist_item_noun}'), -checklist_item_noun: checklist_item_noun, complete_count: sum.complete_count, total_count: sum.item_count) + checklist_item_noun: checklist_item_noun, complete_count: sum.complete_count, total_count: sum.item_count) else format(s_('Tasks|%{complete_count} of %{total_count} %{checklist_item_noun} completed'), -checklist_item_noun: checklist_item_noun, complete_count: sum.complete_count, total_count: sum.item_count) + checklist_item_noun: checklist_item_noun, complete_count: sum.complete_count, total_count: sum.item_count) end end diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index d91ec161b84..cc3e8f174b3 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -86,7 +86,7 @@ module TokenAuthenticatable def token_authenticatable_module @token_authenticatable_module ||= - const_set(:TokenAuthenticatable, Module.new).tap(&method(:include)) + const_set(:TokenAuthenticatable, Module.new).tap { |mod| include mod } end end end diff --git a/app/models/concerns/vulnerability_finding_helpers.rb b/app/models/concerns/vulnerability_finding_helpers.rb index b5d48260072..1e8a290c050 100644 --- a/app/models/concerns/vulnerability_finding_helpers.rb +++ b/app/models/concerns/vulnerability_finding_helpers.rb @@ -49,7 +49,7 @@ module VulnerabilityFindingHelpers finding_data = report_finding.to_hash.except(:compare_key, :identifiers, :location, :scanner, :links, :signatures, :flags, :evidence) - identifiers = report_finding.identifiers.map do |identifier| + identifiers = report_finding.identifiers.uniq(&:fingerprint).map do |identifier| Vulnerabilities::Identifier.new(identifier.to_hash.merge({ project: project })) end signatures = report_finding.signatures.map do |signature| diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb new file mode 100644 index 00000000000..2cc17a6f185 --- /dev/null +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module WebHooks + module AutoDisabling + extend ActiveSupport::Concern + + included do + # A hook is disabled if: + # + # - we are no longer in the grace-perod (recent_failures > ?) + # - and either: + # - disabled_until is nil (i.e. this was set by WebHook#fail!) + # - or disabled_until is in the future (i.e. this was set by WebHook#backoff!) + scope :disabled, -> do + where('recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)', + WebHook::FAILURE_THRESHOLD, Time.current) + end + + # A hook is executable if: + # + # - we are still in the grace-period (recent_failures <= ?) + # - OR we have exceeded the grace period and neither of the following is true: + # - disabled_until is nil (i.e. this was set by WebHook#fail!) + # - disabled_until is in the future (i.e. this was set by WebHook#backoff!) + scope :executable, -> do + where('recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))', + WebHook::FAILURE_THRESHOLD, WebHook::FAILURE_THRESHOLD, Time.current) + end + end + + def executable? + !temporarily_disabled? && !permanently_disabled? + end + + def temporarily_disabled? + return false if recent_failures <= WebHook::FAILURE_THRESHOLD + + disabled_until.present? && disabled_until >= Time.current + end + + def permanently_disabled? + return false if disabled_until.present? + + recent_failures > WebHook::FAILURE_THRESHOLD + end + + def disable! + return if permanently_disabled? + + super + end + + def backoff! + return if permanently_disabled? || (backoff_count >= WebHook::MAX_FAILURES && temporarily_disabled?) + + super + end + + def alert_status + if temporarily_disabled? + :temporarily_disabled + elsif permanently_disabled? + :disabled + else + :executable + end + end + end +end diff --git a/app/models/concerns/web_hooks/has_web_hooks.rb b/app/models/concerns/web_hooks/has_web_hooks.rb new file mode 100644 index 00000000000..161ce106b9b --- /dev/null +++ b/app/models/concerns/web_hooks/has_web_hooks.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module WebHooks + module HasWebHooks + extend ActiveSupport::Concern + + WEB_HOOK_CACHE_EXPIRY = 1.hour + + def any_hook_failed? + hooks.disabled.exists? + end + + def web_hook_failure_redis_key + "any_web_hook_failed:#{id}" + end + + def last_failure_redis_key + "web_hooks:last_failure:project-#{id}" + end + + def get_web_hook_failure + Gitlab::Redis::SharedState.with do |redis| + current = redis.get(web_hook_failure_redis_key) + + Gitlab::Utils.to_boolean(current) if current + end + end + + def fetch_web_hook_failure + Gitlab::Redis::SharedState.with do |_redis| + current = get_web_hook_failure + next current unless current.nil? + + cache_web_hook_failure + end + end + + def cache_web_hook_failure(state = any_hook_failed?) + Gitlab::Redis::SharedState.with do |redis| + redis.set(web_hook_failure_redis_key, state.to_s, ex: WEB_HOOK_CACHE_EXPIRY) + + state + end + end + end +end diff --git a/app/models/concerns/web_hooks/unstoppable.rb b/app/models/concerns/web_hooks/unstoppable.rb new file mode 100644 index 00000000000..26284fe3c36 --- /dev/null +++ b/app/models/concerns/web_hooks/unstoppable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module WebHooks + module Unstoppable + extend ActiveSupport::Concern + + included do + scope :executable, -> { all } + + scope :disabled, -> { none } + end + + def executable? + true + end + + def temporarily_disabled? + false + end + + def permanently_disabled? + false + end + + def alert_status + :executable + end + end +end diff --git a/app/models/concerns/work_item_resource_event.rb b/app/models/concerns/work_item_resource_event.rb index d0323feb029..ddf39787f63 100644 --- a/app/models/concerns/work_item_resource_event.rb +++ b/app/models/concerns/work_item_resource_event.rb @@ -5,6 +5,18 @@ module WorkItemResourceEvent included do belongs_to :work_item, foreign_key: 'issue_id' + + scope :with_work_item, -> { preload(:work_item) } + + # These events are created also on non work items, e.g. MRs, Epic however system notes subscription + # is only implemented on work items, so we do check if this event is linked to an work item. This can be + # expanded to other issuables later on. + after_commit :trigger_note_subscription_create, on: :create, if: -> { work_item.present? } + end + + # System notes are not updated or deleted, so firing just the noteCreated event. + def trigger_note_subscription_create(events: self) + GraphqlTriggers.work_item_note_created(work_item.to_gid, events) end def work_item_synthetic_system_note(events: nil) diff --git a/app/models/concerns/x509_serial_number_attribute.rb b/app/models/concerns/x509_serial_number_attribute.rb index 9dc53859ac0..b65736b7924 100644 --- a/app/models/concerns/x509_serial_number_attribute.rb +++ b/app/models/concerns/x509_serial_number_attribute.rb @@ -5,7 +5,7 @@ module X509SerialNumberAttribute class_methods do def x509_serial_number_attribute(name) - return if ENV['STATIC_VERIFICATION'] + return if Gitlab::Environment.static_verification? validate_binary_column_exists!(name) unless Rails.env.production? diff --git a/app/models/container_registry/event.rb b/app/models/container_registry/event.rb index d4075e1ff1b..c4d06be8841 100644 --- a/app/models/container_registry/event.rb +++ b/app/models/container_registry/event.rb @@ -8,6 +8,21 @@ module ContainerRegistry PUSH_ACTION = 'push' DELETE_ACTION = 'delete' EVENT_TRACKING_CATEGORY = 'container_registry:notification' + EVENT_PREFIX = "i_container_registry" + + ALLOWED_ACTOR_TYPES = %w( + personal_access_token + build + gitlab_or_ldap + ).freeze + + TRACKABLE_ACTOR_EVENTS = %w( + push_tag + delete_tag + push_repository + delete_repository + create_repository + ).freeze attr_reader :event @@ -32,6 +47,9 @@ module ContainerRegistry end ::Gitlab::Tracking.event(EVENT_TRACKING_CATEGORY, tracking_action) + + event = usage_data_event_for(tracking_action) + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id) if event end private @@ -81,6 +99,29 @@ module ContainerRegistry container_registry_path&.repository_project end + # counter name for unique user tracking (for MAU) + def usage_data_event_for(tracking_action) + return unless originator + return unless TRACKABLE_ACTOR_EVENTS.include?(tracking_action) + + "#{EVENT_PREFIX}_#{tracking_action}_user" + end + + def originator_type + event.dig('actor', 'user_type') + end + + def originator + return unless ALLOWED_ACTOR_TYPES.include?(originator_type) + + username = event.dig('actor', 'name') + return unless username + + strong_memoize(:originator) do + User.find_by_username(username) + end + end + def update_project_statistics return unless supported? return unless target_tag? || (action_delete? && target_digest?) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index db0fcd915b3..98ce981ad8e 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -395,7 +395,7 @@ class ContainerRepository < ApplicationRecord end def migrated? - MIGRATION_PHASE_1_ENDED_AT < self.created_at || import_done? + (self.created_at && MIGRATION_PHASE_1_ENDED_AT < self.created_at) || import_done? end def last_import_step_done_at @@ -497,7 +497,7 @@ class ContainerRepository < ApplicationRecord digests = tags.map { |tag| tag.digest }.compact.to_set - digests.map(&method(:delete_tag_by_digest)).all? + digests.map { |digest| delete_tag_by_digest(digest) }.all? end def delete_tag_by_digest(digest) diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index aaafa396337..ef31bedc3a8 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -9,9 +9,10 @@ class DeployKey < Key has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :deploy_keys_projects - has_many :deploy_keys_projects_with_write_access, -> { with_write_access }, class_name: "DeployKeysProject" + has_many :deploy_keys_projects_with_write_access, -> { with_write_access }, class_name: "DeployKeysProject", inverse_of: :deploy_key has_many :projects_with_write_access, -> { includes(:route) }, class_name: 'Project', through: :deploy_keys_projects_with_write_access, source: :project - has_many :protected_branch_push_access_levels, class_name: '::ProtectedBranch::PushAccessLevel' + has_many :protected_branch_push_access_levels, class_name: '::ProtectedBranch::PushAccessLevel', inverse_of: :deploy_key + has_many :protected_tag_create_access_levels, class_name: '::ProtectedTag::CreateAccessLevel', inverse_of: :deploy_key scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where(deploy_keys_projects: { project_id: projects }) } scope :with_write_access, -> { joins(:deploy_keys_projects).merge(DeployKeysProject.with_write_access) } diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 1ae7d9925a5..f8873d388a3 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -105,7 +105,11 @@ class Deployment < ApplicationRecord after_transition any => :running do |deployment, transition| deployment.run_after_commit do - Deployments::HooksWorker.perform_async(deployment_id: id, status: transition.to, status_changed_at: Time.current) + perform_params = { deployment_id: id, status: transition.to, status_changed_at: Time.current } + + serialize_params_for_sidekiq!(perform_params) + + Deployments::HooksWorker.perform_async(perform_params) end end @@ -119,7 +123,11 @@ class Deployment < ApplicationRecord after_transition any => FINISHED_STATUSES do |deployment, transition| deployment.run_after_commit do - Deployments::HooksWorker.perform_async(deployment_id: id, status: transition.to, status_changed_at: Time.current) + perform_params = { deployment_id: id, status: transition.to, status_changed_at: Time.current } + + serialize_params_for_sidekiq!(perform_params) + + Deployments::HooksWorker.perform_async(perform_params) end end @@ -464,6 +472,11 @@ class Deployment < ApplicationRecord end end + def serialize_params_for_sidekiq!(perform_params) + perform_params[:status_changed_at] = perform_params[:status_changed_at].to_s + perform_params.stringify_keys! + end + def self.last_deployment_group_associations { deployable: { diff --git a/app/models/design_user_mention.rb b/app/models/design_user_mention.rb index baf4db29a0f..87899f65cb1 100644 --- a/app/models/design_user_mention.rb +++ b/app/models/design_user_mention.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class DesignUserMention < UserMention + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' + belongs_to :design, class_name: 'DesignManagement::Design' belongs_to :note end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 9eb3308b901..83c85f30178 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -10,7 +10,8 @@ class Discussion # Bump this if we need to refresh the cached versions of discussions CACHE_VERSION = 1 - attr_reader :notes, :context_noteable + attr_reader :context_noteable + attr_accessor :notes delegate :created_at, :project, @@ -183,4 +184,11 @@ class Discussion resolved_at ].join(':') end + + # Consolidate discussions GID. There is no need to have different GID for different class names as the discussion_id + # hash is already unique per discussion. This also fixes the issue where same discussion may return different GIDs + # depending on number of notes it has. + def to_global_id(options = {}) + GlobalID.new(::Gitlab::GlobalId.build(model_name: Discussion.to_s, id: id)) + end end diff --git a/app/models/environment.rb b/app/models/environment.rb index 7d99f10822d..f1de41674c6 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -28,20 +28,18 @@ class Environment < ApplicationRecord has_many :self_managed_prometheus_alert_events, inverse_of: :environment has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment - # NOTE: - # 1) no-op arguments is to prevent accidental legacy preloading. See: https://gitlab.com/gitlab-org/gitlab/-/issues/369240 - # 2) If you preload multiple last deployments of environments, use Preloaders::Environments::DeploymentPreloader. - has_one :last_deployment, -> (_env) { success.ordered }, class_name: 'Deployment', inverse_of: :environment - has_one :last_visible_deployment, -> (_env) { visible.order(id: :desc) }, inverse_of: :environment, class_name: 'Deployment' - has_one :upcoming_deployment, -> (_env) { upcoming.order(id: :desc) }, class_name: 'Deployment', inverse_of: :environment + # NOTE: If you preload multiple last deployments of environments, use Preloaders::Environments::DeploymentPreloader. + has_one :last_deployment, -> { success.ordered }, class_name: 'Deployment', inverse_of: :environment + has_one :last_visible_deployment, -> { visible.order(id: :desc) }, inverse_of: :environment, class_name: 'Deployment' + has_one :upcoming_deployment, -> { upcoming.order(id: :desc) }, class_name: 'Deployment', inverse_of: :environment Deployment::FINISHED_STATUSES.each do |status| - has_one :"last_#{status}_deployment", -> (_env) { where(status: status).ordered }, + has_one :"last_#{status}_deployment", -> { where(status: status).ordered }, class_name: 'Deployment', inverse_of: :environment end Deployment::UPCOMING_STATUSES.each do |status| - has_one :"last_#{status}_deployment", -> (_env) { where(status: status).ordered_as_upcoming }, + has_one :"last_#{status}_deployment", -> { where(status: status).ordered_as_upcoming }, class_name: 'Deployment', inverse_of: :environment end @@ -74,7 +72,11 @@ class Environment < ApplicationRecord # Currently, the tier presence is validaed for newly created environments. # After the `BackfillEnvironmentTiers` background migration has been completed, we should remove `on: :create`. # See https://gitlab.com/gitlab-org/gitlab/-/issues/385253. - validates :tier, presence: true, on: :create + # Todo: Remove along with FF `validate_environment_tier_presence`. + validates :tier, presence: true, on: :create, unless: :validate_environment_tier_present? + + validates :tier, presence: true, if: :validate_environment_tier_present? + validate :safe_external_url validate :merge_request_not_changed @@ -600,6 +602,10 @@ class Environment < ApplicationRecord self.class.tiers[:other] end end + + def validate_environment_tier_present? + Feature.enabled?(:validate_environment_tier_presence, self.project) + end end Environment.prepend_mod_with('Environment') diff --git a/app/models/grafana_integration.rb b/app/models/grafana_integration.rb index 5cd5aa1b085..71abfd3f6da 100644 --- a/app/models/grafana_integration.rb +++ b/app/models/grafana_integration.rb @@ -45,7 +45,7 @@ class GrafanaIntegration < ApplicationRecord end def token - decrypt(:token, encrypted_token) + attr_decrypt(:token, encrypted_token) end def check_token_changes diff --git a/app/models/group.rb b/app/models/group.rb index c7ad4d61ddb..7e09280dfff 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -21,9 +21,12 @@ class Group < Namespace include ChronicDurationAttribute include RunnerTokenExpirationInterval include Todoable + include IssueParent extend ::Gitlab::Utils::Override + README_PROJECT_PATH = 'gitlab-profile' + def self.sti_name 'Group' end @@ -43,7 +46,10 @@ class Group < Namespace has_many :requesters, -> { where.not(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' # rubocop:disable Cop/ActiveRecordDependent has_many :namespace_requesters, -> { where.not(requested_at: nil).unscope(where: %i[source_id source_type]) }, foreign_key: :member_namespace_id, inverse_of: :group, class_name: 'GroupMember' + has_many :members_and_requesters, as: :source, class_name: 'GroupMember' + has_many :namespace_members_and_requesters, -> { unscope(where: %i[source_id source_type]) }, + foreign_key: :member_namespace_id, inverse_of: :group, class_name: 'GroupMember' has_many :milestones has_many :integrations @@ -422,15 +428,14 @@ class Group < Namespace ) end - def add_member(user, access_level, current_user: nil, expires_at: nil, ldap: false, blocking_refresh: true) + def add_member(user, access_level, current_user: nil, expires_at: nil, ldap: false) Members::Groups::CreatorService.add_member( # rubocop:disable CodeReuse/ServiceClass self, user, access_level, current_user: current_user, expires_at: expires_at, - ldap: ldap, - blocking_refresh: blocking_refresh + ldap: ldap ) end @@ -539,7 +544,6 @@ class Group < Namespace # rubocop: disable CodeReuse/ServiceClass def refresh_members_authorized_projects( - blocking: true, priority: UserProjectAccessChangedService::HIGH_PRIORITY, direct_members_only: false ) @@ -552,7 +556,7 @@ class Group < Namespace UserProjectAccessChangedService .new(user_ids) - .execute(blocking: blocking, priority: priority) + .execute(priority: priority) end # rubocop: enable CodeReuse/ServiceClass @@ -748,7 +752,7 @@ class Group < Namespace end def refresh_project_authorizations - refresh_members_authorized_projects(blocking: false) + refresh_members_authorized_projects end # each existing group needs to have a `runners_token`. @@ -915,10 +919,6 @@ class Group < Namespace feature_flag_enabled_for_self_or_ancestor?(:work_items_mvc_2) end - def work_items_create_from_markdown_feature_flag_enabled? - feature_flag_enabled_for_self_or_ancestor?(:work_items_create_from_markdown) - end - def usage_quotas_enabled? ::Feature.enabled?(:usage_quotas_for_all_editions, self) && root? end @@ -948,6 +948,16 @@ class Group < Namespace direct_and_indirect_members.find_each(&:update_two_factor_requirement) end + def readme_project + projects.find_by(path: README_PROJECT_PATH) + end + strong_memoize_attr :readme_project + + def group_readme + readme_project&.repository&.readme + end + strong_memoize_attr :group_readme + private def feature_flag_enabled_for_self_or_ancestor?(feature_flag) diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index dcba136d163..8e9a74a68d0 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -2,6 +2,7 @@ class ProjectHook < WebHook include TriggerableHooks + include WebHooks::AutoDisabling include Presentable include Limitable extend ::Gitlab::Utils::Override @@ -45,14 +46,18 @@ class ProjectHook < WebHook override :update_last_failure def update_last_failure - return if executable? + if executable? + project.cache_web_hook_failure if project.get_web_hook_failure # may need update + else + project.cache_web_hook_failure(true) # definitely failing, no need to check - key = "web_hooks:last_failure:project-#{project_id}" - time = Time.current.utc.iso8601 + Gitlab::Redis::SharedState.with do |redis| + last_failure_key = project.last_failure_redis_key + time = Time.current.utc.iso8601 + prev = redis.get(last_failure_key) - Gitlab::Redis::SharedState.with do |redis| - prev = redis.get(key) - redis.set(key, time) if !prev || prev < time + redis.set(last_failure_key, time) if !prev || prev < time + end end end end diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb index 94ced96bbde..6af70c249a0 100644 --- a/app/models/hooks/service_hook.rb +++ b/app/models/hooks/service_hook.rb @@ -1,7 +1,9 @@ # frozen_string_literal: true class ServiceHook < WebHook + include WebHooks::Unstoppable include Presentable + extend ::Gitlab::Utils::Override belongs_to :integration @@ -13,9 +15,4 @@ class ServiceHook < WebHook override :parent delegate :parent, to: :integration - - override :executable? - def executable? - true - end end diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index 3c7f0ef9ffc..eaffe83cab3 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -2,6 +2,7 @@ class SystemHook < WebHook include TriggerableHooks + include WebHooks::Unstoppable triggerable_hooks [ :repository_update_hooks, diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 49418cda3ac..819152a38c8 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -56,31 +56,6 @@ class WebHook < ApplicationRecord all_branches: 2 }, _prefix: true - scope :executable, -> do - where('recent_failures <= ? AND (disabled_until IS NULL OR disabled_until < ?)', FAILURE_THRESHOLD, Time.current) - end - - # Inverse of executable - scope :disabled, -> do - where('recent_failures > ? OR disabled_until >= ?', FAILURE_THRESHOLD, Time.current) - end - - def executable? - !temporarily_disabled? && !permanently_disabled? - end - - def temporarily_disabled? - return false if recent_failures <= FAILURE_THRESHOLD - - disabled_until.present? && disabled_until >= Time.current - end - - def permanently_disabled? - return false if disabled_until.present? - - recent_failures > FAILURE_THRESHOLD - end - # rubocop: disable CodeReuse/ServiceClass def execute(data, hook_name, force: false) # hook.executable? is checked in WebHookService#execute @@ -112,8 +87,6 @@ class WebHook < ApplicationRecord end def disable! - return if permanently_disabled? - update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD) end @@ -127,8 +100,6 @@ class WebHook < ApplicationRecord # Don't actually back-off until FAILURE_THRESHOLD failures have been seen # we mark the grace-period using the recent_failures counter def backoff! - return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?) - attrs = { recent_failures: next_failure_count } if recent_failures >= FAILURE_THRESHOLD @@ -137,7 +108,7 @@ class WebHook < ApplicationRecord end assign_attributes(attrs) - save(validate: false) + save(validate: false) if changed? end def failed! @@ -167,16 +138,6 @@ class WebHook < ApplicationRecord { related_class: type } end - def alert_status - if temporarily_disabled? - :temporarily_disabled - elsif permanently_disabled? - :disabled - else - :executable - end - end - # Exclude binary columns by default - they have no sensible JSON encoding def serializable_hash(options = nil) options = options.try(:dup) || {} diff --git a/app/models/hooks/web_hook_log.rb b/app/models/hooks/web_hook_log.rb index 9de6f2a1b57..e08294058e4 100644 --- a/app/models/hooks/web_hook_log.rb +++ b/app/models/hooks/web_hook_log.rb @@ -9,6 +9,8 @@ class WebHookLog < ApplicationRecord OVERSIZE_REQUEST_DATA = { 'oversize' => true }.freeze + attr_accessor :interpolated_url + self.primary_key = :id partitioned_by :created_at, strategy: :monthly, retain_for: 3.months @@ -23,6 +25,7 @@ class WebHookLog < ApplicationRecord before_save :obfuscate_basic_auth before_save :redact_user_emails + before_save :set_url_hash, if: -> { interpolated_url.present? } def self.recent where(created_at: 2.days.ago.beginning_of_day..Time.zone.now) @@ -66,4 +69,8 @@ class WebHookLog < ApplicationRecord value.to_s =~ URI::MailTo::EMAIL_REGEXP ? _('[REDACTED]') : value end end + + def set_url_hash + self.url_hash = Gitlab::CryptoHelper.sha256(interpolated_url) + end end diff --git a/app/models/incident_management/timeline_event_tag.rb b/app/models/incident_management/timeline_event_tag.rb index d1e3fbc2a6a..97b896d369d 100644 --- a/app/models/incident_management/timeline_event_tag.rb +++ b/app/models/incident_management/timeline_event_tag.rb @@ -4,8 +4,14 @@ module IncidentManagement class TimelineEventTag < ApplicationRecord self.table_name = 'incident_management_timeline_event_tags' - START_TIME_TAG_NAME = 'Start time' - END_TIME_TAG_NAME = 'End time' + PREDEFINED_TAGS = [ + 'Start time', + 'End time', + 'Impact detected', + 'Response initiated', + 'Impact mitigated', + 'Cause identified' + ].freeze belongs_to :project, inverse_of: :incident_management_timeline_event_tags diff --git a/app/models/integration.rb b/app/models/integration.rb index 54eeab10360..8bef8b08c19 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -493,9 +493,9 @@ class Integration < ApplicationRecord def reencrypt_properties unless properties.nil? || properties.empty? - alg = self.class.encrypted_attributes[:properties][:algorithm] + alg = self.class.attr_encrypted_attributes[:properties][:algorithm] iv = generate_iv(alg) - ep = self.class.encrypt(:properties, properties, { iv: iv }) + ep = self.class.attr_encrypt(:properties, properties, { iv: iv }) end { 'encrypted_properties' => ep, 'encrypted_properties_iv' => iv } diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index 8700b673370..963ba918089 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -23,6 +23,7 @@ module Integrations ].freeze SECRET_MASK = '************' + CHANNEL_LIMIT_PER_EVENT = 10 attribute :category, default: 'chat' @@ -37,7 +38,8 @@ module Integrations presence: true, public_url: true, if: -> (integration) { integration.activated? && integration.requires_webhook? } - validates :labels_to_be_notified_behavior, inclusion: { in: LABEL_NOTIFICATION_BEHAVIOURS }, allow_blank: true + validates :labels_to_be_notified_behavior, inclusion: { in: LABEL_NOTIFICATION_BEHAVIOURS }, allow_blank: true, if: :activated? + validate :validate_channel_limit, if: :activated? def initialize_properties super @@ -132,17 +134,15 @@ module Integrations return false unless message - event_type = data[:event_type] || object_kind - - channel_names = event_channel_value(event_type).presence || channel.presence - channels = channel_names&.split(',')&.map(&:strip) + event = data[:event_type] || object_kind + channels = channels_for_event(event) opts = {} opts[:channel] = channels if channels.present? opts[:username] = username if username if notify(message, opts) - log_usage(event_type, user_id_from_hook_data(data)) + log_usage(event, user_id_from_hook_data(data)) return true end @@ -297,6 +297,34 @@ module Integrations false end end + + def channels_for_event(event) + channel_names = event_channel_value(event).presence || channel.presence + return [] unless channel_names + + channel_names.split(',').map(&:strip).uniq + end + + def unique_channels + @unique_channels ||= supported_events.flat_map do |event| + channels_for_event(event) + end.uniq + end + + def validate_channel_limit + supported_events.each do |event| + count = channels_for_event(event).count + next unless count > CHANNEL_LIMIT_PER_EVENT + + errors.add( + event_channel_name(event).to_sym, + format( + s_('SlackIntegration|cannot have more than %{limit} channels'), + limit: CHANNEL_LIMIT_PER_EVENT + ) + ) + end + end end end diff --git a/app/models/integrations/chat_message/base_message.rb b/app/models/integrations/chat_message/base_message.rb index 554b422c0fa..501b214a769 100644 --- a/app/models/integrations/chat_message/base_message.rb +++ b/app/models/integrations/chat_message/base_message.rb @@ -5,10 +5,6 @@ module Integrations class BaseMessage RELATIVE_LINK_REGEX = %r{!\[[^\]]*\]\((/uploads/[^\)]*)\)}.freeze - # Markup characters which are used for links in HTML, Markdown, - # and Slack "mrkdwn" syntax (`<http://example.com|Label>`). - UNSAFE_MARKUP_CHARACTERS = '<>[]|' - attr_reader :markdown attr_reader :user_full_name attr_reader :user_name @@ -85,7 +81,7 @@ module Integrations # - https://api.slack.com/reference/surfaces/formatting#escaping # - https://gitlab.com/gitlab-org/slack-notifier#escaping def strip_markup(string) - string&.delete(UNSAFE_MARKUP_CHARACTERS) + SlackMarkdownSanitizer.sanitize(string) end def attachment_color diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 45302a0bd09..d96a848c72e 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -48,21 +48,21 @@ module Integrations section: SECTION_TYPE_CONNECTION, required: true, title: -> { s_('JiraService|Web URL') }, - help: -> { s_('JiraService|Base URL of the Jira instance.') }, + help: -> { s_('JiraService|Base URL of the Jira instance') }, placeholder: 'https://jira.example.com', exposes_secrets: true field :api_url, section: SECTION_TYPE_CONNECTION, title: -> { s_('JiraService|Jira API URL') }, - help: -> { s_('JiraService|If different from Web URL.') }, + help: -> { s_('JiraService|If different from the Web URL') }, exposes_secrets: true field :username, section: SECTION_TYPE_CONNECTION, required: true, - title: -> { s_('JiraService|Username or Email') }, - help: -> { s_('JiraService|Use a username for server version and an email for cloud version.') } + title: -> { s_('JiraService|Username or email') }, + help: -> { s_('JiraService|Username for the server version or an email for the cloud version') } field :password, section: SECTION_TYPE_CONNECTION, @@ -70,7 +70,7 @@ module Integrations title: -> { s_('JiraService|Password or API token') }, non_empty_password_title: -> { s_('JiraService|Enter new password or API token') }, non_empty_password_help: -> { s_('JiraService|Leave blank to use your current password or API token.') }, - help: -> { s_('JiraService|Use a password for server version and an API token for cloud version.') } + help: -> { s_('JiraService|Password for the server version or an API token for the cloud version') } field :jira_issue_transition_id, api_only: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 6744ee230b0..bea86168c8d 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -25,6 +25,7 @@ class Issue < ApplicationRecord include FromUnion include EachBatch include PgFullTextSearchable + include Exportable extend ::Gitlab::Utils::Override @@ -180,11 +181,7 @@ class Issue < ApplicationRecord scope :confidential_only, -> { where(confidential: true) } scope :without_hidden, -> { - if Feature.enabled?(:ban_user_feature_flag) - where('NOT EXISTS (?)', Users::BannedUser.select(1).where('issues.author_id = banned_users.user_id')) - else - all - end + where('NOT EXISTS (?)', Users::BannedUser.select(1).where('issues.author_id = banned_users.user_id')) } scope :counts_by_state, -> { reorder(nil).group(:state_id).count } @@ -328,13 +325,22 @@ class Issue < ApplicationRecord '#' end + # Alternative prefix for situations where the standard prefix would be + # interpreted as a comment, most notably to begin commit messages with + # (e.g. "GL-123: My commit") + def self.alternative_reference_prefix + 'GL-' + end + # Pattern used to extract `#123` issue references from text # # This pattern supports cross-project references. def self.reference_pattern @reference_pattern ||= %r{ - (#{Project.reference_pattern})? - #{Regexp.escape(reference_prefix)}#{Gitlab::Regex.issue} + (?: + (#{Project.reference_pattern})?#{Regexp.escape(reference_prefix)} | + #{Regexp.escape(alternative_reference_prefix)} + )#{Gitlab::Regex.issue} }x end @@ -672,6 +678,12 @@ class Issue < ApplicationRecord true end + # we want to have subscriptions working on work items only, legacy issues do not support graphql subscriptions, yet so + # we need sometimes GID of an issue instance to be represented as WorkItem GID. E.g. notes subscriptions. + def to_work_item_global_id + ::Gitlab::GlobalId.as_global_id(id, model_name: WorkItem.name) + end + private def due_date_after_start_date diff --git a/app/models/issue_email_participant.rb b/app/models/issue_email_participant.rb index dd963bc9e7e..9d7e2afa1d9 100644 --- a/app/models/issue_email_participant.rb +++ b/app/models/issue_email_participant.rb @@ -2,6 +2,7 @@ class IssueEmailParticipant < ApplicationRecord include BulkInsertSafe + include Presentable belongs_to :issue diff --git a/app/models/issue_user_mention.rb b/app/models/issue_user_mention.rb index 3eadd580f7f..bb13b83d3ba 100644 --- a/app/models/issue_user_mention.rb +++ b/app/models/issue_user_mention.rb @@ -3,4 +3,7 @@ class IssueUserMention < UserMention belongs_to :issue belongs_to :note + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' end diff --git a/app/models/jira_connect_installation.rb b/app/models/jira_connect_installation.rb index 0e88d1ceae9..f07f979a06d 100644 --- a/app/models/jira_connect_installation.rb +++ b/app/models/jira_connect_installation.rb @@ -31,7 +31,7 @@ class JiraConnectInstallation < ApplicationRecord end def oauth_authorization_url - return Gitlab.config.gitlab.url if instance_url.blank? || Feature.disabled?(:jira_connect_oauth_self_managed) + return Gitlab.config.gitlab.url if instance_url.blank? instance_url end diff --git a/app/models/key.rb b/app/models/key.rb index 1f2234129ed..596186276bb 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -11,6 +11,8 @@ class Key < ApplicationRecord belongs_to :user + has_many :ssh_signatures, class_name: 'CommitSignatures::SshSignature' + before_validation :generate_fingerprint validates :title, @@ -136,6 +138,10 @@ class Key < ApplicationRecord save if generate_fingerprint end + def signing? + super || auth_and_signing? + end + private def generate_fingerprint diff --git a/app/models/legacy_diff_discussion.rb b/app/models/legacy_diff_discussion.rb index 7d78c580fa2..984205044a7 100644 --- a/app/models/legacy_diff_discussion.rb +++ b/app/models/legacy_diff_discussion.rb @@ -27,10 +27,10 @@ class LegacyDiffDiscussion < Discussion true end - def active?(*args) + def active?(...) return @active if @active.present? - @active = first_note.active?(*args) + @active = first_note.active?(...) end def collapsed? diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index e1f28c0e117..2619a7cca99 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -12,7 +12,6 @@ class LfsObject < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: LfsObjectUploader::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: LfsObjectUploader::Store::REMOTE) } scope :for_oids, -> (oids) { where(oid: oids) } - scope :for_oid_and_size, -> (oid, size) { find_by(oid: oid, size: size) } validates :oid, presence: true, uniqueness: true, format: { with: /\A\h{64}\z/ } @@ -20,6 +19,10 @@ class LfsObject < ApplicationRecord BATCH_SIZE = 3000 + def self.for_oid_and_size(oid, size) + find_by(oid: oid, size: size) + end + def self.not_linked_to_project(project) where('NOT EXISTS (?)', project.lfs_objects_projects.select(1).where('lfs_objects_projects.lfs_object_id = lfs_objects.id')) diff --git a/app/models/main_clusterwide/application_record.rb b/app/models/main_clusterwide/application_record.rb new file mode 100644 index 00000000000..dc61ea695c8 --- /dev/null +++ b/app/models/main_clusterwide/application_record.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module MainClusterwide + class ApplicationRecord < ::ApplicationRecord + self.abstract_class = true + + if Gitlab::Database.has_config?(:main_clusterwide) + connects_to database: { writing: :main_clusterwide, reading: :main_clusterwide } + end + end +end diff --git a/app/models/member.rb b/app/models/member.rb index ecf9013f197..e97c9e929ac 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -22,7 +22,6 @@ class Member < ApplicationRecord STATE_AWAITING = 1 attr_accessor :raw_invite_token - attr_writer :blocking_refresh belongs_to :created_by, class_name: "User" belongs_to :user @@ -279,12 +278,8 @@ class Member < ApplicationRecord after_save :log_invitation_token_cleanup after_commit :send_request, if: :request?, unless: :importing?, on: [:create] - after_commit on: [:create, :update], unless: :importing? do - refresh_member_authorized_projects(blocking: blocking_refresh) - end - - after_commit on: [:destroy], unless: :importing? do - refresh_member_authorized_projects(blocking: false) + after_commit on: [:create, :update, :destroy], unless: :importing? do + refresh_member_authorized_projects end attribute :notification_level, default: -> { NotificationSetting.levels[:global] } @@ -555,8 +550,8 @@ class Member < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass # This method is overridden in the test environment, see stubbed_member.rb - def refresh_member_authorized_projects(blocking:) - UserProjectAccessChangedService.new(user_id).execute(blocking: blocking) + def refresh_member_authorized_projects + UserProjectAccessChangedService.new(user_id).execute end # rubocop: enable CodeReuse/ServiceClass @@ -642,12 +637,6 @@ class Member < ApplicationRecord error = StandardError.new("Invitation token is present but invite was already accepted!") Gitlab::ErrorTracking.track_exception(error, attributes.slice(%w["invite_accepted_at created_at source_type source_id user_id id"])) end - - def blocking_refresh - return true if @blocking_refresh.nil? - - @blocking_refresh - end end Member.prepend_mod_with('Member') diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 796b05b7fff..f23d7208b6e 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -64,7 +64,7 @@ class GroupMember < Member private override :refresh_member_authorized_projects - def refresh_member_authorized_projects(blocking:) + def refresh_member_authorized_projects # Here, `destroyed_by_association` will be present if the # GroupMember is being destroyed due to the `dependent: :destroy` # callback on Group. In this case, there is no need to refresh the diff --git a/app/models/members/member_role.rb b/app/models/members/member_role.rb index 36cbc97d049..42ce228c318 100644 --- a/app/models/members/member_role.rb +++ b/app/models/members/member_role.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass +class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass include IgnorableColumns ignore_column :download_code, remove_with: '15.9', remove_after: '2023-01-22' @@ -15,6 +15,8 @@ class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass validates_associated :members + before_destroy :prevent_delete_after_member_associated + private def belongs_to_top_level_namespace @@ -35,4 +37,13 @@ class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass errors.add(:base, s_("MemberRole|cannot be changed because it is already assigned to a user. "\ "Please create a new Member Role instead")) end + + def prevent_delete_after_member_associated + return unless members.present? + + errors.add(:base, s_("MemberRole|cannot be deleted because it is already assigned to a user. "\ + "Please disassociate the member role from all users before deletion.")) + + throw :abort # rubocop:disable Cop/BanCatchThrow + end end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 6aa6afb595d..733b7c4bc87 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -109,28 +109,24 @@ class ProjectMember < Member end end + # This method is overridden in the test environment, see stubbed_member.rb override :refresh_member_authorized_projects - def refresh_member_authorized_projects(blocking:) + def refresh_member_authorized_projects return unless user - # rubocop:disable CodeReuse/ServiceClass - if blocking - blocking_project_authorizations_refresh - else - AuthorizedProjectUpdate::ProjectRecalculatePerUserWorker.perform_async(project.id, user.id) - end + execute_project_authorizations_refresh + # rubocop:disable CodeReuse/ServiceClass # Until we compare the inconsistency rates of the new, specialized service and # the old approach, we still run AuthorizedProjectsWorker # but with some delay and lower urgency as a safety net. UserProjectAccessChangedService.new(user_id) - .execute(blocking: false, priority: UserProjectAccessChangedService::LOW_PRIORITY) + .execute(priority: UserProjectAccessChangedService::LOW_PRIORITY) # rubocop:enable CodeReuse/ServiceClass end - # This method is overridden in the test environment, see stubbed_member.rb - def blocking_project_authorizations_refresh - AuthorizedProjectUpdate::ProjectRecalculatePerUserWorker.bulk_perform_and_wait([[project.id, user.id]]) + def execute_project_authorizations_refresh + AuthorizedProjectUpdate::ProjectRecalculatePerUserWorker.perform_async(project.id, user.id) end # TODO: https://gitlab.com/groups/gitlab-org/-/epics/7054 diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 0012f098ab2..485ca3a3850 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -194,9 +194,7 @@ class MergeRequest < ApplicationRecord end before_transition any => :merged do |merge_request| - if ::Feature.enabled?(:reset_merge_error_on_transition, merge_request.project) - merge_request.merge_error = nil - end + merge_request.merge_error = nil end after_transition any => :opened do |merge_request| @@ -289,7 +287,7 @@ class MergeRequest < ApplicationRecord validates :merge_user, presence: true, if: :auto_merge_enabled?, unless: :importing? validate :validate_branches, unless: [:allow_broken, :importing?, :closed_or_merged_without_fork?] validate :validate_fork, unless: :closed_or_merged_without_fork? - validate :validate_target_project, on: :create + validate :validate_target_project, on: :create, unless: :importing? validate :validate_reviewer_size_length, unless: :importing? scope :by_source_or_target_branch, ->(branch_name) do @@ -394,6 +392,7 @@ class MergeRequest < ApplicationRecord scope :order_closed_at_desc, -> { order_by_metric(:latest_closed_at, 'DESC') } scope :preload_source_project, -> { preload(:source_project) } scope :preload_target_project, -> { preload(:target_project) } + scope :preload_target_project_with_namespace, -> { preload(target_project: [:namespace]) } scope :preload_routables, -> do preload(target_project: [:route, { namespace: :route }], source_project: [:route, { namespace: :route }]) @@ -1017,7 +1016,6 @@ class MergeRequest < ApplicationRecord end def validate_reviewer_size_length - return true unless Feature.enabled?(:limit_reviewer_and_assignee_size) return true unless reviewers.size > MAX_NUMBER_OF_ASSIGNEES_OR_REVIEWERS errors.add :reviewers, @@ -2019,6 +2017,18 @@ class MergeRequest < ApplicationRecord Feature.enabled?(:hide_merge_requests_from_banned_users) && author&.banned? end + def diffs_batch_cache_with_max_age? + Feature.enabled?(:diffs_batch_cache_with_max_age, project) + end + + def prepared? + prepared_at.present? + end + + def prepare + NewMergeRequestWorker.perform_async(id, author_id) + end + private attr_accessor :skip_fetch_ref @@ -2070,7 +2080,11 @@ class MergeRequest < ApplicationRecord end def report_type_enabled?(report_type) - !!actual_head_pipeline&.batch_lookup_report_artifact_for_file_type(report_type) + if report_type == :license_scanning + ::Gitlab::LicenseScanning.scanner_for_pipeline(project, actual_head_pipeline).has_data? + else + !!actual_head_pipeline&.batch_lookup_report_artifact_for_file_type(report_type) + end end end diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb index c546a5a0025..87d8704561f 100644 --- a/app/models/merge_request/metrics.rb +++ b/app/models/merge_request/metrics.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class MergeRequest::Metrics < ApplicationRecord + include IgnorableColumns + belongs_to :merge_request, inverse_of: :metrics belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :pipeline_id belongs_to :latest_closed_by, class_name: 'User' @@ -14,6 +16,8 @@ class MergeRequest::Metrics < ApplicationRecord scope :with_valid_time_to_merge, -> { where(arel_table[:merged_at].gt(arel_table[:created_at])) } scope :by_target_project, ->(project) { where(target_project_id: project) } + ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' + class << self def time_to_merge_expression Arel.sql('EXTRACT(epoch FROM SUM(AGE(merge_request_metrics.merged_at, merge_request_metrics.created_at)))') diff --git a/app/models/merge_request_user_mention.rb b/app/models/merge_request_user_mention.rb index 222d9c1aa8c..d946fd14628 100644 --- a/app/models/merge_request_user_mention.rb +++ b/app/models/merge_request_user_mention.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class MergeRequestUserMention < UserMention + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' + belongs_to :merge_request belongs_to :note end diff --git a/app/models/ml/candidate.rb b/app/models/ml/candidate.rb index 3ea46a8b703..f973b00c568 100644 --- a/app/models/ml/candidate.rb +++ b/app/models/ml/candidate.rb @@ -2,6 +2,8 @@ module Ml class Candidate < ApplicationRecord + include Sortable + PACKAGE_PREFIX = 'ml_candidate_' enum status: { running: 0, scheduled: 1, finished: 2, failed: 3, killed: 4 } @@ -19,6 +21,30 @@ module Ml attribute :iid, default: -> { SecureRandom.uuid } scope :including_relationships, -> { includes(:latest_metrics, :params, :user) } + scope :by_name, ->(name) { where("ml_candidates.name LIKE ?", "%#{sanitize_sql_like(name)}%") } # rubocop:disable GitlabSecurity/SqlInjection + scope :order_by_metric, ->(metric, direction) do + subquery = Ml::CandidateMetric.latest.where(name: metric) + column_expression = Arel::Table.new('latest')[:value] + metric_order_expression = direction.to_sym == :desc ? column_expression.desc : column_expression.asc + + joins("INNER JOIN (#{subquery.to_sql}) latest ON latest.candidate_id = ml_candidates.id") + .select("ml_candidates.*", "latest.value as metric_value") + .order( + Gitlab::Pagination::Keyset::Order.build( + [ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'metric_value', + order_expression: metric_order_expression, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: arel_table[:id].desc + ) + ]) + ) + end delegate :project_id, :project, to: :experiment diff --git a/app/models/ml/experiment.rb b/app/models/ml/experiment.rb index 0a326b0e005..7bb80a170c5 100644 --- a/app/models/ml/experiment.rb +++ b/app/models/ml/experiment.rb @@ -12,6 +12,12 @@ module Ml has_many :candidates, class_name: 'Ml::Candidate' has_many :metadata, class_name: 'Ml::ExperimentMetadata' + scope :with_candidate_count, -> { + left_outer_joins(:candidates) + .select("ml_experiments.*, count(ml_candidates.id) as candidate_count") + .group(:id) + } + has_internal_id :iid, scope: :project class << self diff --git a/app/models/namespace.rb b/app/models/namespace.rb index cf638f9b16c..9d9b09e3562 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -35,8 +35,6 @@ class Namespace < ApplicationRecord SHARED_RUNNERS_SETTINGS = [SR_DISABLED_AND_UNOVERRIDABLE, SR_DISABLED_WITH_OVERRIDE, SR_DISABLED_AND_OVERRIDABLE, SR_ENABLED].freeze URL_MAX_LENGTH = 255 - PATH_TRAILING_VIOLATIONS = %w[.git .atom .].freeze - # This date is just a placeholder until namespace storage enforcement timeline is confirmed at which point # this should be replaced, see https://about.gitlab.com/pricing/faq-efficient-free-tier/#user-limits-on-gitlab-saas-free-tier MIN_STORAGE_ENFORCEMENT_DATE = 3.months.from_now.to_date @@ -85,6 +83,8 @@ class Namespace < ApplicationRecord has_many :timelog_categories, class_name: 'TimeTracking::TimelogCategory' has_many :achievements, class_name: 'Achievements::Achievement' has_many :namespace_commit_emails, class_name: 'Users::NamespaceCommitEmail' + has_many :cycle_analytics_stages, class_name: 'Analytics::CycleAnalytics::Stage', foreign_key: :group_id, inverse_of: :namespace + has_many :value_streams, class_name: 'Analytics::CycleAnalytics::ValueStream', foreign_key: :group_id, inverse_of: :namespace validates :owner, presence: true, if: ->(n) { n.owner_required? } validates :name, @@ -141,12 +141,14 @@ class Namespace < ApplicationRecord :npm_package_requests_forwarding, to: :package_settings + before_save :update_new_emails_created_column, if: -> { emails_disabled_changed? } before_create :sync_share_with_group_lock_with_parent before_update :sync_share_with_group_lock_with_parent, if: :parent_changed? after_update :force_share_with_group_lock_on_descendants, if: -> { saved_change_to_share_with_group_lock? && share_with_group_lock? } after_update :expire_first_auto_devops_config_cache, if: -> { saved_change_to_auto_devops_enabled? } after_update :move_dir, if: :saved_change_to_path_or_parent?, unless: -> { is_a?(Namespaces::ProjectNamespace) } after_destroy :rm_dir + after_save :reload_namespace_details after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } @@ -240,27 +242,9 @@ class Namespace < ApplicationRecord end def clean_path(path, limited_to: Namespace.all) - path = path.dup - # Get the email username by removing everything after an `@` sign. - path.gsub!(/@.*\z/, "") - # Remove everything that's not in the list of allowed characters. - path.gsub!(/[^a-zA-Z0-9_\-\.]/, "") - # Remove trailing violations ('.atom', '.git', or '.') - loop do - orig = path - PATH_TRAILING_VIOLATIONS.each { |ext| path = path.chomp(ext) } - break if orig == path - end - - # Remove leading violations ('-') - path.gsub!(/\A\-+/, "") - - # Users with the great usernames of "." or ".." would end up with a blank username. - # Work around that by setting their username to "blank", followed by a counter. - path = "blank" if path.blank? - - uniquify = Uniquify.new - uniquify.string(path) { |s| limited_to.find_by_path_or_name(s) } + slug = Gitlab::Slug::Path.new(path).generate + path = Namespaces::RandomizedSuffixPath.new(slug) + Uniquify.new.string(path) { |s| limited_to.find_by_path_or_name(s) } end def clean_name(value) @@ -617,6 +601,17 @@ class Namespace < ApplicationRecord private + def update_new_emails_created_column + return if namespace_settings.nil? + return if namespace_settings.emails_enabled == !emails_disabled + + if namespace_settings.persisted? + namespace_settings.update!(emails_enabled: !emails_disabled) + elsif namespace_settings + namespace_settings.emails_enabled = !emails_disabled + end + end + def cluster_enabled_granted? (Gitlab.com? || Gitlab.dev_or_test_env?) && root_ancestor.cluster_enabled_grant.present? end @@ -678,7 +673,6 @@ class Namespace < ApplicationRecord groups_requiring_authorizations_refresh.find_each do |group| group.refresh_members_authorized_projects( - blocking: false, priority: priority ) end diff --git a/app/models/namespace/detail.rb b/app/models/namespace/detail.rb index a5643ab9f79..2660d11171e 100644 --- a/app/models/namespace/detail.rb +++ b/app/models/namespace/detail.rb @@ -11,3 +11,5 @@ class Namespace::Detail < ApplicationRecord self.primary_key = :namespace_id end + +Namespace::Detail.prepend_mod diff --git a/app/models/namespaces/randomized_suffix_path.rb b/app/models/namespaces/randomized_suffix_path.rb new file mode 100644 index 00000000000..586d7bff5c3 --- /dev/null +++ b/app/models/namespaces/randomized_suffix_path.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Namespaces + class RandomizedSuffixPath + MAX_TRIES = 4 + LEADING_ZEROS = /^0+/.freeze + + def initialize(path) + @path = path + end + + def call(new_count) + @count = new_count.to_i + to_s + end + + def to_s + "#{path}#{suffix}" + end + + private + + attr_reader :count, :path + + def randomized_suffix + Time.current.strftime('%L%M%V').sub(LEADING_ZEROS, '').to_i + offset + end + + def offset + count - MAX_TRIES - 1 + end + + def suffix + return if count.nil? + return randomized_suffix if count > MAX_TRIES + return count if count > 0 + end + end +end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 16a9c20dfdc..0e9760832af 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -47,6 +47,9 @@ module Namespaces # This uses rails internal before_commit API to sync traversal_ids on namespace create, right before transaction is committed. # This helps reduce the time during which the root namespace record is locked to ensure updated traversal_ids are valid before_commit :sync_traversal_ids, on: [:create] + after_commit :set_traversal_ids, + if: -> { traversal_ids.empty? || saved_change_to_parent_id? }, + on: [:create, :update] define_model_callbacks :sync_traversal_ids end @@ -78,6 +81,15 @@ module Namespaces end end + def traversal_ids=(ids) + super(ids) + self.transient_traversal_ids = nil + end + + def traversal_ids + read_attribute(:traversal_ids).presence || transient_traversal_ids || [] + end + def use_traversal_ids? return false unless Feature.enabled?(:use_traversal_ids) @@ -174,12 +186,11 @@ module Namespaces # we need to preserve those specific parameters for super. hierarchy_order ||= :desc - # Get all ancestor IDs inclusively between top and our parent. - top_index = top ? traversal_ids.find_index(top.id) : 0 - ids = traversal_ids[top_index...-1] - ids_string = ids.map { |id| Integer(id) }.join(',') + top_index = ancestors_upto_top_index(top) + ids = traversal_ids[top_index...-1].reverse # WITH ORDINALITY lets us order the result to match traversal_ids order. + ids_string = ids.map { |id| Integer(id) }.join(',') from_sql = <<~SQL unnest(ARRAY[#{ids_string}]::bigint[]) WITH ORDINALITY AS ancestors(id, ord) INNER JOIN namespaces ON namespaces.id = ancestors.id @@ -206,6 +217,8 @@ module Namespaces private + attr_accessor :transient_traversal_ids + # Update the traversal_ids for the full hierarchy. # # NOTE: self.traversal_ids will be stale. Reload for a fresh record. @@ -218,6 +231,27 @@ module Namespaces end end + def set_traversal_ids + # This is a temporary guard and will be removed. + return if is_a?(Namespaces::ProjectNamespace) + + return unless Feature.enabled?(:set_traversal_ids_on_save, root_ancestor) + + self.transient_traversal_ids = if parent_id + parent.traversal_ids + [id] + else + [id] + end + + # Clear root_ancestor memo if changed. + if read_attribute(traversal_ids)&.first != transient_traversal_ids.first + clear_memoization(:root_ancestor) + end + + # Update traversal_ids for any associated child objects. + children.each(&:reload) if children.loaded? + end + # Lock the root of the hierarchy we just left, and lock the root of the hierarchy # we just joined. In most cases the two hierarchies will be the same. def lock_both_roots @@ -266,6 +300,17 @@ module Namespaces skope end + + def ancestors_upto_top_index(top) + return 0 if top.nil? + + index = traversal_ids.find_index(top.id) + if index.nil? + 0 + else + index + 1 + end + end end end end diff --git a/app/models/note.rb b/app/models/note.rb index 73c8e72d8b0..a64f7311725 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -23,6 +23,9 @@ class Note < ApplicationRecord include FromUnion include Sortable include EachBatch + include IgnorableColumns + + ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' ISSUE_TASK_SYSTEM_NOTE_PATTERN = /\A.*marked\sthe\stask.+as\s(completed|incomplete).*\z/.freeze @@ -138,8 +141,7 @@ class Note < ApplicationRecord relations = [{ project: :group }, { author: :status }, :updated_by, :resolved_by, :award_emoji, { system_note_metadata: :description_version }, :suggestions] - if noteable.nil? || DiffNote.noteable_types.include?(noteable.class.name) || - Feature.disabled?(:skip_notes_diff_include) + if noteable.nil? || DiffNote.noteable_types.include?(noteable.class.name) relations += [:note_diff_file, :diff_note_positions] end @@ -183,6 +185,39 @@ class Note < ApplicationRecord after_commit :notify_after_create, on: :create after_commit :notify_after_destroy, on: :destroy + after_commit :trigger_note_subscription_create, on: :create + after_commit :trigger_note_subscription_update, on: :update + after_commit :trigger_note_subscription_destroy, on: :destroy + + def trigger_note_subscription_create + return unless trigger_note_subscription? + + GraphqlTriggers.work_item_note_created(noteable.to_work_item_global_id, self) + end + + def trigger_note_subscription_update + return unless trigger_note_subscription? + + GraphqlTriggers.work_item_note_updated(noteable.to_work_item_global_id, self) + end + + def trigger_note_subscription_destroy + return unless trigger_note_subscription? + + # when deleting a note, we cannot pass it on as a Note instance, as GitlabSchema.object_from_id + # would try to resolve the given Note and fetch it from DB which would raise NotFound exception. + # So instead we just pass over the string representations of the note and discussion IDs, + # so that the subscriber can identify the discussion and the note. + deleted_note_data = { + id: self.id, + model_name: self.class.name, + discussion_id: self.discussion_id, + last_discussion_note: discussion.notes == [self] + } + + GraphqlTriggers.work_item_note_deleted(noteable.to_work_item_global_id, deleted_note_data) + end + class << self extend Gitlab::Utils::Override @@ -711,8 +746,18 @@ class Note < ApplicationRecord confidential? ? :read_internal_note : :read_note end + def exportable_record?(user) + return true unless system? + + readable_by?(user) + end + private + def trigger_note_subscription? + for_issue? && noteable + end + def system_note_viewable_by?(user) return true unless system_note_metadata diff --git a/app/models/note_diff_file.rb b/app/models/note_diff_file.rb index 67a6d5d6d6b..4238de0a2f8 100644 --- a/app/models/note_diff_file.rb +++ b/app/models/note_diff_file.rb @@ -2,6 +2,9 @@ class NoteDiffFile < ApplicationRecord include DiffFile + include IgnorableColumns + + ignore_column :diff_note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' scope :referencing_sha, -> (oids, project_id:) do joins(:diff_note).where(notes: { project_id: project_id, commit_id: oids }) diff --git a/app/models/onboarding/completion.rb b/app/models/onboarding/completion.rb index 49fdb102209..269283df826 100644 --- a/app/models/onboarding/completion.rb +++ b/app/models/onboarding/completion.rb @@ -6,13 +6,13 @@ module Onboarding include Gitlab::Experiment::Dsl ACTION_ISSUE_IDS = { - pipeline_created: 7, trial_started: 2, required_mr_approvals_enabled: 11, code_owners_enabled: 10 }.freeze ACTION_PATHS = [ + :pipeline_created, :issue_created, :git_write, :merge_request_created, diff --git a/app/models/onboarding/learn_gitlab.rb b/app/models/onboarding/learn_gitlab.rb deleted file mode 100644 index d7a189ed6e2..00000000000 --- a/app/models/onboarding/learn_gitlab.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Onboarding - class LearnGitlab - PROJECT_NAME = 'Learn GitLab' - PROJECT_NAME_ULTIMATE_TRIAL = 'Learn GitLab - Ultimate trial' - BOARD_NAME = 'GitLab onboarding' - LABEL_NAME = 'Novice' - - def initialize(current_user) - @current_user = current_user - end - - def available? - project && board && label - end - - def project - @project ||= current_user.projects.find_by_name([PROJECT_NAME, PROJECT_NAME_ULTIMATE_TRIAL]) - end - - def board - return unless project - - @board ||= project.boards.find_by_name(BOARD_NAME) - end - - def label - return unless project - - @label ||= project.labels.find_by_name(LABEL_NAME) - end - - private - - attr_reader :current_user - end -end diff --git a/app/models/package_metadata/application_record.rb b/app/models/package_metadata/application_record.rb new file mode 100644 index 00000000000..1bf0222ada4 --- /dev/null +++ b/app/models/package_metadata/application_record.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module PackageMetadata + class ApplicationRecord < ::ApplicationRecord + self.abstract_class = true + + def self.table_name_prefix + 'pm_' + end + end +end diff --git a/app/models/packages/composer/metadatum.rb b/app/models/packages/composer/metadatum.rb index 363858a3ed1..8b0b71ca86f 100644 --- a/app/models/packages/composer/metadatum.rb +++ b/app/models/packages/composer/metadatum.rb @@ -10,8 +10,18 @@ module Packages validates :package, :target_sha, :composer_json, presence: true + validate :composer_package_type + scope :for_package, ->(name, project_id) { joins(:package).where(packages_packages: { name: name, project_id: project_id, package_type: Packages::Package.package_types[:composer] }) } scope :locked_for_update, -> { lock('FOR UPDATE') } + + private + + def composer_package_type + return if package&.composer? + + errors.add(:base, _('Package type must be Composer')) + end end end end diff --git a/app/models/packages/debian.rb b/app/models/packages/debian.rb index 2daafe0ebcf..9c615c20250 100644 --- a/app/models/packages/debian.rb +++ b/app/models/packages/debian.rb @@ -2,6 +2,8 @@ module Packages module Debian + TEMPORARY_PACKAGE_NAME = 'debian-temporary-package' + DISTRIBUTION_REGEX = %r{[a-z0-9][a-z0-9.-]*}i.freeze COMPONENT_REGEX = DISTRIBUTION_REGEX.freeze ARCHITECTURE_REGEX = %r{[a-z0-9][-a-z0-9]*}.freeze diff --git a/app/models/packages/debian/file_entry.rb b/app/models/packages/debian/file_entry.rb index b70b6c460d2..eb66f4acfa9 100644 --- a/app/models/packages/debian/file_entry.rb +++ b/app/models/packages/debian/file_entry.rb @@ -4,7 +4,6 @@ module Packages module Debian class FileEntry include ActiveModel::Model - include ::Packages::FIPS DIGESTS = %i[md5 sha1 sha256].freeze FILENAME_REGEX = %r{\A[a-zA-Z0-9][a-zA-Z0-9_.~+-]*\z}.freeze @@ -32,8 +31,6 @@ module Packages private def valid_package_file_digests - raise DisabledError, 'Debian registry is not FIPS compliant' if Gitlab::FIPS.enabled? - DIGESTS.each do |digest| package_file_digest = package_file["file_#{digest}"] sum = public_send("#{digest}sum") # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/models/packages/debian/file_metadatum.rb b/app/models/packages/debian/file_metadatum.rb index af51f256e18..eb1b03a8e9d 100644 --- a/app/models/packages/debian/file_metadatum.rb +++ b/app/models/packages/debian/file_metadatum.rb @@ -13,10 +13,11 @@ class Packages::Debian::FileMetadatum < ApplicationRecord } validates :file_type, presence: true - validates :file_type, inclusion: { in: %w[unknown] }, if: -> { package_file&.package&.debian_incoming? } + validates :file_type, inclusion: { in: %w[unknown] }, + if: -> { package_file&.package&.debian_incoming? || package_file&.package&.processing? } validates :file_type, inclusion: { in: %w[source dsc deb udeb buildinfo changes] }, - if: -> { package_file&.package&.debian_package? } + if: -> { package_file&.package&.debian_package? && !package_file&.package&.processing? } validates :component, presence: true, diff --git a/app/models/packages/debian/group_distribution.rb b/app/models/packages/debian/group_distribution.rb index 01938f4a2ec..dba38c1b538 100644 --- a/app/models/packages/debian/group_distribution.rb +++ b/app/models/packages/debian/group_distribution.rb @@ -10,6 +10,7 @@ class Packages::Debian::GroupDistribution < ApplicationRecord def packages Packages::Package .for_projects(group.all_projects.public_only) + .debian .with_debian_codename(codename) end end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 966165f9ad7..970538b45e7 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -138,10 +138,12 @@ class Packages::Package < ApplicationRecord joins(:conan_metadatum).where(packages_conan_metadata: { package_username: package_username }) end - scope :with_debian_codename, -> (codename) do - debian - .joins(:debian_distribution) - .where(Packages::Debian::ProjectDistribution.table_name => { codename: codename }) + scope :with_debian_codename, ->(codename) do + joins(:debian_distribution).where(Packages::Debian::ProjectDistribution.table_name => { codename: codename }) + end + scope :with_debian_codename_or_suite, ->(codename_or_suite) do + joins(:debian_distribution).where(Packages::Debian::ProjectDistribution.table_name => { codename: codename_or_suite }) + .or(where(Packages::Debian::ProjectDistribution.table_name => { suite: codename_or_suite })) end scope :preload_debian_file_metadata, -> { preload(package_files: :debian_file_metadatum) } scope :with_composer_target, -> (target) do @@ -160,7 +162,8 @@ class Packages::Package < ApplicationRecord scope :preload_files, -> { preload(:installable_package_files) } scope :preload_nuget_files, -> { preload(:installable_nuget_package_files) } scope :preload_pipelines, -> { preload(pipelines: :user) } - scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) } + scope :last_of_each_version, -> { where(id: all.last_of_each_version_ids) } + scope :last_of_each_version_ids, -> { select('MAX(id) AS id').unscope(where: :id).group(:version) } scope :limit_recent, ->(limit) { order_created_desc.limit(limit) } scope :select_distinct_name, -> { select(:name).distinct } @@ -277,6 +280,7 @@ class Packages::Package < ApplicationRecord project.packages .preload_pipelines .including_tags + .displayable .with_name(name) .where.not(version: version) .with_package_type(package_type) diff --git a/app/models/packages/tag.rb b/app/models/packages/tag.rb index 14a1ae98ed4..9c17a147bf4 100644 --- a/app/models/packages/tag.rb +++ b/app/models/packages/tag.rb @@ -10,8 +10,8 @@ class Packages::Tag < ApplicationRecord scope :preload_package, -> { preload(:package) } scope :with_name, -> (name) { where(name: name) } - def self.for_packages(packages) - where(package_id: packages.select(:id)) + def self.for_package_ids(package_ids) + where(package_id: package_ids) .order(updated_at: :desc) .limit(FOR_PACKAGES_TAGS_LIMIT) end diff --git a/app/models/performance_monitoring/prometheus_dashboard.rb b/app/models/performance_monitoring/prometheus_dashboard.rb index 37bf080ae49..6fea3abf3d9 100644 --- a/app/models/performance_monitoring/prometheus_dashboard.rb +++ b/app/models/performance_monitoring/prometheus_dashboard.rb @@ -58,7 +58,7 @@ module PerformanceMonitoring rescue Gitlab::Metrics::Dashboard::Errors::LayoutError => e [e.message] rescue ActiveModel::ValidationError => e - e.model.errors.map { |attr, error| "#{attr}: #{error}" } + e.model.errors.map { |error| "#{error.attribute}: #{error.message}" } end private diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 0da205f86a5..f99c4c6c39d 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -24,7 +24,7 @@ class PersonalAccessToken < ApplicationRecord # During the implementation of Admin Mode for API, tokens of # administrators should automatically get the `admin_mode` scope as well # See https://gitlab.com/gitlab-org/gitlab/-/issues/42692 - before_create :add_admin_mode_scope, if: :user_admin? + before_create :add_admin_mode_scope, if: -> { Feature.disabled?(:admin_mode_for_api) && user_admin? } scope :active, -> { not_revoked.not_expired } scope :expiring_and_not_notified, ->(date) { where(["revoked = false AND expire_notification_delivered = false AND expires_at >= CURRENT_DATE AND expires_at <= ?", date]) } @@ -84,10 +84,8 @@ class PersonalAccessToken < ApplicationRecord protected def validate_scopes - # During the implementation of Admin Mode for API, - # the `admin_mode` scope is not yet part of `all_available_scopes` but still valid. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/42692 - valid_scopes = Gitlab::Auth.all_available_scopes + [Gitlab::Auth::ADMIN_MODE_SCOPE] + valid_scopes = Gitlab::Auth.all_available_scopes + valid_scopes += [Gitlab::Auth::ADMIN_MODE_SCOPE] if Feature.disabled?(:admin_mode_for_api) unless revoked || scopes.all? { |scope| valid_scopes.include?(scope.to_sym) } errors.add :scopes, "can only contain available scopes" diff --git a/app/models/plan_limits.rb b/app/models/plan_limits.rb index bf08da6a1e1..bf69f425189 100644 --- a/app/models/plan_limits.rb +++ b/app/models/plan_limits.rb @@ -2,8 +2,8 @@ class PlanLimits < ApplicationRecord include IgnorableColumns - ignore_column :ci_max_artifact_size_running_container_scanning, remove_with: '14.3', remove_after: '2021-08-22' + ignore_column :web_hook_calls_high, remove_with: '15.10', remove_after: '2022-02-22' LimitUndefinedError = Class.new(StandardError) diff --git a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb index 8df986b47a2..0c747ad9c84 100644 --- a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb @@ -32,11 +32,12 @@ module Preloaders end def preload_with_traversal_ids - max_access_levels = GroupMember.active_without_invites_and_requests - .where(user: @user) - .joins("INNER JOIN (#{traversal_join_sql}) as hierarchy ON members.source_id = hierarchy.traversal_id") - .group('hierarchy.id') - .maximum(:access_level) + # Diagrammatic representation of this step: + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111157#note_1271550140 + max_access_levels = GroupMember.from_union(all_memberships) + .joins("INNER JOIN (#{traversal_join_sql}) as hierarchy ON members.source_id = hierarchy.traversal_id") + .group('hierarchy.id') + .maximum(:access_level) @groups.each do |group| max_access_level = max_access_levels[group.id] || Gitlab::Access::NO_ACCESS @@ -44,6 +45,58 @@ module Preloaders end end + def all_memberships + if Feature.enabled?(:include_memberships_from_group_shares_in_preloader) + [ + direct_memberships.select(*GroupMember.cached_column_list), + memberships_from_group_shares + ] + else + [direct_memberships] + end + end + + def direct_memberships + GroupMember.active_without_invites_and_requests.where(user: @user) + end + + def memberships_from_group_shares + alter_direct_memberships_to_make_it_act_like_memberships_in_shared_groups + end + + def alter_direct_memberships_to_make_it_act_like_memberships_in_shared_groups + group_group_link_table = GroupGroupLink.arel_table + group_member_table = GroupMember.arel_table + + altered_columns = GroupMember.attribute_names.map do |column_name| + case column_name + when 'access_level' + # Consider the limiting effect of group share's access level + smallest_value_arel([group_group_link_table[:group_access], group_member_table[:access_level]], 'access_level') + when 'source_id' + # Alter the `source_id` of the `Member` record that is currently pointing to the `shared_with_group` + # such that this record would now behave like a `Member` record of this user pointing to the `shared_group` group. + Arel::Nodes::As.new(group_group_link_table[:shared_group_id], Arel::Nodes::SqlLiteral.new('source_id')) + else + group_member_table[column_name] + end + end + + direct_memberships_in_groups_that_have_been_shared_with_other_groups.select(*altered_columns) + end + + def direct_memberships_in_groups_that_have_been_shared_with_other_groups + direct_memberships.joins( + "INNER JOIN group_group_links ON members.source_id = group_group_links.shared_with_group_id" + ) + end + + def smallest_value_arel(args, column_alias) + Arel::Nodes::As.new( + Arel::Nodes::NamedFunction.new('LEAST', args), + Arel::Nodes::SqlLiteral.new(column_alias)) + end + def traversal_join_sql Namespace.select('id, unnest(traversal_ids) as traversal_id').where(id: @groups.map(&:id)).to_sql end diff --git a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb index c9fd5e7718a..09854ec5ff1 100644 --- a/app/models/preloaders/user_max_access_level_in_projects_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_projects_preloader.rb @@ -7,12 +7,10 @@ module Preloaders def initialize(projects, user) @projects = if projects.is_a?(Array) Project.where(id: projects) - elsif Feature.enabled?(:projects_preloader_fix) + else # Push projects base query in to a sub-select to avoid # table name clashes. Performs better than aliasing. Project.where(id: projects.subquery(:id)) - else - Project.where(id: projects.reselect(:id)) end @user = user diff --git a/app/models/programming_language.rb b/app/models/programming_language.rb index 4156c672518..e3693046423 100644 --- a/app/models/programming_language.rb +++ b/app/models/programming_language.rb @@ -7,7 +7,7 @@ class ProgrammingLanguage < ApplicationRecord # Returns all programming languages which match any of the given names (case # insensitively). scope :with_name_case_insensitive, ->(*names) do - sanitized_names = names.map(&method(:sanitize_sql_like)) + sanitized_names = names.map { |name| sanitize_sql_like(name) } where(arel_table[:name].matches_any(sanitized_names)) end diff --git a/app/models/project.rb b/app/models/project.rb index 561a842f23a..43ec26be786 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -40,6 +40,8 @@ class Project < ApplicationRecord include RunnerTokenExpirationInterval include BlocksUnsafeSerialization include Subquery + include IssueParent + include WebHooks::HasWebHooks extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -118,6 +120,7 @@ class Project < ApplicationRecord before_validation :remove_leading_spaces_on_name after_validation :check_pending_delete before_save :ensure_runners_token + before_save :update_new_emails_created_column, if: -> { emails_disabled_changed? } after_create -> { create_or_load_association(:project_feature) } after_create -> { create_or_load_association(:ci_cd_settings) } @@ -306,6 +309,9 @@ class Project < ApplicationRecord primary_key: :project_namespace_id, foreign_key: :member_namespace_id, inverse_of: :project, class_name: 'ProjectMember' has_many :members_and_requesters, as: :source, class_name: 'ProjectMember' + has_many :namespace_members_and_requesters, -> { unscope(where: %i[source_id source_type]) }, + primary_key: :project_namespace_id, foreign_key: :member_namespace_id, inverse_of: :project, + class_name: 'ProjectMember' has_many :users, through: :project_members @@ -395,9 +401,6 @@ class Project < ApplicationRecord has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :remote_mirrors, inverse_of: :project - has_many :cycle_analytics_stages, class_name: 'Analytics::CycleAnalytics::ProjectStage', inverse_of: :project - has_many :value_streams, class_name: 'Analytics::CycleAnalytics::ProjectValueStream', inverse_of: :project - has_many :external_pull_requests, inverse_of: :project has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_project_id @@ -750,7 +753,7 @@ class Project < ApplicationRecord return public_to_user unless user if user.is_a?(DeployToken) - user.accessible_projects + where(id: user.accessible_projects) else where('EXISTS (?) OR projects.visibility_level IN (?)', user.authorizations_for_projects(min_access_level: min_access_level), @@ -824,6 +827,7 @@ class Project < ApplicationRecord scope :for_group, -> (group) { where(group: group) } scope :for_group_and_its_subgroups, ->(group) { where(namespace_id: group.self_and_descendants.select(:id)) } scope :for_group_and_its_ancestor_groups, ->(group) { where(namespace_id: group.self_and_ancestors.select(:id)) } + scope :is_importing, -> { with_import_state.where(import_state: { status: %w[started scheduled] }) } class << self # Searches for a list of projects based on the query given in `query`. @@ -991,6 +995,13 @@ class Project < ApplicationRecord namespace.owner == user end + def invalidate_personal_projects_count_of_owner + return unless personal? + return unless namespace.owner + + namespace.owner.invalidate_personal_projects_count + end + def project_setting super.presence || build_project_setting end @@ -1249,6 +1260,10 @@ class Project < ApplicationRecord import_state&.status || 'none' end + def import_checksums + import_state&.checksums || {} + end + def jira_import_status latest_jira_import&.status || 'initial' end @@ -2789,6 +2804,18 @@ class Project < ApplicationRecord protected_branches.limit(limit) end + def group_protected_branches + root_namespace.is_a?(Group) ? root_namespace.protected_branches : ProtectedBranch.none + end + + def all_protected_branches + if Feature.enabled?(:group_protected_branches) + @all_protected_branches ||= ProtectedBranch.from_union([protected_branches, group_protected_branches]) + else + protected_branches + end + end + def self_monitoring? Gitlab::CurrentSettings.self_monitoring_project_id == id end @@ -3045,13 +3072,8 @@ class Project < ApplicationRecord group&.work_items_mvc_2_feature_flag_enabled? || Feature.enabled?(:work_items_mvc_2) end - def work_items_create_from_markdown_feature_flag_enabled? - group&.work_items_create_from_markdown_feature_flag_enabled? || Feature.enabled?(:work_items_create_from_markdown) - end - def enqueue_record_project_target_platforms return unless Gitlab.com? - return unless Feature.enabled?(:record_projects_target_platforms, self) Projects::RecordTargetPlatformsWorker.perform_async(id) end @@ -3368,6 +3390,17 @@ class Project < ApplicationRecord ProjectFeature::PRIVATE end end + + def update_new_emails_created_column + return if project_setting.nil? + return if project_setting.emails_enabled == !emails_disabled + + if project_setting.persisted? + project_setting.update!(emails_enabled: !emails_disabled) + elsif project_setting + project_setting.emails_enabled = !emails_disabled + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index 3623b3be20d..cb578496f26 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -31,7 +31,7 @@ class ProjectAuthorization < ApplicationRecord def self.insert_all_in_batches(attributes, per_batch = BATCH_SIZE) add_delay = add_delay_between_batches?(entire_size: attributes.size, batch_size: per_batch) - log_details(entire_size: attributes.size) if add_delay + log_details(entire_size: attributes.size, batch_size: per_batch) if add_delay attributes.each_slice(per_batch) do |attributes_batch| insert_all(attributes_batch) @@ -41,7 +41,7 @@ class ProjectAuthorization < ApplicationRecord def self.delete_all_in_batches_for_project(project:, user_ids:, per_batch: BATCH_SIZE) add_delay = add_delay_between_batches?(entire_size: user_ids.size, batch_size: per_batch) - log_details(entire_size: user_ids.size) if add_delay + log_details(entire_size: user_ids.size, batch_size: per_batch) if add_delay user_ids.each_slice(per_batch) do |user_ids_batch| project.project_authorizations.where(user_id: user_ids_batch).delete_all @@ -51,7 +51,7 @@ class ProjectAuthorization < ApplicationRecord def self.delete_all_in_batches_for_user(user:, project_ids:, per_batch: BATCH_SIZE) add_delay = add_delay_between_batches?(entire_size: project_ids.size, batch_size: per_batch) - log_details(entire_size: project_ids.size) if add_delay + log_details(entire_size: project_ids.size, batch_size: per_batch) if add_delay project_ids.each_slice(per_batch) do |project_ids_batch| user.project_authorizations.where(project_id: project_ids_batch).delete_all @@ -64,14 +64,15 @@ class ProjectAuthorization < ApplicationRecord # catch up with the primary when large batches of records are being added/removed. # Hance, we add a delay only if the GitLab installation has a replica database configured. entire_size > batch_size && - !::Gitlab::Database::LoadBalancing.primary_only? && - Feature.enabled?(:enable_minor_delay_during_project_authorizations_refresh) + !::Gitlab::Database::LoadBalancing.primary_only? end - private_class_method def self.log_details(entire_size:) + private_class_method def self.log_details(entire_size:, batch_size:) Gitlab::AppLogger.info( entire_size: entire_size, - message: 'Project authorizations refresh performed with delay' + total_delay: (entire_size / batch_size.to_f).ceil * SLEEP_DELAY, + message: 'Project authorizations refresh performed with delay', + **Gitlab::ApplicationContext.current ) end diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index cc9003423be..8741a341ad3 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -20,6 +20,10 @@ class ProjectCiCdSetting < ApplicationRecord attribute :forward_deployment_enabled, default: true attribute :separated_caches, default: true + default_value_for :inbound_job_token_scope_enabled do |settings| + Feature.enabled?(:ci_inbound_job_token_scope, settings.project) + end + chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval def keep_latest_artifacts_available? diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 11f4a3f3b6f..168646bbe41 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -63,32 +63,23 @@ class ProjectFeature < ApplicationRecord validate :repository_children_level - default_value_for :builds_access_level, value: ENABLED, allows_nil: false - default_value_for :issues_access_level, value: ENABLED, allows_nil: false - default_value_for :forking_access_level, value: ENABLED, allows_nil: false - default_value_for :merge_requests_access_level, value: ENABLED, allows_nil: false - default_value_for :snippets_access_level, value: ENABLED, allows_nil: false - default_value_for :wiki_access_level, value: ENABLED, allows_nil: false - default_value_for :repository_access_level, value: ENABLED, allows_nil: false - default_value_for :analytics_access_level, value: ENABLED, allows_nil: false - default_value_for :metrics_dashboard_access_level, value: PRIVATE, allows_nil: false - default_value_for :operations_access_level, value: ENABLED, allows_nil: false - default_value_for :security_and_compliance_access_level, value: PRIVATE, allows_nil: false - default_value_for :monitor_access_level, value: ENABLED, allows_nil: false - default_value_for :infrastructure_access_level, value: ENABLED, allows_nil: false - default_value_for :feature_flags_access_level, value: ENABLED, allows_nil: false - default_value_for :environments_access_level, value: ENABLED, allows_nil: false - default_value_for :releases_access_level, value: ENABLED, allows_nil: false - - default_value_for(:pages_access_level, allows_nil: false) do |feature| - if ::Gitlab::Pages.access_control_is_forced? - PRIVATE - else - feature.project&.public? ? ENABLED : PRIVATE - end - end - - default_value_for(:package_registry_access_level) do |feature| + attribute :builds_access_level, default: ENABLED + attribute :issues_access_level, default: ENABLED + attribute :forking_access_level, default: ENABLED + attribute :merge_requests_access_level, default: ENABLED + attribute :snippets_access_level, default: ENABLED + attribute :wiki_access_level, default: ENABLED + attribute :repository_access_level, default: ENABLED + attribute :analytics_access_level, default: ENABLED + attribute :metrics_dashboard_access_level, default: PRIVATE + attribute :operations_access_level, default: ENABLED + attribute :security_and_compliance_access_level, default: PRIVATE + attribute :monitor_access_level, default: ENABLED + attribute :infrastructure_access_level, default: ENABLED + attribute :feature_flags_access_level, default: ENABLED + attribute :environments_access_level, default: ENABLED + + attribute :package_registry_access_level, default: -> do if ::Gitlab.config.packages.enabled ENABLED else @@ -96,7 +87,7 @@ class ProjectFeature < ApplicationRecord end end - default_value_for(:container_registry_access_level) do |feature| + attribute :container_registry_access_level, default: -> do if gitlab_config_features.container_registry ENABLED else @@ -104,6 +95,9 @@ class ProjectFeature < ApplicationRecord end end + after_initialize :set_pages_access_level, if: :new_record? + after_initialize :set_default_values, unless: :new_record? + # "enabled" here means "not disabled". It includes private features! scope :with_feature_enabled, ->(feature) { feature_access_level_attribute = arel_table[access_level_attribute(feature)] @@ -170,6 +164,23 @@ class ProjectFeature < ApplicationRecord private + def set_pages_access_level + self.pages_access_level ||= if ::Gitlab::Pages.access_control_is_forced? + PRIVATE + else + self.project&.public? ? ENABLED : PRIVATE + end + end + + def set_default_values + self.class.column_names.each do |column_name| + next unless has_attribute?(column_name) + next unless read_attribute(column_name).nil? + + write_attribute(column_name, self.class.column_defaults[column_name]) + end + end + # Validates builds and merge requests access level # which cannot be higher than repository access level def repository_children_level diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index 7711c6d604a..f16d661d4bb 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -11,6 +11,7 @@ class ProjectImportState < ApplicationRecord belongs_to :project, inverse_of: :import_state validates :project, presence: true + validates :checksums, json_schema: { filename: "project_import_stats" } alias_attribute :correlation_id, :correlation_id_value @@ -68,6 +69,16 @@ class ProjectImportState < ApplicationRecord state.project.remove_import_data end + before_transition started: [:finished, :canceled, :failed] do |state, _| + project = state.project + + if project.github_import? + import_stats = ::Gitlab::GithubImport::ObjectCounter.summary(state.project) + + state.update_column(:checksums, import_stats) + end + end + after_transition started: :finished do |state, _| project = state.project diff --git a/app/models/projects/data_transfer.rb b/app/models/projects/data_transfer.rb new file mode 100644 index 00000000000..a93aea55781 --- /dev/null +++ b/app/models/projects/data_transfer.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Tracks egress of various services per project +# This class ensures that we keep 1 record per project per month. +module Projects + class DataTransfer < ApplicationRecord + self.table_name = 'project_data_transfers' + + belongs_to :project + belongs_to :namespace + + scope :current_month, -> { where(date: beginning_of_month) } + + def self.beginning_of_month(time = Time.current) + time.utc.beginning_of_month + end + end +end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 050db3b6870..b3331b99a6b 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -3,6 +3,7 @@ class ProtectedBranch < ApplicationRecord include ProtectedRef include Gitlab::SQL::Pattern + include FromUnion belongs_to :group, foreign_key: :namespace_id, touch: true, inverse_of: :protected_branches @@ -11,6 +12,9 @@ class ProtectedBranch < ApplicationRecord scope :requiring_code_owner_approval, -> { where(code_owner_approval_required: true) } scope :allowing_force_push, -> { where(allow_force_push: true) } scope :sorted_by_name, -> { order(name: :asc) } + scope :sorted_by_namespace_and_name, -> { order(:namespace_id, :name) } + + scope :for_group, ->(group) { where(group: group) } protected_ref_access_levels :merge, :push @@ -43,14 +47,12 @@ class ProtectedBranch < ApplicationRecord end def self.new_cache(project, ref_name, dry_run: true) - if Feature.enabled?(:hash_based_cache_for_protected_branches, project) - ProtectedBranches::CacheService.new(project).fetch(ref_name, dry_run: dry_run) do # rubocop: disable CodeReuse/ServiceClass - self.matching(ref_name, protected_refs: protected_refs(project)).present? - end + ProtectedBranches::CacheService.new(project).fetch(ref_name, dry_run: dry_run) do # rubocop: disable CodeReuse/ServiceClass + self.matching(ref_name, protected_refs: protected_refs(project)).present? end end - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/368279 + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/370608 # ---------------------------------------------------------------- CACHE_EXPIRE_IN = 1.hour @@ -66,7 +68,19 @@ class ProtectedBranch < ApplicationRecord # End of deprecation -------------------------------------------- def self.allow_force_push?(project, ref_name) - project.protected_branches.allowing_force_push.matching(ref_name).any? + if Feature.enabled?(:group_protected_branches) + protected_branches = project.all_protected_branches.matching(ref_name) + + project_protected_branches, group_protected_branches = protected_branches.partition(&:project_id) + + # Group owner can be able to enforce the settings + return group_protected_branches.any?(&:allow_force_push) if group_protected_branches.present? + return project_protected_branches.any?(&:allow_force_push) if project_protected_branches.present? + + false + else + project.protected_branches.allowing_force_push.matching(ref_name).any? + end end def self.any_protected?(project, ref_names) @@ -78,7 +92,11 @@ class ProtectedBranch < ApplicationRecord end def self.protected_refs(project) - project.protected_branches + if Feature.enabled?(:group_protected_branches) + project.all_protected_branches + else + project.protected_branches + end end # overridden in EE @@ -104,6 +122,14 @@ class ProtectedBranch < ApplicationRecord name == project.default_branch end + def group_level? + entity.is_a?(Group) + end + + def project_level? + entity.is_a?(Project) + end + def entity group || project end diff --git a/app/models/protected_tag/create_access_level.rb b/app/models/protected_tag/create_access_level.rb index 5d8b1fb4f71..abb233d3800 100644 --- a/app/models/protected_tag/create_access_level.rb +++ b/app/models/protected_tag/create_access_level.rb @@ -4,9 +4,43 @@ class ProtectedTag::CreateAccessLevel < ApplicationRecord include Importable include ProtectedTagAccess + belongs_to :deploy_key + + validates :access_level, uniqueness: { scope: :protected_tag_id, if: :role?, + conditions: -> { where(user_id: nil, group_id: nil, deploy_key_id: nil) } } + validates :deploy_key_id, uniqueness: { scope: :protected_tag_id, allow_nil: true } + validate :validate_deploy_key_membership + + def type + if deploy_key.present? + :deploy_key + else + super + end + end + def check_access(user) return false if access_level == Gitlab::Access::NO_ACCESS + if user && deploy_key.present? + return user.can?(:read_project, project) && enabled_deploy_key_for_user?(deploy_key, user) + end + super end + + private + + def validate_deploy_key_membership + return unless deploy_key + + return if project.deploy_keys_projects.where(deploy_key: deploy_key).exists? + + errors.add(:deploy_key, 'is not enabled for this project') + end + + def enabled_deploy_key_for_user?(deploy_key, user) + deploy_key.user_id == user.id && + DeployKey.with_write_access_for_project(protected_tag.project, deploy_key: deploy_key).any? + end end diff --git a/app/models/release.rb b/app/models/release.rb index b770f3934ef..0f00732b62e 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -11,7 +11,6 @@ class Release < ApplicationRecord cache_markdown_field :description belongs_to :project, touch: true - # releases prior to 11.7 have no author belongs_to :author, class_name: 'User' has_many :links, class_name: 'Releases::Link' @@ -26,7 +25,7 @@ class Release < ApplicationRecord before_create :set_released_at validates :project, :tag, presence: true - validates :author_id, presence: true, if: :validate_release_with_author? + validates :author_id, presence: true, on: :create validates :tag, uniqueness: { scope: :project_id } @@ -119,10 +118,6 @@ class Release < ApplicationRecord end end - def validate_release_with_author? - Feature.enabled?(:validate_release_with_author, self.project) - end - def set_released_at self.released_at ||= created_at end diff --git a/app/models/release_highlight.rb b/app/models/release_highlight.rb index c2d498ecb13..7cead8a42cd 100644 --- a/app/models/release_highlight.rb +++ b/app/models/release_highlight.rb @@ -2,7 +2,6 @@ class ReleaseHighlight CACHE_DURATION = 1.hour - FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml') FREE_PACKAGE = 'Free' PREMIUM_PACKAGE = 'Premium' @@ -48,13 +47,17 @@ class ReleaseHighlight nil end + def self.whats_new_path + Rails.root.join('data/whats_new/*.yml') + end + def self.file_paths @file_paths ||= self.relative_file_paths.map { |path| path.prepend(Rails.root.to_s) } end def self.relative_file_paths Rails.cache.fetch(self.cache_key('file_paths'), expires_in: CACHE_DURATION) do - Dir.glob(FILES_PATH).sort.reverse.map { |path| path.delete_prefix(Rails.root.to_s) } + Dir.glob(whats_new_path).sort.reverse.map { |path| path.delete_prefix(Rails.root.to_s) } end end @@ -119,3 +122,5 @@ class ReleaseHighlight item['available_in']&.include?(current_package) end end + +ReleaseHighlight.prepend_mod diff --git a/app/models/releases/link.rb b/app/models/releases/link.rb index 347adbdf96a..e02486fbc5b 100644 --- a/app/models/releases/link.rb +++ b/app/models/releases/link.rb @@ -37,6 +37,7 @@ module Releases url.start_with?(release.project.web_url) end + # `external?` is deprecated in 15.9 and will be removed in 16.0. def external? !internal? end @@ -44,7 +45,7 @@ module Releases def hook_attrs { id: id, - external: external?, + external: external?, # `external` is deprecated in 15.9 and will be removed in 16.0. link_type: link_type, name: name, url: url diff --git a/app/models/repository.rb b/app/models/repository.rb index cedfed16b20..d15f2a430fa 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -189,9 +189,7 @@ class Repository return [] end - query = Feature.enabled?(:commit_search_trailing_spaces) ? query.strip : query - - commits = raw_repository.find_commits_by_message(query, ref, path, limit, offset).map do |c| + commits = raw_repository.find_commits_by_message(query.strip, ref, path, limit, offset).map do |c| commit(c) end CommitCollection.new(container, commits, ref) @@ -633,11 +631,7 @@ class Repository end def readme_path - if Feature.enabled?(:readme_from_gitaly) - readme_path_gitaly - else - head_tree&.readme_path - end + head_tree&.readme_path end cache_method :readme_path @@ -702,14 +696,14 @@ class Repository end def head_tree(skip_flat_paths: true) - if head_commit - @head_tree ||= Tree.new(self, head_commit.sha, nil, skip_flat_paths: skip_flat_paths) - end + return if empty? || root_ref.nil? + + @head_tree ||= Tree.new(self, root_ref, nil, skip_flat_paths: skip_flat_paths) end def tree(sha = :head, path = nil, recursive: false, skip_flat_paths: true, pagination_params: nil) if sha == :head - return unless head_commit + return if empty? || root_ref.nil? if path.nil? return head_tree(skip_flat_paths: skip_flat_paths) @@ -878,25 +872,45 @@ class Repository end def merge(user, source_sha, merge_request, message) + merge_to_branch(user, + source_sha: source_sha, + target_branch: merge_request.target_branch, + message: message) do |commit_id| + merge_request.update_and_mark_in_progress_merge_commit_sha(commit_id) + nil # Return value does not matter. + end + end + + def merge_to_branch(user, source_sha:, target_branch:, message:, target_sha: nil) with_cache_hooks do - raw_repository.merge(user, source_sha, merge_request.target_branch, message) do |commit_id| - merge_request.update_and_mark_in_progress_merge_commit_sha(commit_id) - nil # Return value does not matter. + raw_repository.merge(user, + source_sha: source_sha, + target_branch: target_branch, + message: message, + target_sha: target_sha + ) do |commit_id| + yield commit_id if block_given? end end end - def delete_refs(*ref_names) - raw.delete_refs(*ref_names) + def delete_refs(...) + raw.delete_refs(...) end - def ff_merge(user, source, target_branch, merge_request: nil) + def ff_merge(user, source, target_branch, target_sha: nil, merge_request: nil) their_commit_id = commit(source)&.id raise 'Invalid merge source' if their_commit_id.nil? merge_request&.update_and_mark_in_progress_merge_commit_sha(their_commit_id) - with_cache_hooks { raw.ff_merge(user, their_commit_id, target_branch) } + with_cache_hooks do + raw.ff_merge(user, + source_sha: their_commit_id, + target_branch: target_branch, + target_sha: target_sha + ) + end end def revert( @@ -1245,29 +1259,6 @@ class Repository container.full_path, container: container) end - - def readme_path_gitaly - return if empty? || root_ref.nil? - - # (?i) to enable case-insensitive mode - # - # Note: `Gitlab::FileDetector::PATTERNS[:readme]#to_s` won't work because of - # incompatibility of regex engines between Rails and Gitaly. - regex = "(?i)#{Gitlab::FileDetector::PATTERNS[:readme].source}" - - readmes = search_files_by_regexp(regex, root_ref) - - choose_readme_to_display(readmes) - end - - # Extracted from Tree#readme_path - def choose_readme_to_display(readmes) - previewable_readme = readmes.find { |name| Gitlab::MarkupHelper.previewable?(name) } - - return previewable_readme if previewable_readme - - readmes.find { |name| Gitlab::MarkupHelper.plain?(name) } - end end Repository.prepend_mod_with('Repository') diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 8fea0d6d993..1a0a65df6a3 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class SentNotification < ApplicationRecord + include IgnorableColumns + serialize :position, Gitlab::Diff::Position # rubocop:disable Cop/ActiveRecordSerialize belongs_to :project @@ -14,6 +16,8 @@ class SentNotification < ApplicationRecord validates :in_reply_to_discussion_id, format: { with: /\A\h{40}\z/, allow_nil: true } validate :note_valid + ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' + after_save :keep_around_commit, if: :for_commit? class << self diff --git a/app/models/service_desk_setting.rb b/app/models/service_desk_setting.rb index 738f18ca5e3..5152746abb4 100644 --- a/app/models/service_desk_setting.rb +++ b/app/models/service_desk_setting.rb @@ -3,6 +3,14 @@ class ServiceDeskSetting < ApplicationRecord include Gitlab::Utils::StrongMemoize + attribute :custom_email_enabled, default: false + attr_encrypted :custom_email_smtp_password, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_iv: false + belongs_to :project validates :project_id, presence: true validate :valid_issue_template @@ -13,8 +21,42 @@ class ServiceDeskSetting < ApplicationRecord allow_blank: true, format: { with: /\A[a-z0-9_]+\z/, message: -> (setting, data) { _("can contain only lowercase letters, digits, and '_'.") } } + validates :custom_email, + length: { maximum: 255 }, + uniqueness: true, + allow_nil: true, + format: /\A[\w\-._]+@[\w\-.]+\.{1}[a-zA-Z]{2,}\z/ + validates :custom_email_smtp_address, length: { maximum: 255 } + validates :custom_email_smtp_username, length: { maximum: 255 } + + validates :custom_email, + presence: true, + devise_email: true, + if: :custom_email_enabled? + validates :custom_email_smtp_address, + presence: true, + hostname: { allow_numeric_hostname: true, require_valid_tld: true }, + if: :custom_email_enabled? + validates :custom_email_smtp_username, + presence: true, + if: :custom_email_enabled? + validates :custom_email_smtp_port, + presence: true, + numericality: { only_integer: true, greater_than: 0 }, + if: :custom_email_enabled? + scope :with_project_key, ->(key) { where(project_key: key) } + def custom_email_delivery_options + { + user_name: custom_email_smtp_username, + password: custom_email_smtp_password, + address: custom_email_smtp_address, + domain: Mail::Address.new(custom_email).domain, + port: custom_email_smtp_port || 587 + } + end + def issue_template_content strong_memoize(:issue_template_content) do next unless issue_template_key.present? diff --git a/app/models/snippet_repository.rb b/app/models/snippet_repository.rb index a959ad4d548..9139dc22a94 100644 --- a/app/models/snippet_repository.rb +++ b/app/models/snippet_repository.rb @@ -121,7 +121,7 @@ class SnippetRepository < ApplicationRecord def invalid_signature_error?(err) err.is_a?(ArgumentError) && - err.message.downcase.match?(/failed to parse signature/) + err.message.downcase.include?('failed to parse signature') end def only_rename_action?(action) diff --git a/app/models/snippet_user_mention.rb b/app/models/snippet_user_mention.rb index 87ce77a5787..138feb6ab29 100644 --- a/app/models/snippet_user_mention.rb +++ b/app/models/snippet_user_mention.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class SnippetUserMention < UserMention + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' + belongs_to :snippet belongs_to :note end diff --git a/app/models/suggestion.rb b/app/models/suggestion.rb index ca2ad8bf88c..267be5fe5c2 100644 --- a/app/models/suggestion.rb +++ b/app/models/suggestion.rb @@ -3,6 +3,9 @@ class Suggestion < ApplicationRecord include Importable include Suggestible + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' belongs_to :note, inverse_of: :suggestions validates :note, presence: true, unless: :importing? diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 36166bdbc9a..bb8527d8c01 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -2,6 +2,9 @@ class SystemNoteMetadata < ApplicationRecord include Importable + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' # These notes's action text might contain a reference that is external. # We should always force a deep validation upon references that are found diff --git a/app/models/timelog.rb b/app/models/timelog.rb index 07c61f64f29..dc976816ad9 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -2,6 +2,9 @@ class Timelog < ApplicationRecord include Importable + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' before_save :set_project diff --git a/app/models/todo.rb b/app/models/todo.rb index 47dabc1533d..62252912c32 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -4,6 +4,9 @@ class Todo < ApplicationRecord include Sortable include FromUnion include EachBatch + include IgnorableColumns + + ignore_column :note_id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' # Time to wait for todos being removed when not visible for user anymore. # Prevents TODOs being removed by mistake, for example, removing access from a user @@ -72,7 +75,9 @@ class Todo < ApplicationRecord scope :for_type, -> (type) { where(target_type: type) } scope :for_target, -> (id) { where(target_id: id) } scope :for_commit, -> (id) { where(commit_id: id) } - scope :with_entity_associations, -> { preload(:target, :author, :note, group: :route, project: [:route, { namespace: [:route, :owner] }]) } + scope :with_entity_associations, -> do + preload(:target, :author, :note, group: :route, project: [:route, { namespace: [:route, :owner] }, :project_setting]) + end scope :joins_issue_and_assignees, -> { left_joins(issue: :assignees) } scope :for_internal_notes, -> { joins(:note).where(note: { confidential: true }) } @@ -169,6 +174,7 @@ class Todo < ApplicationRecord done = grouped_count.where(state: :done).select("'done' AS state") pending = grouped_count.where(state: :pending).select("'pending' AS state") union = unscoped.from_union([done, pending], remove_duplicates: false) + .select(:user_id, :count, :state) connection.select_all(union).each_with_object({}) do |row, counts| counts[[row['user_id'], row['state']]] = row['count'] @@ -249,7 +255,7 @@ class Todo < ApplicationRecord end def for_issue_or_work_item? - [Issue.name, WorkItem.name].any? { |klass_name| target_type == klass_name } + [Issue.name, WorkItem.name].any?(target_type) end # override to return commits, which are not active record diff --git a/app/models/user.rb b/app/models/user.rb index da6e1abad07..f3e8f14adf5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -101,7 +101,7 @@ class User < ApplicationRecord MINIMUM_DAYS_CREATED = 7 - ignore_columns %i[linkedin twitter skype website_url location organization], remove_with: '15.8', remove_after: '2023-01-22' + ignore_columns %i[linkedin twitter skype website_url location organization], remove_with: '15.10', remove_after: '2023-02-22' # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour @@ -337,7 +337,7 @@ class User < ApplicationRecord enum layout: { fixed: 0, fluid: 1 } # User's Dashboard preference - enum dashboard: { projects: 0, stars: 1, project_activity: 2, starred_project_activity: 3, groups: 4, todos: 5, issues: 6, merge_requests: 7, operations: 8, followed_user_activity: 9 } + enum dashboard: { projects: 0, stars: 1, your_activity: 10, project_activity: 2, starred_project_activity: 3, groups: 4, todos: 5, issues: 6, merge_requests: 7, operations: 8, followed_user_activity: 9 } # User's Project preference enum project_view: { readme: 0, activity: 1, files: 2 } @@ -380,6 +380,7 @@ class User < ApplicationRecord delegate :website_url, :website_url=, to: :user_detail, allow_nil: true delegate :location, :location=, to: :user_detail, allow_nil: true delegate :organization, :organization=, to: :user_detail, allow_nil: true + delegate :discord, :discord=, to: :user_detail, allow_nil: true accepts_nested_attributes_for :user_preference, update_only: true accepts_nested_attributes_for :user_detail, update_only: true @@ -406,6 +407,15 @@ class User < ApplicationRecord transition deactivated: :ldap_blocked end + # aliasing system_block to set ldap_blocked statuses + # ldap_blocked is used for LDAP, SAML, and SCIM blocked users + # Issue for improving this naming: + # https://gitlab.com/gitlab-org/gitlab/-/issues/388487 + event :system_block do + transition active: :ldap_blocked + transition deactivated: :ldap_blocked + end + event :activate do transition deactivated: :active transition blocked: :active @@ -1025,19 +1035,32 @@ class User < ApplicationRecord def disable_two_factor! transaction do - update( - otp_required_for_login: false, - encrypted_otp_secret: nil, - encrypted_otp_secret_iv: nil, - encrypted_otp_secret_salt: nil, - otp_grace_period_started_at: nil, - otp_backup_codes: nil - ) - self.u2f_registrations.destroy_all # rubocop: disable Cop/DestroyAll - self.webauthn_registrations.destroy_all # rubocop: disable Cop/DestroyAll + self.u2f_registrations.destroy_all # rubocop:disable Cop/DestroyAll + self.disable_webauthn! + self.disable_two_factor_otp! + self.reset_backup_codes! end end + def disable_two_factor_otp! + update( + otp_required_for_login: false, + encrypted_otp_secret: nil, + encrypted_otp_secret_iv: nil, + encrypted_otp_secret_salt: nil, + otp_grace_period_started_at: nil, + otp_secret_expires_at: nil + ) + end + + def disable_webauthn! + self.webauthn_registrations.destroy_all # rubocop:disable Cop/DestroyAll + end + + def reset_backup_codes! + update(otp_backup_codes: nil) + end + def two_factor_enabled? two_factor_otp_enabled? || two_factor_webauthn_u2f_enabled? end @@ -1719,12 +1742,6 @@ class User < ApplicationRecord end end - def manageable_groups_with_routes(include_groups_with_developer_maintainer_access: false) - manageable_groups(include_groups_with_developer_maintainer_access: include_groups_with_developer_maintainer_access) - .eager_load(:route) - .order('routes.path') - end - def namespaces(owned_only: false) user_groups = owned_only ? owned_groups : groups personal_namespace = Namespace.where(id: namespace.id) diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index b6765cb0285..9d3df3d6400 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -14,11 +14,13 @@ class UserDetail < ApplicationRecord DEFAULT_FIELD_LENGTH = 500 + validates :discord, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validate :discord_format validates :linkedin, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true - validates :twitter, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true - validates :skype, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true validates :location, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true validates :organization, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :skype, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true + validates :twitter, length: { maximum: DEFAULT_FIELD_LENGTH }, allow_blank: true validates :website_url, length: { maximum: DEFAULT_FIELD_LENGTH }, url: true, allow_blank: true, if: :website_url_changed? before_validation :sanitize_attrs @@ -27,7 +29,7 @@ class UserDetail < ApplicationRecord enum registration_objective: REGISTRATION_OBJECTIVE_PAIRS, _suffix: true def sanitize_attrs - %i[linkedin skype twitter website_url].each do |attr| + %i[discord linkedin skype twitter website_url].each do |attr| value = self[attr] self[attr] = Sanitize.clean(value) if value.present? end @@ -41,13 +43,20 @@ class UserDetail < ApplicationRecord def prevent_nil_fields self.bio = '' if bio.nil? + self.discord = '' if discord.nil? self.linkedin = '' if linkedin.nil? - self.twitter = '' if twitter.nil? - self.skype = '' if skype.nil? self.location = '' if location.nil? self.organization = '' if organization.nil? + self.skype = '' if skype.nil? + self.twitter = '' if twitter.nil? self.website_url = '' if website_url.nil? end end +def discord_format + return if discord.blank? || discord =~ %r{\A\d{17,20}\z} + + errors.add(:discord, _('must contain only a discord user ID.')) +end + UserDetail.prepend_mod_with('UserDetail') diff --git a/app/models/user_synced_attributes_metadata.rb b/app/models/user_synced_attributes_metadata.rb index 5aacf11b1cb..4cceffda19e 100644 --- a/app/models/user_synced_attributes_metadata.rb +++ b/app/models/user_synced_attributes_metadata.rb @@ -14,7 +14,7 @@ class UserSyncedAttributesMetadata < ApplicationRecord def read_only_attributes return [] unless sync_profile_from_provider? - SYNCABLE_ATTRIBUTES.select { |key| synced?(key) } + self.class.syncable_attributes.select { |key| synced?(key) } end def synced?(attribute) @@ -25,6 +25,20 @@ class UserSyncedAttributesMetadata < ApplicationRecord write_attribute("#{attribute}_synced", value) end + class << self + def syncable_attributes + return SYNCABLE_ATTRIBUTES if sync_name? + + SYNCABLE_ATTRIBUTES - %i[name] + end + + private + + def sync_name? + Gitlab.config.ldap.sync_name + end + end + private def sync_profile_from_provider? diff --git a/app/models/users/saved_reply.rb b/app/models/users/saved_reply.rb index 7737d826b05..f0ae5445a46 100644 --- a/app/models/users/saved_reply.rb +++ b/app/models/users/saved_reply.rb @@ -9,11 +9,11 @@ module Users validates :user_id, :name, :content, presence: true validates :name, length: { maximum: 255 }, - uniqueness: { scope: [:user_id] }, - format: { - with: Gitlab::Regex.saved_reply_name_regex, - message: Gitlab::Regex.saved_reply_name_regex_message - } + uniqueness: { scope: [:user_id] } validates :content, length: { maximum: 10000 } + + def self.find_saved_reply(user_id:, id:) + ::Users::SavedReply.find_by(user_id: user_id, id: id) + end end end diff --git a/app/models/wiki_directory.rb b/app/models/wiki_directory.rb index 3a2613e15d9..76fe664f23d 100644 --- a/app/models/wiki_directory.rb +++ b/app/models/wiki_directory.rb @@ -6,7 +6,7 @@ class WikiDirectory attr_accessor :slug, :entries validates :slug, presence: true - + alias_method :to_param, :slug # Groups a list of wiki pages into a nested collection of WikiPage and WikiDirectory objects, # preserving the order of the passed pages. # @@ -25,6 +25,7 @@ class WikiDirectory parent = File.dirname(path) parent = '' if parent == '.' directories[parent].entries << directory + directories[parent].entries.delete_if { |item| item.is_a?(WikiPage) && item.slug == directory.slug } end end end @@ -48,6 +49,6 @@ class WikiDirectory # Relative path to the partial to be used when rendering collections # of this object. def to_partial_path - '../shared/wikis/wiki_directory' + 'shared/wikis/wiki_directory' end end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 24b0b94eeb7..b04aa196883 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -259,7 +259,7 @@ class WikiPage # Relative path to the partial to be used when rendering collections # of this object. def to_partial_path - '../shared/wikis/wiki_page' + 'shared/wikis/wiki_page' end def sha diff --git a/app/models/work_item.rb b/app/models/work_item.rb index f94e831437a..5ae3fb6cf78 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -3,6 +3,10 @@ class WorkItem < Issue include Gitlab::Utils::StrongMemoize + COMMON_QUICK_ACTIONS_COMMANDS = [ + :title, :reopen, :close, :cc, :tableflip, :shrug + ].freeze + self.table_name = 'issues' self.inheritance_column = :_type_disabled @@ -13,11 +17,14 @@ class WorkItem < Issue has_many :child_links, class_name: '::WorkItems::ParentLink', foreign_key: :work_item_parent_id has_many :work_item_children, through: :child_links, class_name: 'WorkItem', foreign_key: :work_item_id, source: :work_item - has_many :work_item_children_by_created_at, -> { order(:created_at) }, through: :child_links, class_name: 'WorkItem', - foreign_key: :work_item_id, source: :work_item + has_many :work_item_children_by_relative_position, -> { work_item_children_keyset_order }, + through: :child_links, class_name: 'WorkItem', + foreign_key: :work_item_id, source: :work_item scope :inc_relations_for_permission_check, -> { includes(:author, project: :project_feature) } + delegate :supports_assignee?, to: :work_item_type + class << self def assignee_association_name 'issue' @@ -26,6 +33,26 @@ class WorkItem < Issue def test_reports_join_column 'issues.id' end + + def work_item_children_keyset_order + keyset_order = Gitlab::Pagination::Keyset::Order.build([ + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :relative_position, + column_expression: WorkItems::ParentLink.arel_table[:relative_position], + order_expression: WorkItems::ParentLink.arel_table[:relative_position].asc.nulls_last, + nullable: :nulls_last, + distinct: false + ), + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: :created_at, + order_expression: WorkItem.arel_table[:created_at].asc, + nullable: :not_nullable, + distinct: false + ) + ]) + + includes(:child_links).order(keyset_order) + end end def noteable_target_type_name @@ -52,6 +79,12 @@ class WorkItem < Issue hierarchy(same_type: true).max_descendants_depth.to_i end + def supported_quick_action_commands + commands_for_widgets = work_item_type.widgets.flat_map(&:quick_action_commands).uniq + + COMMON_QUICK_ACTIONS_COMMANDS + commands_for_widgets + end + private override :parent_link_confidentiality diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index e1f6a13f7a7..6a619dbab21 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -35,56 +35,6 @@ module WorkItems key_result: { name: TYPE_NAMES[:key_result], icon_name: 'issue-type-keyresult', enum_value: 6 } ## EE-only }.freeze - WIDGETS_FOR_TYPE = { - issue: [ - Widgets::Assignees, - Widgets::Labels, - Widgets::Description, - Widgets::Hierarchy, - Widgets::StartAndDueDate, - Widgets::Milestone, - Widgets::Notes - ], - incident: [ - Widgets::Description, - Widgets::Hierarchy, - Widgets::Notes - ], - test_case: [ - Widgets::Description, - Widgets::Notes - ], - requirement: [ - Widgets::Description, - Widgets::Notes - ], - task: [ - Widgets::Assignees, - Widgets::Labels, - Widgets::Description, - Widgets::Hierarchy, - Widgets::StartAndDueDate, - Widgets::Milestone, - Widgets::Notes - ], - objective: [ - Widgets::Assignees, - Widgets::Labels, - Widgets::Description, - Widgets::Hierarchy, - Widgets::Milestone, - Widgets::Notes - ], - key_result: [ - Widgets::Assignees, - Widgets::Labels, - Widgets::Description, - Widgets::Hierarchy, - Widgets::StartAndDueDate, - Widgets::Notes - ] - }.freeze - # A list of types user can change between - both original and new # type must be included in this list. This is needed for legacy issues # where it's possible to switch between issue and incident. @@ -98,6 +48,9 @@ module WorkItems belongs_to :namespace, optional: true has_many :work_items, class_name: 'Issue', foreign_key: :work_item_type_id, inverse_of: :work_item_type + has_many :widget_definitions, foreign_key: :work_item_type_id, inverse_of: :work_item_type + has_many :enabled_widget_definitions, -> { where(disabled: false) }, foreign_key: :work_item_type_id, + inverse_of: :work_item_type, class_name: 'WorkItems::WidgetDefinition' before_validation :strip_whitespace @@ -112,10 +65,6 @@ module WorkItems scope :order_by_name_asc, -> { order(arel_table[:name].lower.asc) } scope :by_type, ->(base_type) { where(base_type: base_type) } - def self.available_widgets - WIDGETS_FOR_TYPE.values.flatten.uniq - end - def self.default_by_type(type) found_type = find_by(namespace_id: nil, base_type: type) return found_type if found_type @@ -138,7 +87,15 @@ module WorkItems end def widgets - WIDGETS_FOR_TYPE[base_type.to_sym] + enabled_widget_definitions.filter_map(&:widget_class) + end + + def supports_assignee? + widgets.include? ::WorkItems::Widgets::Assignees + end + + def default_issue? + name == WorkItems::Type::TYPE_NAMES[:issue] end private @@ -148,5 +105,3 @@ module WorkItems end end end - -WorkItems::Type.prepend_mod diff --git a/app/models/work_items/widget_definition.rb b/app/models/work_items/widget_definition.rb new file mode 100644 index 00000000000..5d4414e95d8 --- /dev/null +++ b/app/models/work_items/widget_definition.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module WorkItems + class WidgetDefinition < ApplicationRecord + self.table_name = 'work_item_widget_definitions' + + belongs_to :namespace, optional: true + belongs_to :work_item_type, class_name: 'WorkItems::Type', inverse_of: :widget_definitions + + validates :name, presence: true + validates :name, uniqueness: { case_sensitive: false, scope: [:namespace_id, :work_item_type_id] } + validates :name, length: { maximum: 255 } + + scope :enabled, -> { where(disabled: false) } + scope :global, -> { where(namespace: nil) } + + enum widget_type: { + assignees: 0, + description: 1, + hierarchy: 2, + labels: 3, + milestone: 4, + notes: 5, + start_and_due_date: 6, + health_status: 7, # EE-only + weight: 8, # EE-only + iteration: 9, # EE-only + progress: 10, # EE-only + status: 11, # EE-only + requirement_legacy: 12, # EE-only + test_reports: 13 # EE-only + } + + def self.available_widgets + global.enabled.filter_map(&:widget_class).uniq + end + + def self.widget_classes + WorkItems::WidgetDefinition.widget_types.keys.filter_map do |type| + WorkItems::Widgets.const_get(type.camelize, false) + rescue NameError + nil + end + end + + def widget_class + return unless widget_type + + WorkItems::Widgets.const_get(widget_type.camelize, false) + rescue NameError + nil + end + end +end diff --git a/app/models/work_items/widgets/assignees.rb b/app/models/work_items/widgets/assignees.rb index ecbbee1bcfb..0707b03e647 100644 --- a/app/models/work_items/widgets/assignees.rb +++ b/app/models/work_items/widgets/assignees.rb @@ -5,6 +5,14 @@ module WorkItems class Assignees < Base delegate :assignees, to: :work_item delegate :allows_multiple_assignees?, to: :work_item + + def self.quick_action_commands + [:assign, :unassign, :reassign] + end + + def self.quick_action_params + [:assignee_ids] + end end end end diff --git a/app/models/work_items/widgets/base.rb b/app/models/work_items/widgets/base.rb index e7075a7a0e8..3a5b03bd514 100644 --- a/app/models/work_items/widgets/base.rb +++ b/app/models/work_items/widgets/base.rb @@ -11,6 +11,10 @@ module WorkItems "#{type}_widget".to_sym end + def self.quick_action_commands + [] + end + def type self.class.type end diff --git a/app/models/work_items/widgets/hierarchy.rb b/app/models/work_items/widgets/hierarchy.rb index ee10c631bcc..8f54cb32f43 100644 --- a/app/models/work_items/widgets/hierarchy.rb +++ b/app/models/work_items/widgets/hierarchy.rb @@ -8,7 +8,7 @@ module WorkItems end def children - work_item.work_item_children_by_created_at + work_item.work_item_children_by_relative_position end end end diff --git a/app/models/work_items/widgets/labels.rb b/app/models/work_items/widgets/labels.rb index 4ad8319ffac..e8b36156fec 100644 --- a/app/models/work_items/widgets/labels.rb +++ b/app/models/work_items/widgets/labels.rb @@ -5,6 +5,14 @@ module WorkItems class Labels < Base delegate :labels, to: :work_item delegate :allows_scoped_labels?, to: :work_item + + def self.quick_action_commands + [:label, :labels, :relabel, :remove_label, :unlabel] + end + + def self.quick_action_params + [:add_label_ids, :remove_label_ids, :label_ids] + end end end end diff --git a/app/models/work_items/widgets/start_and_due_date.rb b/app/models/work_items/widgets/start_and_due_date.rb index 0b828c5b5a9..22ef262534e 100644 --- a/app/models/work_items/widgets/start_and_due_date.rb +++ b/app/models/work_items/widgets/start_and_due_date.rb @@ -4,6 +4,14 @@ module WorkItems module Widgets class StartAndDueDate < Base delegate :start_date, :due_date, to: :work_item + + def self.quick_action_commands + [:due, :remove_due_date] + end + + def self.quick_action_params + [:due_date] + end end end end |