diff options
Diffstat (limited to 'app/models')
98 files changed, 1494 insertions, 386 deletions
diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index a53fa39c58f..1ec3cb62c76 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -27,6 +27,7 @@ module AlertManagement has_many :notes, as: :noteable, inverse_of: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :ordered_notes, -> { fresh }, as: :noteable, class_name: 'Note' has_many :user_mentions, class_name: 'AlertManagement::AlertUserMention', foreign_key: :alert_management_alert_id + has_many :metric_images, class_name: '::AlertManagement::MetricImage' has_internal_id :iid, scope: :project @@ -142,6 +143,10 @@ module AlertManagement reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end + def metric_images_available? + ::AlertManagement::MetricImage.available_for?(project) + end + def prometheus? monitoring_tool == Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] end diff --git a/app/models/alert_management/metric_image.rb b/app/models/alert_management/metric_image.rb new file mode 100644 index 00000000000..8175a31be7a --- /dev/null +++ b/app/models/alert_management/metric_image.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module AlertManagement + class MetricImage < ApplicationRecord + include MetricImageUploading + self.table_name = 'alert_management_alert_metric_images' + + belongs_to :alert, class_name: 'AlertManagement::Alert', foreign_key: 'alert_id', inverse_of: :metric_images + + def self.available_for?(project) + true + end + + private + + def local_path + Gitlab::Routing.url_helpers.alert_metric_image_upload_path( + filename: file.filename, + id: file.upload.model_id, + model: model_name.param_key, + mounted_as: 'file' + ) + end + end +end diff --git a/app/models/analytics/cycle_analytics/aggregation.rb b/app/models/analytics/cycle_analytics/aggregation.rb index 44d2dc369f7..2c04e67a04b 100644 --- a/app/models/analytics/cycle_analytics/aggregation.rb +++ b/app/models/analytics/cycle_analytics/aggregation.rb @@ -1,15 +1,53 @@ # frozen_string_literal: true class Analytics::CycleAnalytics::Aggregation < ApplicationRecord + include IgnorableColumns include FromUnion belongs_to :group, optional: false - validates :incremental_runtimes_in_seconds, :incremental_processed_records, :last_full_run_runtimes_in_seconds, :last_full_run_processed_records, presence: true, length: { maximum: 10 }, allow_blank: true + validates :incremental_runtimes_in_seconds, :incremental_processed_records, :full_runtimes_in_seconds, :full_processed_records, presence: true, length: { maximum: 10 }, allow_blank: true scope :priority_order, -> (column_to_sort = :last_incremental_run_at) { order(arel_table[column_to_sort].asc.nulls_first) } scope :enabled, -> { where('enabled IS TRUE') } + # These columns were added with wrong naming convention, the columns were never used. + ignore_column :last_full_run_processed_records, remove_with: '15.1', remove_after: '2022-05-22' + ignore_column :last_full_run_runtimes_in_seconds, remove_with: '15.1', remove_after: '2022-05-22' + ignore_column :last_full_run_issues_updated_at, remove_with: '15.1', remove_after: '2022-05-22' + ignore_column :last_full_run_mrs_updated_at, remove_with: '15.1', remove_after: '2022-05-22' + ignore_column :last_full_run_issues_id, remove_with: '15.1', remove_after: '2022-05-22' + ignore_column :last_full_run_merge_requests_id, remove_with: '15.1', remove_after: '2022-05-22' + + def cursor_for(mode, model) + { + updated_at: self["last_#{mode}_#{model.table_name}_updated_at"], + id: self["last_#{mode}_#{model.table_name}_id"] + }.compact + end + + def refresh_last_run(mode) + self["last_#{mode}_run_at"] = Time.current + end + + def reset_full_run_cursors + self.last_full_issues_id = nil + self.last_full_issues_updated_at = nil + self.last_full_merge_requests_id = nil + self.last_full_merge_requests_updated_at = nil + end + + def set_cursor(mode, model, cursor) + self["last_#{mode}_#{model.table_name}_id"] = cursor[:id] + self["last_#{mode}_#{model.table_name}_updated_at"] = cursor[:updated_at] + end + + def set_stats(mode, runtime, processed_records) + # We only store the last 10 data points + self["#{mode}_runtimes_in_seconds"] = (self["#{mode}_runtimes_in_seconds"] + [runtime]).last(10) + self["#{mode}_processed_records"] = (self["#{mode}_processed_records"] + [processed_records]).last(10) + end + def estimated_next_run_at return unless enabled return if last_incremental_run_at.nil? diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c7aad7ff861..7cd2fe705e3 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -387,7 +387,7 @@ class ApplicationSetting < ApplicationRecord validates :invisible_captcha_enabled, inclusion: { in: [true, false], message: _('must be a boolean value') } - SUPPORTED_KEY_TYPES.each do |type| + Gitlab::SSHPublicKey.supported_types.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -576,6 +576,17 @@ class ApplicationSetting < ApplicationRecord length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, allow_nil: false + validates :public_runner_releases_url, addressable_url: true, presence: true + + validates :inactive_projects_min_size_mb, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :inactive_projects_delete_after_months, + numericality: { only_integer: true, greater_than: 0 } + + validates :inactive_projects_send_warning_email_after_months, + numericality: { only_integer: true, greater_than: 0, less_than: :inactive_projects_delete_after_months } + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, @@ -609,6 +620,9 @@ class ApplicationSetting < ApplicationRecord attr_encrypted :cloud_license_auth_token, encryption_options_base_32_aes_256_gcm attr_encrypted :external_pipeline_validation_service_token, encryption_options_base_32_aes_256_gcm attr_encrypted :mailgun_signing_key, encryption_options_base_32_aes_256_gcm.merge(encode: false) + attr_encrypted :database_grafana_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :arkose_labs_public_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :arkose_labs_private_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) validates :disable_feed_token, inclusion: { in: [true, false], message: _('must be a boolean value') } diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 42049713883..194356acc51 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -14,7 +14,6 @@ module ApplicationSettingImplementation # Setting a key restriction to `-1` means that all keys of this type are # forbidden. FORBIDDEN_KEY_VALUE = KeyRestrictionValidator::FORBIDDEN - SUPPORTED_KEY_TYPES = Gitlab::SSHPublicKey.supported_types VALID_RUNNER_REGISTRAR_TYPES = %w(project group).freeze DEFAULT_PROTECTED_PATHS = [ @@ -67,11 +66,11 @@ module ApplicationSettingImplementation disabled_oauth_sign_in_sources: [], dns_rebinding_protection_enabled: true, domain_allowlist: Settings.gitlab['domain_allowlist'], - dsa_key_restriction: 0, - ecdsa_key_restriction: 0, - ecdsa_sk_key_restriction: 0, - ed25519_key_restriction: 0, - ed25519_sk_key_restriction: 0, + dsa_key_restriction: default_min_key_size(:dsa), + ecdsa_key_restriction: default_min_key_size(:ecdsa), + ecdsa_sk_key_restriction: default_min_key_size(:ecdsa_sk), + ed25519_key_restriction: default_min_key_size(:ed25519), + ed25519_sk_key_restriction: default_min_key_size(:ed25519_sk), eks_access_key_id: nil, eks_account_id: nil, eks_integration_enabled: false, @@ -96,7 +95,6 @@ module ApplicationSettingImplementation help_page_text: nil, help_page_documentation_base_url: nil, hide_third_party_offers: false, - housekeeping_bitmaps_enabled: true, housekeeping_enabled: true, housekeeping_full_repack_period: 50, housekeeping_gc_period: 200, @@ -143,7 +141,7 @@ module ApplicationSettingImplementation require_admin_approval_after_user_signup: true, require_two_factor_authentication: false, restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], - rsa_key_restriction: 0, + rsa_key_restriction: default_min_key_size(:rsa), send_user_confirmation_email: false, session_expire_delay: Settings.gitlab['session_expire_delay'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], @@ -244,6 +242,20 @@ module ApplicationSettingImplementation "users.noreply.#{Gitlab.config.gitlab.host}" end + # Return the default allowed minimum key size for a type. + # By default this is 0 (unrestricted), but in FIPS mode + # this will return the smallest allowed key size. If no + # size is available, this type is denied. + # + # @return [Integer] + def default_min_key_size(name) + if Gitlab::FIPS.enabled? + Gitlab::SSHPublicKey.supported_sizes(name).select(&:positive?).min || -1 + else + 0 + end + end + def create_from_defaults build_from_defaults.tap(&:save) end @@ -442,7 +454,7 @@ module ApplicationSettingImplementation alias_method :usage_ping_enabled?, :usage_ping_enabled def allowed_key_types - SUPPORTED_KEY_TYPES.select do |type| + Gitlab::SSHPublicKey.supported_types.select do |type| key_restriction_for(type) != FORBIDDEN_KEY_VALUE end end diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index b665f3d5d8c..22e5436dc5c 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -19,6 +19,8 @@ class AwardEmoji < ApplicationRecord participant :user + delegate :resource_parent, to: :awardable, allow_nil: true + scope :downvotes, -> { named(DOWNVOTE_NAME) } scope :upvotes, -> { named(UPVOTE_NAME) } scope :named, -> (names) { where(name: names) } @@ -60,6 +62,12 @@ class AwardEmoji < ApplicationRecord self.name == UPVOTE_NAME end + def url + return if TanukiEmoji.find_by_alpha_code(name) + + CustomEmoji.for_resource(resource_parent).by_name(name).select(:url).first&.url + end + def expire_cache awardable.try(:bump_updated_at) awardable.try(:expire_etag_cache) diff --git a/app/models/blob.rb b/app/models/blob.rb index cc7758d9674..a12d856dc36 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -8,6 +8,7 @@ class Blob < SimpleDelegator include BlobActiveModel MODE_SYMLINK = '120000' # The STRING 120000 is the git-reported octal filemode for a symlink + MODE_EXECUTABLE = '100755' # The STRING 100755 is the git-reported octal filemode for an executable file CACHE_TIME = 60 # Cache raw blobs referred to by a (mutable) ref for 1 minute CACHE_TIME_IMMUTABLE = 3600 # Cache blobs referred to by an immutable reference for 1 hour @@ -35,7 +36,6 @@ class Blob < SimpleDelegator BlobViewer::Image, BlobViewer::Sketch, - BlobViewer::Balsamiq, BlobViewer::Video, BlobViewer::Audio, @@ -182,6 +182,10 @@ class Blob < SimpleDelegator mode == MODE_SYMLINK end + def executable? + mode == MODE_EXECUTABLE + end + def extension @extension ||= extname.downcase.delete('.') end diff --git a/app/models/blob_viewer/balsamiq.rb b/app/models/blob_viewer/balsamiq.rb deleted file mode 100644 index 6ab73730222..00000000000 --- a/app/models/blob_viewer/balsamiq.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module BlobViewer - class Balsamiq < Base - include Rich - include ClientSide - - self.partial_name = 'balsamiq' - self.extensions = %w(bmpr) - self.binary = true - self.switcher_icon = 'doc-image' - self.switcher_title = 'preview' - end -end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 949902fbb77..b255c774347 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -32,6 +32,19 @@ class BroadcastMessage < ApplicationRecord after_commit :flush_redis_cache + enum theme: { + indigo: 0, + 'light-indigo': 1, + blue: 2, + 'light-blue': 3, + green: 4, + 'light-green': 5, + red: 6, + 'light-red': 7, + dark: 8, + light: 9 + }, _default: 0, _prefix: true + enum broadcast_type: { banner: 1, notification: 2 diff --git a/app/models/bulk_import.rb b/app/models/bulk_import.rb index 818ae04ba29..2200a66b3c2 100644 --- a/app/models/bulk_import.rb +++ b/app/models/bulk_import.rb @@ -16,10 +16,14 @@ class BulkImport < ApplicationRecord enum source_type: { gitlab: 0 } + scope :stale, -> { where('created_at < ?', 8.hours.ago).where(status: [0, 1]) } + scope :order_by_created_at, -> (direction) { order(created_at: direction) } + state_machine :status, initial: :created do state :created, value: 0 state :started, value: 1 state :finished, value: 2 + state :timeout, value: 3 state :failed, value: -1 event :start do @@ -30,6 +34,11 @@ class BulkImport < ApplicationRecord transition started: :finished end + event :cleanup_stale do + transition created: :timeout + transition started: :timeout + end + event :fail_op do transition any => :failed end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index a7e1384641c..dee533944e9 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -51,11 +51,15 @@ class BulkImports::Entity < ApplicationRecord enum source_type: { group_entity: 0, project_entity: 1 } scope :by_user_id, ->(user_id) { joins(:bulk_import).where(bulk_imports: { user_id: user_id }) } + scope :stale, -> { where('created_at < ?', 8.hours.ago).where(status: [0, 1]) } + scope :by_bulk_import_id, ->(bulk_import_id) { where(bulk_import_id: bulk_import_id)} + scope :order_by_created_at, -> (direction) { order(created_at: direction) } state_machine :status, initial: :created do state :created, value: 0 state :started, value: 1 state :finished, value: 2 + state :timeout, value: 3 state :failed, value: -1 event :start do @@ -70,6 +74,11 @@ class BulkImports::Entity < ApplicationRecord event :fail_op do transition any => :failed end + + event :cleanup_stale do + transition created: :timeout + transition started: :timeout + end end def self.all_human_statuses @@ -83,9 +92,9 @@ class BulkImports::Entity < ApplicationRecord def pipelines @pipelines ||= case source_type when 'group_entity' - BulkImports::Groups::Stage.new(bulk_import).pipelines + BulkImports::Groups::Stage.new(self).pipelines when 'project_entity' - BulkImports::Projects::Stage.new(bulk_import).pipelines + BulkImports::Projects::Stage.new(self).pipelines end end diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb index cae6aad27da..a9750a76987 100644 --- a/app/models/bulk_imports/export_status.rb +++ b/app/models/bulk_imports/export_status.rb @@ -32,10 +32,12 @@ module BulkImports strong_memoize(:export_status) do status = fetch_export_status + relation_export_status = status&.find { |item| item['relation'] == relation } + # Consider empty response as failed export - raise StandardError, 'Empty export status response' unless status&.present? + raise StandardError, 'Empty relation export status' unless relation_export_status&.present? - status.find { |item| item['relation'] == relation } + relation_export_status end rescue StandardError => e { 'status' => Export::FAILED, 'error' => e.message } diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index cfe33c013ba..a994cc3f8ce 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -46,6 +46,7 @@ class BulkImports::Tracker < ApplicationRecord state :started, value: 1 state :finished, value: 2 state :enqueued, value: 3 + state :timeout, value: 4 state :failed, value: -1 state :skipped, value: -2 @@ -76,5 +77,9 @@ class BulkImports::Tracker < ApplicationRecord event :fail_op do transition any => :failed end + + event :cleanup_stale do + transition [:created, :started] => :timeout + end end end diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 2ff777bfc89..ff444ddefa3 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -57,10 +57,6 @@ module Ci end end - def self.retry(bridge, current_user) - raise NotImplementedError - end - def self.with_preloads preload( :metadata, @@ -69,6 +65,10 @@ module Ci ) end + def retryable? + false + end + def inherit_status_from_downstream!(pipeline) case pipeline.status when 'success' @@ -274,7 +274,8 @@ module Ci # The order of this list refers to the priority of the variables downstream_yaml_variables(expand_variables) + - downstream_pipeline_variables(expand_variables) + downstream_pipeline_variables(expand_variables) + + downstream_pipeline_schedule_variables(expand_variables) end def downstream_yaml_variables(expand_variables) @@ -293,6 +294,15 @@ module Ci end end + def downstream_pipeline_schedule_variables(expand_variables) + return [] unless forward_pipeline_variables? + return [] unless pipeline.pipeline_schedule + + pipeline.pipeline_schedule.variables.to_a.map do |variable| + { key: variable.key, value: ::ExpandVariables.expand(variable.value, expand_variables) } + end + end + def forward_yaml_variables? strong_memoize(:forward_yaml_variables) do result = options&.dig(:trigger, :forward, :yaml_variables) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 68ec196a9ee..16c9aa212d0 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -218,17 +218,21 @@ module Ci pending.unstarted.order('created_at ASC').first end - def retry(build, current_user) - # rubocop: disable CodeReuse/ServiceClass - Ci::RetryBuildService - .new(build.project, current_user) - .execute(build) - # rubocop: enable CodeReuse/ServiceClass - end - def with_preloads preload(:job_artifacts_archive, :job_artifacts, :tags, project: [:namespace]) end + + def extra_accessors + [] + end + + def clone_accessors + %i[pipeline project ref tag options name + allow_failure stage stage_id stage_idx trigger_request + yaml_variables when environment coverage_regex + description tag_list protected needs_attributes + job_variables_attributes resource_group scheduling_type].freeze + end end state_machine :status do @@ -351,7 +355,9 @@ module Ci if build.auto_retry_allowed? begin - Ci::Build.retry(build, build.user) + # rubocop: disable CodeReuse/ServiceClass + Ci::RetryJobService.new(build.project, build.user).execute(build) + # rubocop: enable CodeReuse/ServiceClass rescue Gitlab::Access::AccessDeniedError => ex Gitlab::AppLogger.error "Unable to auto-retry job #{build.id}: #{ex}" end @@ -472,12 +478,6 @@ module Ci active? || created? end - def retryable? - return false if retried? || archived? || deployment_rejected? - - success? || failed? || canceled? - end - def retries_count pipeline.builds.retried.where(name: self.name).count end @@ -504,7 +504,11 @@ module Ci if metadata&.expanded_environment_name.present? metadata.expanded_environment_name else - ExpandVariables.expand(environment, -> { simple_variables }) + if ::Feature.enabled?(:ci_expand_environment_name_and_url, project, default_enabled: :yaml) + ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) + else + ExpandVariables.expand(environment, -> { simple_variables }) + end end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 3426c4d5248..dff8bb89021 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -186,6 +186,7 @@ module Ci scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) } + scope :order_expired_asc, -> { order(expire_at: :asc) } scope :order_expired_desc, -> { order(expire_at: :desc) } scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } @@ -273,6 +274,10 @@ module Ci self.where(project: project).sum(:size) end + def self.pluck_job_id + pluck(:job_id) + end + ## # FastDestroyAll concerns # rubocop: disable CodeReuse/ServiceClass diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index d5cbbb96134..e8f08db597f 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -4,6 +4,8 @@ module Ci # This model represents a record in a shadow table of the main database's namespaces table. # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. class NamespaceMirror < ApplicationRecord + include FromUnion + belongs_to :namespace scope :by_group_and_descendants, -> (id) do @@ -14,6 +16,24 @@ module Ci where('traversal_ids && ARRAY[?]::int[]', ids) end + scope :contains_traversal_ids, -> (traversal_ids) do + mirrors = [] + + traversal_ids.group_by(&:count).each do |columns_count, traversal_ids_group| + columns = Array.new(columns_count) { |i| "(traversal_ids[#{i + 1}])" } + pairs = traversal_ids_group.map do |ids| + ids = ids.map { |id| Arel::Nodes.build_quoted(id).to_sql } + "(#{ids.join(",")})" + end + + # Create condition in format: + # ((traversal_ids[1]),(traversal_ids[2])) IN ((1,2),(2,3)) + mirrors << Ci::NamespaceMirror.where("(#{columns.join(",")}) IN (#{pairs.join(",")})") # rubocop:disable GitlabSecurity/SqlInjection + end + + self.from_union(mirrors) + end + scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) } class << self diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ae3ea7aa03f..2d0479e02a3 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -824,6 +824,8 @@ module Ci variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: open_merge_requests_refs.join(',')) end + variables.append(key: 'CI_GITLAB_FIPS_MODE', value: 'true') if Gitlab::FIPS.enabled? + variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if has_kubernetes_active? variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if freeze_period? @@ -836,6 +838,8 @@ module Ci def predefined_commit_variables strong_memoize(:predefined_commit_variables) do Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless sha.present? + variables.append(key: 'CI_COMMIT_SHA', value: sha) variables.append(key: 'CI_COMMIT_SHORT_SHA', value: short_sha) variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: before_sha) @@ -955,7 +959,7 @@ module Ci Ci::Build.latest.where(pipeline: self_and_descendants) end - def environments_in_self_and_descendants + def environments_in_self_and_descendants(deployment_status: nil) # We limit to 100 unique environments for application safety. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 expanded_environment_names = @@ -965,7 +969,7 @@ module Ci .limit(100) .pluck(:expanded_environment_name) - Environment.where(project: project, name: expanded_environment_names).with_deployment(sha) + Environment.where(project: project, name: expanded_environment_names).with_deployment(sha, status: deployment_status) end # With multi-project and parent-child pipelines @@ -1285,6 +1289,12 @@ module Ci end end + def has_expired_test_reports? + strong_memoize(:artifacts_expired) do + !has_reports?(::Ci::JobArtifact.test_reports.not_expired) + end + end + private def add_message(severity, content) diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 4d119706a43..d79ff74753a 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -101,6 +101,12 @@ module Ci :merge_train_pipeline?, to: :pipeline + def retryable? + return false if retried? || archived? || deployment_rejected? + + success? || failed? || canceled? + end + def aggregated_needs_names read_attribute(:aggregated_needs_names) end diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 18f0093ea41..6a26a5341aa 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -15,7 +15,9 @@ module Ci validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } validates :checksum, :file_store, :name, :permissions, :project_id, presence: true + validates :name, uniqueness: { scope: :project } + after_initialize :generate_key_data before_validation :assign_checksum enum permissions: { read_only: 0, read_write: 1, execute: 2 } @@ -33,5 +35,11 @@ module Ci def assign_checksum self.checksum = file.checksum if file.present? && file_changed? end + + def generate_key_data + return if key_data.present? + + self.key_data = SecureRandom.hex(64) + end end end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index 691d628524f..1607d0b6d19 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -18,7 +18,7 @@ module Clusters validates :description, length: { maximum: 1024 } validates :name, presence: true, length: { maximum: 255 } - scope :order_last_used_at_desc, -> { order(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } + scope :order_last_used_at_desc, -> { order(arel_table[:last_used_at].desc.nulls_last) } scope :with_status, -> (status) { where(status: status) } enum status: { diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 07eaca87fad..e62b6fa5fc5 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.37.1' + VERSION = '0.39.0' self.table_name = 'clusters_applications_runners' diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 21e2e21e9b3..08fed353755 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -33,7 +33,7 @@ class CommitStatus < Ci::ApplicationRecord where(allow_failure: true, status: [:failed, :canceled]) end - scope :order_id_desc, -> { order('ci_builds.id DESC') } + scope :order_id_desc, -> { order(id: :desc) } scope :exclude_ignored, -> do # We want to ignore failed but allowed to fail jobs. diff --git a/app/models/concerns/batch_nullify_dependent_associations.rb b/app/models/concerns/batch_nullify_dependent_associations.rb new file mode 100644 index 00000000000..c95b5b64a43 --- /dev/null +++ b/app/models/concerns/batch_nullify_dependent_associations.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +# Provides a way to execute nullify behaviour in batches +# to avoid query timeouts for really big tables +# Assumes that associations have `dependent: :nullify` statement +module BatchNullifyDependentAssociations + extend ActiveSupport::Concern + + class_methods do + def dependent_associations_to_nullify + reflect_on_all_associations(:has_many).select { |assoc| assoc.options[:dependent] == :nullify } + end + end + + def nullify_dependent_associations_in_batches(exclude: [], batch_size: 100) + self.class.dependent_associations_to_nullify.each do |association| + next if association.name.in?(exclude) + + loop do + # rubocop:disable GitlabSecurity/PublicSend + update_count = public_send(association.name).limit(batch_size).update_all(association.foreign_key => nil) + # rubocop:enable GitlabSecurity/PublicSend + break if update_count < batch_size + end + end + end +end diff --git a/app/models/concerns/bulk_users_by_email_load.rb b/app/models/concerns/bulk_users_by_email_load.rb new file mode 100644 index 00000000000..edbd3e21458 --- /dev/null +++ b/app/models/concerns/bulk_users_by_email_load.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkUsersByEmailLoad + extend ActiveSupport::Concern + + included do + def users_by_emails(emails) + Gitlab::SafeRequestLoader.execute(resource_key: user_by_email_resource_key, resource_ids: emails) do |emails| + # have to consider all emails - even secondary, so use all_emails here + grouped_users_by_email = User.by_any_email(emails).preload(:emails).group_by(&:all_emails) + + grouped_users_by_email.each_with_object({}) do |(found_emails, users), h| + found_emails.each { |e| h[e] = users.first if emails.include?(e) } # don't include all emails for an account, only the ones we want + end + end + end + + private + + def user_by_email_resource_key + "user_by_email_for_#{User.name.underscore.pluralize}:#{self.class}:#{self.id}" + end + end +end diff --git a/app/models/concerns/featurable.rb b/app/models/concerns/featurable.rb index 70d67fc7559..08189d83534 100644 --- a/app/models/concerns/featurable.rb +++ b/app/models/concerns/featurable.rb @@ -50,7 +50,7 @@ module Featurable end def available_features - @available_features + @available_features || [] end def access_level_attribute(feature) @@ -74,6 +74,12 @@ module Featurable STRING_OPTIONS.key(level) end + def required_minimum_access_level(feature) + ensure_feature!(feature) + + Gitlab::Access::GUEST + end + def ensure_feature!(feature) feature = feature.model_name.plural if feature.respond_to?(:model_name) feature = feature.to_sym @@ -91,8 +97,8 @@ module Featurable public_send(self.class.access_level_attribute(feature)) # rubocop:disable GitlabSecurity/PublicSend end - def feature_available?(feature, user) - get_permission(user, feature) + def feature_available?(feature, user = nil) + has_permission?(user, feature) end def string_access_level(feature) @@ -115,4 +121,30 @@ module Featurable def feature_validation_exclusion [] end + + def has_permission?(user, feature) + case access_level(feature) + when DISABLED + false + when PRIVATE + member?(user, feature) + when ENABLED + true + when PUBLIC + true + else + true + end + end + + def member?(user, feature) + return false unless user + return true if user.can_read_all_resources? + + resource_member?(user, feature) + end + + def resource_member?(user, feature) + raise NotImplementedError + end end diff --git a/app/models/concerns/from_set_operator.rb b/app/models/concerns/from_set_operator.rb index c6d63631c84..ce3a83e9fa1 100644 --- a/app/models/concerns/from_set_operator.rb +++ b/app/models/concerns/from_set_operator.rb @@ -11,7 +11,12 @@ module FromSetOperator raise "Trying to redefine method '#{method(method_name)}'" if methods.include?(method_name) define_method(method_name) do |members, remove_duplicates: true, remove_order: true, alias_as: table_name| - operator_sql = operator.new(members, remove_duplicates: remove_duplicates, remove_order: remove_order).to_sql + operator_sql = + if members.any? + operator.new(members, remove_duplicates: remove_duplicates, remove_order: remove_order).to_sql + else + where("1=0").to_sql + end from(Arel.sql("(#{operator_sql}) #{alias_as}")) end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 1eb30e88f16..dbd760a9c45 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -195,7 +195,7 @@ module Issuable end def supports_escalation? - return false unless ::Feature.enabled?(:incident_escalations, project) + return false unless ::Feature.enabled?(:incident_escalations, project, default_enabled: :yaml) incident? end @@ -318,12 +318,16 @@ module Issuable # 2. We can't ORDER BY a column that isn't in the GROUP BY and doesn't # have an aggregate function applied, so we do a useless MIN() instead. # - milestones_due_date = 'MIN(milestones.due_date)' + milestones_due_date = Milestone.arel_table[:due_date].minimum + milestones_due_date_with_direction = direction == 'ASC' ? milestones_due_date.asc : milestones_due_date.desc + + highest_priority_arel = Arel.sql('highest_priority') + highest_priority_arel_with_direction = direction == 'ASC' ? highest_priority_arel.asc : highest_priority_arel.desc order_milestone_due_asc .order_labels_priority(excluded_labels: excluded_labels, extra_select_columns: [milestones_due_date]) - .reorder(Gitlab::Database.nulls_last_order(milestones_due_date, direction), - Gitlab::Database.nulls_last_order('highest_priority', direction)) + .reorder(milestones_due_date_with_direction.nulls_last, + highest_priority_arel_with_direction.nulls_last) end def order_labels_priority(direction = 'ASC', excluded_labels: [], extra_select_columns: [], with_cte: false) @@ -341,12 +345,15 @@ module Issuable extra_select_columns.unshift("highest_priorities.label_priority as highest_priority") + highest_priority_arel = Arel.sql('highest_priority') + highest_priority_arel_with_direction = direction == 'ASC' ? highest_priority_arel.asc : highest_priority_arel.desc + select(issuable_columns) .select(extra_select_columns) .from("#{table_name}") .joins("JOIN LATERAL(#{highest_priority}) as highest_priorities ON TRUE") .group(group_columns) - .reorder(Gitlab::Database.nulls_last_order('highest_priority', direction)) + .reorder(highest_priority_arel_with_direction.nulls_last) end def with_label(title, sort = nil) @@ -524,6 +531,10 @@ module Issuable labels.order('title ASC').pluck(:title) end + def labels_hook_attrs + labels.map(&:hook_attrs) + end + # Convert this Issuable class name to a format usable by Ability definitions # # Examples: diff --git a/app/models/concerns/issuable_link.rb b/app/models/concerns/issuable_link.rb index 3e14507bc70..c319d685362 100644 --- a/app/models/concerns/issuable_link.rb +++ b/app/models/concerns/issuable_link.rb @@ -29,6 +29,8 @@ module IssuableLink validate :check_self_relation validate :check_opposite_relation + scope :for_source_or_target, ->(issuable) { where(source: issuable).or(where(target: issuable)) } + enum link_type: { TYPE_RELATES_TO => 0, TYPE_BLOCKS => 1 } private diff --git a/app/models/concerns/metric_image_uploading.rb b/app/models/concerns/metric_image_uploading.rb new file mode 100644 index 00000000000..3f7797f56c5 --- /dev/null +++ b/app/models/concerns/metric_image_uploading.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module MetricImageUploading + extend ActiveSupport::Concern + + MAX_FILE_SIZE = 1.megabyte.freeze + + included do + include Gitlab::FileTypeDetection + include FileStoreMounter + include WithUploads + + validates :file, presence: true + validate :validate_file_is_image + validates :url, length: { maximum: 255 }, public_url: { allow_blank: true } + validates :url_text, length: { maximum: 128 } + + scope :order_created_at_asc, -> { order(created_at: :asc) } + + attribute :file_store, :integer, default: -> { MetricImageUploader.default_store } + + mount_file_store_uploader MetricImageUploader + end + + def filename + @filename ||= file&.filename + end + + def file_path + @file_path ||= begin + return file&.url unless file&.upload + + # If we're using a CDN, we need to use the full URL + asset_host = ActionController::Base.asset_host || Gitlab.config.gitlab.base_url + + Gitlab::Utils.append_path(asset_host, local_path) + end + end + + private + + def valid_file_extensions + Gitlab::FileTypeDetection::SAFE_IMAGE_EXT + end + + def validate_file_is_image + unless image? + message = _('does not have a supported extension. Only %{extension_list} are supported') % { + extension_list: valid_file_extensions.to_sentence + } + errors.add(:file, message) + end + end +end diff --git a/app/models/concerns/sensitive_serializable_hash.rb b/app/models/concerns/sensitive_serializable_hash.rb index 725ec60e9b6..94451fcd2c2 100644 --- a/app/models/concerns/sensitive_serializable_hash.rb +++ b/app/models/concerns/sensitive_serializable_hash.rb @@ -19,7 +19,6 @@ module SensitiveSerializableHash # In general, prefer NOT to use serializable_hash / to_json / as_json in favor # of serializers / entities instead which has an allowlist of attributes def serializable_hash(options = nil) - return super unless prevent_sensitive_fields_from_serializable_hash? return super if options && options[:unsafe_serialization_hash] options = options.try(:dup) || {} @@ -37,10 +36,4 @@ module SensitiveSerializableHash super(options) end - - private - - def prevent_sensitive_fields_from_serializable_hash? - Feature.enabled?(:prevent_sensitive_fields_from_serializable_hash, default_enabled: :yaml) - end end diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index b475eb79aa3..d27b451892a 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -84,7 +84,8 @@ module Spammable end def unrecoverable_spam_error! - self.errors.add(:base, "Your #{spammable_entity_type} has been recognized as spam and has been discarded.") + self.errors.add(:base, _("Your %{spammable_entity_type} has been recognized as spam and has been discarded.") \ + % { spammable_entity_type: spammable_entity_type }) end def spammable_entity_type diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index e41a0ca28f9..904c96b11b3 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -11,14 +11,16 @@ require 'task_list/filter' module Taskable COMPLETED = 'completed' INCOMPLETE = 'incomplete' - COMPLETE_PATTERN = /(\[[xX]\])/.freeze - INCOMPLETE_PATTERN = /(\[\s\])/.freeze + COMPLETE_PATTERN = /\[[xX]\]/.freeze + INCOMPLETE_PATTERN = /\[[[:space:]]\]/.freeze ITEM_PATTERN = %r{ ^ (?:(?:>\s{0,4})*) # optional blockquote characters ((?:\s*(?:[-+*]|(?:\d+\.)))+) # list prefix (one or more) required - task item has to be always in a list \s+ # whitespace prefix has to be always presented for a list item - (\[\s\]|\[[xX]\]) # checkbox + ( # checkbox + #{COMPLETE_PATTERN}|#{INCOMPLETE_PATTERN} + ) (\s.+) # followed by whitespace and some text. }x.freeze diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index fa03d73646d..78bd520d5d5 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -9,15 +9,17 @@ class ContainerRepository < ApplicationRecord WAITING_CLEANUP_STATUSES = %i[cleanup_scheduled cleanup_unfinished].freeze REQUIRING_CLEANUP_STATUSES = %i[cleanup_unscheduled cleanup_scheduled].freeze + IDLE_MIGRATION_STATES = %w[default pre_import_done import_done import_aborted import_skipped].freeze ACTIVE_MIGRATION_STATES = %w[pre_importing importing].freeze - ABORTABLE_MIGRATION_STATES = (ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze MIGRATION_STATES = (IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze + ABORTABLE_MIGRATION_STATES = (ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze + SKIPPABLE_MIGRATION_STATES = (ABORTABLE_MIGRATION_STATES + %w[import_aborted]).freeze MIGRATION_PHASE_1_STARTED_AT = Date.new(2021, 11, 4).freeze + MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze TooManyImportsError = Class.new(StandardError) - NativeImportError = Class.new(StandardError) belongs_to :project @@ -32,7 +34,17 @@ class ContainerRepository < ApplicationRecord enum status: { delete_scheduled: 0, delete_failed: 1 } enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 } - enum migration_skipped_reason: { not_in_plan: 0, too_many_retries: 1, too_many_tags: 2, root_namespace_in_deny_list: 3 } + + enum migration_skipped_reason: { + not_in_plan: 0, + too_many_retries: 1, + too_many_tags: 2, + root_namespace_in_deny_list: 3, + migration_canceled: 4, + not_found: 5, + native_import: 6, + migration_forced_canceled: 7 + } delegate :client, :gitlab_api_client, to: :registry @@ -57,8 +69,8 @@ class ContainerRepository < ApplicationRecord scope :import_in_process, -> { where(migration_state: %w[pre_importing pre_import_done importing]) } scope :recently_done_migration_step, -> do - where(migration_state: %w[import_done pre_import_done import_aborted]) - .order(Arel.sql('GREATEST(migration_pre_import_done_at, migration_import_done_at, migration_aborted_at) DESC')) + where(migration_state: %w[import_done pre_import_done import_aborted import_skipped]) + .order(Arel.sql('GREATEST(migration_pre_import_done_at, migration_import_done_at, migration_aborted_at, migration_skipped_at) DESC')) end scope :ready_for_import, -> do @@ -110,19 +122,19 @@ class ContainerRepository < ApplicationRecord end event :start_pre_import do - transition default: :pre_importing + transition %i[default pre_importing importing import_aborted] => :pre_importing end event :finish_pre_import do - transition %i[pre_importing import_aborted] => :pre_import_done + transition %i[pre_importing importing import_aborted] => :pre_import_done end event :start_import do - transition pre_import_done: :importing + transition %i[pre_import_done pre_importing importing import_aborted] => :importing end event :finish_import do - transition %i[importing import_aborted] => :import_done + transition %i[default pre_importing importing import_aborted] => :import_done end event :already_migrated do @@ -134,15 +146,15 @@ class ContainerRepository < ApplicationRecord end event :skip_import do - transition ABORTABLE_MIGRATION_STATES.map(&:to_sym) => :import_skipped + transition SKIPPABLE_MIGRATION_STATES.map(&:to_sym) => :import_skipped end event :retry_pre_import do - transition import_aborted: :pre_importing + transition %i[pre_importing importing import_aborted] => :pre_importing end event :retry_import do - transition import_aborted: :importing + transition %i[pre_importing importing import_aborted] => :importing end before_transition any => :pre_importing do |container_repository| @@ -150,13 +162,16 @@ class ContainerRepository < ApplicationRecord container_repository.migration_pre_import_done_at = nil end - after_transition any => :pre_importing do |container_repository| + after_transition any => :pre_importing do |container_repository, transition| + forced = transition.args.first.try(:[], :forced) + next if forced + container_repository.try_import do container_repository.migration_pre_import end end - before_transition %i[pre_importing import_aborted] => :pre_import_done do |container_repository| + before_transition any => :pre_import_done do |container_repository| container_repository.migration_pre_import_done_at = Time.zone.now end @@ -165,13 +180,16 @@ class ContainerRepository < ApplicationRecord container_repository.migration_import_done_at = nil end - after_transition any => :importing do |container_repository| + after_transition any => :importing do |container_repository, transition| + forced = transition.args.first.try(:[], :forced) + next if forced + container_repository.try_import do container_repository.migration_import end end - before_transition %i[importing import_aborted] => :import_done do |container_repository| + before_transition any => :import_done do |container_repository| container_repository.migration_import_done_at = Time.zone.now end @@ -181,6 +199,12 @@ class ContainerRepository < ApplicationRecord container_repository.migration_retries_count += 1 end + after_transition any => :import_aborted do |container_repository| + if container_repository.retried_too_many_times? + container_repository.skip_import(reason: :too_many_retries) + end + end + before_transition import_aborted: any do |container_repository| container_repository.migration_aborted_at = nil container_repository.migration_aborted_in_state = nil @@ -204,6 +228,13 @@ class ContainerRepository < ApplicationRecord ).exists? end + def self.all_migrated? + # check that the set of non migrated repositories is empty + where(created_at: ...MIGRATION_PHASE_1_ENDED_AT) + .where.not(migration_state: 'import_done') + .empty? + end + def self.with_enabled_policy joins('INNER JOIN container_expiration_policies ON container_repositories.project_id = container_expiration_policies.project_id') .where(container_expiration_policies: { enabled: true }) @@ -250,10 +281,10 @@ class ContainerRepository < ApplicationRecord super end - def start_pre_import + def start_pre_import(*args) return false unless ContainerRegistry::Migration.enabled? - super + super(*args) end def retry_pre_import @@ -276,24 +307,38 @@ class ContainerRepository < ApplicationRecord def retry_aborted_migration return unless migration_state == 'import_aborted' - case external_import_status + reconcile_import_status(external_import_status) do + # If the import_status request fails, use the timestamp to guess current state + migration_pre_import_done_at ? retry_import : retry_pre_import + end + end + + def reconcile_import_status(status) + case status when 'native' - raise NativeImportError + finish_import_as(:native_import) + when 'pre_import_in_progress' + return if pre_importing? + + start_pre_import(forced: true) when 'import_in_progress' - nil + return if importing? + + start_import(forced: true) + when 'import_canceled', 'pre_import_canceled' + return if import_skipped? + + skip_import(reason: :migration_canceled) when 'import_complete' finish_import when 'import_failed' retry_import - when 'pre_import_in_progress' - nil when 'pre_import_complete' finish_pre_import_and_start_import when 'pre_import_failed' retry_pre_import else - # If the import_status request fails, use the timestamp to guess current state - migration_pre_import_done_at ? retry_import : retry_pre_import + yield end end @@ -303,9 +348,18 @@ class ContainerRepository < ApplicationRecord try_count = 0 begin try_count += 1 - return true if yield == :ok - abort_import + case yield + when :ok + return true + when :not_found + finish_import_as(:not_found) + when :already_imported + finish_import_as(:native_import) + else + abort_import + end + false rescue TooManyImportsError if try_count <= ::ContainerRegistry::Migration.start_max_retries @@ -318,8 +372,12 @@ class ContainerRepository < ApplicationRecord end end + def retried_too_many_times? + migration_retries_count >= ContainerRegistry::Migration.max_retries + end + def last_import_step_done_at - [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at].compact.max + [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at, migration_skipped_at].compact.max end def external_import_status @@ -416,7 +474,7 @@ class ContainerRepository < ApplicationRecord next if self.created_at.before?(MIGRATION_PHASE_1_STARTED_AT) next unless gitlab_api_client.supports_gitlab_api? - gitlab_api_client.repository_details(self.path, with_size: true)['size_bytes'] + gitlab_api_client.repository_details(self.path, sizing: :self)['size_bytes'] end end @@ -450,6 +508,25 @@ class ContainerRepository < ApplicationRecord response end + def migration_cancel + return :error unless gitlab_api_client.supports_gitlab_api? + + gitlab_api_client.cancel_repository_import(self.path) + end + + # This method is not meant for consumption by the code + # It is meant for manual use in the case that a migration needs to be + # cancelled by an admin or SRE + def force_migration_cancel + return :error unless gitlab_api_client.supports_gitlab_api? + + response = gitlab_api_client.cancel_repository_import(self.path, force: true) + + skip_import(reason: :migration_forced_canceled) if response[:status] == :ok + + response + end + def self.build_from_path(path) self.new(project: path.repository_project, name: path.repository_name) @@ -478,6 +555,13 @@ class ContainerRepository < ApplicationRecord self.find_by(project: path.repository_project, name: path.repository_name) end + + private + + def finish_import_as(reason) + self.migration_skipped_reason = reason + finish_import + end end ContainerRepository.prepend_mod_with('ContainerRepository') diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 173b38b2c63..09fbb93525b 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -28,6 +28,19 @@ class CustomEmoji < ApplicationRecord alias_attribute :url, :file # this might need a change in https://gitlab.com/gitlab-org/gitlab/-/issues/230467 + # Find custom emoji for the given resource. + # A resource can be either a Project or a Group, or anything responding to #root_ancestor. + # Usually it's the return value of #resource_parent on any model. + scope :for_resource, -> (resource) do + return none if resource.nil? + + namespace = resource.root_ancestor + + return none if namespace.nil? || Feature.disabled?(:custom_emoji, namespace) + + namespace.custom_emoji + end + private def valid_emoji_name diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index 4fa2c3fb8cf..cdb449e00bf 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -23,7 +23,7 @@ class CustomerRelations::Contact < ApplicationRecord validates :last_name, presence: true, length: { maximum: 255 } validates :email, length: { maximum: 255 } validates :description, length: { maximum: 1024 } - validates :email, uniqueness: { scope: :group_id } + validates :email, uniqueness: { case_sensitive: false, scope: :group_id } validate :validate_email_format validate :validate_root_group @@ -42,7 +42,7 @@ class CustomerRelations::Contact < ApplicationRecord def self.find_ids_by_emails(group, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK - where(group: group, email: emails).pluck(:id) + where(group: group).where('lower(email) in (?)', emails.map(&:downcase)).pluck(:id) end def self.exists_for_group?(group) @@ -51,6 +51,34 @@ class CustomerRelations::Contact < ApplicationRecord exists?(group: group) end + def self.move_to_root_group(group) + update_query = <<~SQL + UPDATE #{CustomerRelations::IssueContact.table_name} + SET contact_id = new_contacts.id + FROM #{table_name} AS existing_contacts + JOIN #{table_name} AS new_contacts ON new_contacts.group_id = :old_group_id AND LOWER(new_contacts.email) = LOWER(existing_contacts.email) + WHERE existing_contacts.group_id = :new_group_id AND contact_id = existing_contacts.id + SQL + connection.execute(sanitize_sql([ + update_query, + old_group_id: group.root_ancestor.id, + new_group_id: group.id + ])) + + dupes_query = <<~SQL + DELETE FROM #{table_name} AS existing_contacts + USING #{table_name} AS new_contacts + WHERE existing_contacts.group_id = :new_group_id AND new_contacts.group_id = :old_group_id AND LOWER(new_contacts.email) = LOWER(existing_contacts.email) + SQL + connection.execute(sanitize_sql([ + dupes_query, + old_group_id: group.root_ancestor.id, + new_group_id: group.id + ])) + + where(group: group).update_all(group_id: group.root_ancestor.id) + end + private def validate_email_format diff --git a/app/models/customer_relations/issue_contact.rb b/app/models/customer_relations/issue_contact.rb index dc7a3fd87bc..70a30e583d5 100644 --- a/app/models/customer_relations/issue_contact.rb +++ b/app/models/customer_relations/issue_contact.rb @@ -8,6 +8,8 @@ class CustomerRelations::IssueContact < ApplicationRecord validate :contact_belongs_to_root_group + BATCH_DELETE_SIZE = 1_000 + def self.find_contact_ids_by_emails(issue_id, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK @@ -17,9 +19,17 @@ class CustomerRelations::IssueContact < ApplicationRecord end def self.delete_for_project(project_id) - joins(:issue) - .where(issues: { project_id: project_id }) - .delete_all + loop do + deleted_records = joins(:issue).where(issues: { project_id: project_id }).limit(BATCH_DELETE_SIZE).delete_all + break if deleted_records == 0 + end + end + + def self.delete_for_group(group) + loop do + deleted_records = joins(issue: :project).where(projects: { namespace: group.self_and_descendants }).limit(BATCH_DELETE_SIZE).delete_all + break if deleted_records == 0 + end end private diff --git a/app/models/customer_relations/organization.rb b/app/models/customer_relations/organization.rb index a23b9d8fe28..32adcc7492b 100644 --- a/app/models/customer_relations/organization.rb +++ b/app/models/customer_relations/organization.rb @@ -26,6 +26,34 @@ class CustomerRelations::Organization < ApplicationRecord .where('LOWER(name) = LOWER(?)', name) end + def self.move_to_root_group(group) + update_query = <<~SQL + UPDATE #{CustomerRelations::Contact.table_name} + SET organization_id = new_organizations.id + FROM #{table_name} AS existing_organizations + JOIN #{table_name} AS new_organizations ON new_organizations.group_id = :old_group_id AND LOWER(new_organizations.name) = LOWER(existing_organizations.name) + WHERE existing_organizations.group_id = :new_group_id AND organization_id = existing_organizations.id + SQL + connection.execute(sanitize_sql([ + update_query, + old_group_id: group.root_ancestor.id, + new_group_id: group.id + ])) + + dupes_query = <<~SQL + DELETE FROM #{table_name} AS existing_organizations + USING #{table_name} AS new_organizations + WHERE existing_organizations.group_id = :new_group_id AND new_organizations.group_id = :old_group_id AND LOWER(new_organizations.name) = LOWER(existing_organizations.name) + SQL + connection.execute(sanitize_sql([ + dupes_query, + old_group_id: group.root_ancestor.id, + new_group_id: group.id + ])) + + where(group: group).update_all(group_id: group.root_ancestor.id) + end + private def validate_root_group diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 326d3fb8470..360a9ffbc53 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -14,6 +14,11 @@ class DeployToken < ApplicationRecord default_value_for(:expires_at) { Forever.date } + # Do NOT use this `user` for the authentication/authorization of the deploy tokens. + # It's for the auditing purpose on Credential Inventory, only. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/353467#note_859774246 for more information. + belongs_to :user, foreign_key: :creator_id, optional: true + has_many :project_deploy_tokens, inverse_of: :deploy_token has_many :projects, through: :project_deploy_tokens diff --git a/app/models/deployment.rb b/app/models/deployment.rb index c06c809538a..63d531d82c3 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -14,8 +14,8 @@ class Deployment < ApplicationRecord ARCHIVABLE_OFFSET = 50_000 - belongs_to :project, required: true - belongs_to :environment, required: true + belongs_to :project, optional: false + belongs_to :environment, optional: false belongs_to :cluster, class_name: 'Clusters::Cluster', optional: true belongs_to :user belongs_to :deployable, polymorphic: true, optional: true # rubocop:disable Cop/PolymorphicAssociations @@ -46,7 +46,7 @@ class Deployment < ApplicationRecord scope :for_project, -> (project_id) { where(project_id: project_id) } scope :for_projects, -> (projects) { where(project: projects) } - scope :visible, -> { where(status: %i[running success failed canceled blocked]) } + scope :visible, -> { where(status: VISIBLE_STATUSES) } scope :stoppable, -> { where.not(on_stop: nil).where.not(deployable_id: nil).success } scope :active, -> { where(status: %i[created running]) } scope :upcoming, -> { where(status: %i[blocked running]) } @@ -58,6 +58,7 @@ class Deployment < ApplicationRecord scope :ordered, -> { order(finished_at: :desc) } + VISIBLE_STATUSES = %i[running success failed canceled blocked].freeze FINISHED_STATUSES = %i[success failed canceled].freeze state_machine :status, initial: :created do @@ -380,6 +381,12 @@ class Deployment < ApplicationRecord status == params[:status] end + def tier_in_yaml + return unless deployable + + deployable.environment_deployment_tier + end + private def update_status!(status) diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 8a167034629..9eb3308b901 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -47,6 +47,14 @@ class Discussion grouped_notes.values.map { |notes| build(notes, context_noteable) } end + def self.build_discussions(discussion_ids, context_noteable = nil, preload_note_diff_file: false) + notes = Note.where(discussion_id: discussion_ids).fresh + notes = notes.inc_note_diff_file if preload_note_diff_file + + grouped_notes = notes.group_by { |n| n.discussion_id } + grouped_notes.transform_values { |notes| Discussion.build(notes, context_noteable) } + end + def self.lazy_find(discussion_id) BatchLoader.for(discussion_id).batch do |discussion_ids, loader| results = Note.where(discussion_id: discussion_ids).fresh.to_a.group_by(&:discussion_id) diff --git a/app/models/environment.rb b/app/models/environment.rb index 450ed6206d5..9e663b2ee74 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -12,7 +12,7 @@ class Environment < ApplicationRecord self.reactive_cache_hard_limit = 10.megabytes self.reactive_cache_work_type = :external_dependency - belongs_to :project, required: true + belongs_to :project, optional: false use_fast_destroy :all_deployments nullify_if_blank :external_url @@ -26,7 +26,7 @@ class Environment < ApplicationRecord has_many :self_managed_prometheus_alert_events, inverse_of: :environment has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment - has_one :last_deployment, -> { success.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment + has_one :last_deployment, -> { Feature.enabled?(:env_last_deployment_by_finished_at, default_enabled: :yaml) ? success.ordered : success.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment has_one :last_visible_deployment, -> { visible.distinct_on_environment }, inverse_of: :environment, class_name: 'Deployment' has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus', disable_joins: true has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline', disable_joins: true @@ -59,17 +59,17 @@ class Environment < ApplicationRecord allow_nil: true, addressable_url: true - delegate :stop_action, :manual_actions, to: :last_deployment, allow_nil: true + delegate :manual_actions, to: :last_deployment, allow_nil: true delegate :auto_rollback_enabled?, to: :project scope :available, -> { with_state(:available) } scope :stopped, -> { with_state(:stopped) } scope :order_by_last_deployed_at, -> do - order(Gitlab::Database.nulls_first_order("(#{max_deployment_id_sql})", 'ASC')) + order(Arel::Nodes::Grouping.new(max_deployment_id_query).asc.nulls_first) end scope :order_by_last_deployed_at_desc, -> do - order(Gitlab::Database.nulls_last_order("(#{max_deployment_id_sql})", 'DESC')) + order(Arel::Nodes::Grouping.new(max_deployment_id_query).desc.nulls_last) end scope :order_by_name, -> { order('environments.name ASC') } @@ -89,13 +89,19 @@ class Environment < ApplicationRecord scope :for_project, -> (project) { where(project_id: project) } scope :for_tier, -> (tier) { where(tier: tier).where.not(tier: nil) } - scope :with_deployment, -> (sha) { where('EXISTS (?)', Deployment.select(1).where('deployments.environment_id = environments.id').where(sha: sha)) } scope :unfoldered, -> { where(environment_type: nil) } scope :with_rank, -> do select('environments.*, rank() OVER (PARTITION BY project_id ORDER BY id DESC)') end scope :for_id, -> (id) { where(id: id) } + scope :with_deployment, -> (sha, status: nil) do + deployments = Deployment.select(1).where('deployments.environment_id = environments.id').where(sha: sha) + deployments = deployments.where(status: status) if status + + where('EXISTS (?)', deployments) + end + scope :stopped_review_apps, -> (before, limit) do stopped .in_review_folder @@ -145,10 +151,11 @@ class Environment < ApplicationRecord find_by(id: id, slug: slug) end - def self.max_deployment_id_sql - Deployment.select(Deployment.arel_table[:id].maximum) - .where(Deployment.arel_table[:environment_id].eq(arel_table[:id])) - .to_sql + def self.max_deployment_id_query + Arel.sql( + Deployment.select(Deployment.arel_table[:id].maximum) + .where(Deployment.arel_table[:environment_id].eq(arel_table[:id])).to_sql + ) end def self.pluck_names @@ -185,6 +192,23 @@ class Environment < ApplicationRecord last_deployment&.deployable end + def last_deployment_pipeline + last_deployable&.pipeline + end + + # This method returns the deployment records of the last deployment pipeline, that successfully executed to this environment. + # e.g. + # A pipeline contains + # - deploy job A => production environment + # - deploy job B => production environment + # In this case, `last_deployment_group` returns both deployments, whereas `last_deployable` returns only B. + def last_deployment_group + return Deployment.none unless last_deployment_pipeline + + successful_deployments.where( + deployable_id: last_deployment_pipeline.latest_builds.pluck(:id)) + end + # NOTE: Below assocation overrides is a workaround for issue https://gitlab.com/gitlab-org/gitlab/-/issues/339908 # It helps to avoid cross joins with the CI database. # Caveat: It also overrides and losses the default AR caching mechanism. @@ -255,8 +279,8 @@ class Environment < ApplicationRecord external_url.gsub(%r{\A.*?://}, '') end - def stop_action_available? - available? && stop_action.present? + def stop_actions_available? + available? && stop_actions.present? end def cancel_deployment_jobs! @@ -269,11 +293,35 @@ class Environment < ApplicationRecord end end - def stop_with_action!(current_user) + def stop_with_actions!(current_user) return unless available? stop! - stop_action&.play(current_user) + + actions = [] + + stop_actions.each do |stop_action| + Gitlab::OptimisticLocking.retry_lock( + stop_action, + name: 'environment_stop_with_actions' + ) do |build| + actions << build.play(current_user) + end + end + + actions + end + + def stop_actions + strong_memoize(:stop_actions) do + if ::Feature.enabled?(:environment_multiple_stop_actions, project, default_enabled: :yaml) + # Fix N+1 queries it brings to the serializer. + # Tracked in https://gitlab.com/gitlab-org/gitlab/-/issues/358780 + last_deployment_group.map(&:stop_action).compact + else + [last_deployment&.stop_action].compact + end + end end def reset_auto_stop diff --git a/app/models/environment_status.rb b/app/models/environment_status.rb index 07c0983f239..43b2c7899a1 100644 --- a/app/models/environment_status.rb +++ b/app/models/environment_status.rb @@ -51,7 +51,7 @@ class EnvironmentStatus def deployment strong_memoize(:deployment) do - Deployment.where(environment: environment).find_by_sha(sha) + Deployment.where(environment: environment).ordered.find_by_sha(sha) end end diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index 0a429bb7afd..3ecfb895dac 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -135,7 +135,7 @@ module ErrorTracking end end - def update_issue(opts = {} ) + def update_issue(opts = {}) handle_exceptions do { updated: sentry_client.update_issue(opts) } end diff --git a/app/models/event.rb b/app/models/event.rb index a8cf2e2dfb0..e9a98c06b59 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -10,6 +10,7 @@ class Event < ApplicationRecord include UsageStatistics include ShaAttribute + # TODO https://gitlab.com/gitlab-org/gitlab/-/issues/358088 default_scope { reorder(nil) } # rubocop:disable Cop/DefaultScope ACTIONS = HashWithIndifferentAccess.new( @@ -30,8 +31,9 @@ class Event < ApplicationRecord private_constant :ACTIONS WIKI_ACTIONS = [:created, :updated, :destroyed].freeze - DESIGN_ACTIONS = [:created, :updated, :destroyed].freeze + TEAM_ACTIONS = [:joined, :left, :expired].freeze + ISSUE_ACTIONS = [:created, :updated, :closed, :reopened].freeze TARGET_TYPES = HashWithIndifferentAccess.new( issue: Issue, diff --git a/app/models/group.rb b/app/models/group.rb index 14d088dd38b..990c06fdc41 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -17,6 +17,7 @@ class Group < Namespace include GroupAPICompatibility include EachBatch include BulkMemberAccessLoad + include BulkUsersByEmailLoad include ChronicDurationAttribute include RunnerTokenExpirationInterval @@ -42,7 +43,28 @@ class Group < Namespace has_many :milestones has_many :integrations has_many :shared_group_links, foreign_key: :shared_with_group_id, class_name: 'GroupGroupLink' - has_many :shared_with_group_links, foreign_key: :shared_group_id, class_name: 'GroupGroupLink' + has_many :shared_with_group_links, foreign_key: :shared_group_id, class_name: 'GroupGroupLink' do + def of_ancestors + group = proxy_association.owner + + return GroupGroupLink.none unless group.has_parent? + + GroupGroupLink.where(shared_group_id: group.ancestors.reorder(nil).select(:id)) + end + + def of_ancestors_and_self + group = proxy_association.owner + + source_ids = + if group.has_parent? + group.self_and_ancestors.reorder(nil).select(:id) + else + group.id + end + + GroupGroupLink.where(shared_group_id: source_ids) + end + end has_many :shared_groups, through: :shared_group_links, source: :shared_group has_many :shared_with_groups, through: :shared_with_group_links, source: :shared_with_group has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -60,8 +82,9 @@ class Group < Namespace has_many :boards has_many :badges, class_name: 'GroupBadge' - has_many :organizations, class_name: 'CustomerRelations::Organization', inverse_of: :group - has_many :contacts, class_name: 'CustomerRelations::Contact', inverse_of: :group + # AR defaults to nullify when trying to delete via has_many associations unless we set dependent: :delete_all + has_many :organizations, class_name: 'CustomerRelations::Organization', inverse_of: :group, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :contacts, class_name: 'CustomerRelations::Contact', inverse_of: :group, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :cluster_groups, class_name: 'Clusters::Group' has_many :clusters, through: :cluster_groups, class_name: 'Clusters::Cluster' @@ -94,6 +117,8 @@ class Group < Namespace has_many :group_callouts, class_name: 'Users::GroupCallout', foreign_key: :group_id + has_one :group_feature, inverse_of: :group, class_name: 'Groups::FeatureSetting' + delegate :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, :setup_for_company, :jobs_to_be_done, to: :namespace_settings delegate :runner_token_expiration_interval, :runner_token_expiration_interval=, :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval_human_readable=, to: :namespace_settings, allow_nil: true delegate :subgroup_runner_token_expiration_interval, :subgroup_runner_token_expiration_interval=, :subgroup_runner_token_expiration_interval_human_readable, :subgroup_runner_token_expiration_interval_human_readable=, to: :namespace_settings, allow_nil: true @@ -102,6 +127,7 @@ class Group < Namespace has_one :crm_settings, class_name: 'Group::CrmSettings', inverse_of: :group accepts_nested_attributes_for :variables, allow_destroy: true + accepts_nested_attributes_for :group_feature, update_only: true validate :visibility_level_allowed_by_projects validate :visibility_level_allowed_by_sub_groups @@ -117,6 +143,8 @@ class Group < Namespace message: Gitlab::Regex.group_name_regex_message }, if: :name_changed? + validates :group_feature, presence: true + add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption, default_enabled: true) ? :optional : :required }, prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX @@ -125,6 +153,7 @@ class Group < Namespace after_destroy :post_destroy_hook after_save :update_two_factor_requirement after_update :path_changed_hook, if: :saved_change_to_path? + after_create -> { create_or_load_association(:group_feature) } scope :with_users, -> { includes(:users) } @@ -344,14 +373,16 @@ class Group < Namespace ) end - def add_user(user, access_level, current_user: nil, expires_at: nil, ldap: false) - Members::Groups::CreatorService.new(self, # rubocop:disable CodeReuse/ServiceClass - user, - access_level, - current_user: current_user, - expires_at: expires_at, - ldap: ldap) - .execute + def add_user(user, access_level, current_user: nil, expires_at: nil, ldap: false, blocking_refresh: true) + Members::Groups::CreatorService.new( # rubocop:disable CodeReuse/ServiceClass + self, + user, + access_level, + current_user: current_user, + expires_at: expires_at, + ldap: ldap, + blocking_refresh: blocking_refresh + ).execute end def add_guest(user, current_user = nil) @@ -794,6 +825,10 @@ class Group < Namespace super || build_dependency_proxy_setting end + def group_feature + super || build_group_feature + end + def crm_enabled? crm_settings&.enabled? end @@ -813,8 +848,32 @@ class Group < Namespace ].compact.min end + def work_items_feature_flag_enabled? + feature_flag_enabled_for_self_or_ancestor?(:work_items) + end + + # Check for enabled features, similar to `Project#feature_available?` + # NOTE: We still want to keep this after removing `Namespace#feature_available?`. + override :feature_available? + def feature_available?(feature, user = nil) + if ::Groups::FeatureSetting.available_features.include?(feature) + group_feature.feature_available?(feature, user) # rubocop:disable Gitlab/FeatureAvailableUsage + else + super + end + end + private + def feature_flag_enabled_for_self_or_ancestor?(feature_flag) + actors = [root_ancestor] + actors << self if root_ancestor != self + + actors.any? do |actor| + ::Feature.enabled?(feature_flag, actor, default_enabled: :yaml) + end + end + def max_member_access(user_ids) Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(User), resource_ids: user_ids, diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index c4c3fc390e1..b0020f097b5 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -16,6 +16,19 @@ class GroupGroupLink < ApplicationRecord scope :non_guests, -> { where('group_access > ?', Gitlab::Access::GUEST) } scope :preload_shared_with_groups, -> { preload(:shared_with_group) } + scope :distinct_on_shared_with_group_id_with_group_access, -> do + distinct_group_links = select('DISTINCT ON (shared_with_group_id) *') + .order('shared_with_group_id, group_access DESC, expires_at DESC, created_at ASC') + + unscoped.from(distinct_group_links, :group_group_links) + end + + alias_method :shared_from, :shared_group + + def self.search(query) + joins(:shared_with_group).merge(Group.search(query)) + end + def self.access_options Gitlab::Access.options_with_owner end diff --git a/app/models/groups/feature_setting.rb b/app/models/groups/feature_setting.rb new file mode 100644 index 00000000000..72d0851ea85 --- /dev/null +++ b/app/models/groups/feature_setting.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Groups + class FeatureSetting < ApplicationRecord + include Featurable + extend ::Gitlab::Utils::Override + + self.primary_key = :group_id + self.table_name = 'group_features' + + belongs_to :group + + validates :group, presence: true + + private + + override :resource_member? + def resource_member?(user, feature) + group.member?(user, ::Groups::FeatureSetting.required_minimum_access_level(feature)) + end + end +end + +::Groups::FeatureSetting.prepend_mod_with('Groups::FeatureSetting') diff --git a/app/models/integration.rb b/app/models/integration.rb index 274c16507b7..c0e244e38b6 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -10,9 +10,11 @@ class Integration < ApplicationRecord include FromUnion include EachBatch include IgnorableColumns + extend ::Gitlab::Utils::Override ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22' ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22' + ignore_column :properties, remove_with: '15.1', remove_after: '2022-05-22' UnknownType = Class.new(StandardError) @@ -47,10 +49,7 @@ class Integration < ApplicationRecord SECTION_TYPE_CONNECTION = 'connection' - serialize :properties, JSON # rubocop:disable Cop/ActiveRecordSerialize - - attr_encrypted :encrypted_properties_tmp, - attribute: :encrypted_properties, + attr_encrypted :properties, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_32, algorithm: 'aes-256-gcm', @@ -59,6 +58,15 @@ class Integration < ApplicationRecord encode: false, encode_iv: false + # Handle assignment of props with symbol keys. + # To do this correctly, we need to call the method generated by attr_encrypted. + alias_method :attr_encrypted_props=, :properties= + private :attr_encrypted_props= + + def properties=(props) + self.attr_encrypted_props = props&.with_indifferent_access&.freeze + end + alias_attribute :type, :type_new default_value_for :active, false @@ -77,8 +85,6 @@ class Integration < ApplicationRecord default_value_for :wiki_page_events, true after_initialize :initialize_properties - after_initialize :copy_properties_to_encrypted_properties - before_save :copy_properties_to_encrypted_properties after_commit :reset_updated_properties @@ -96,6 +102,9 @@ class Integration < ApplicationRecord validate :validate_belongs_to_project_or_group scope :external_issue_trackers, -> { where(category: 'issue_tracker').active } + # TODO: Will be modified in 15.0 + # Details: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74501#note_744393645 + scope :third_party_wikis, -> { where(type: %w[Integrations::Confluence Integrations::Shimo]).active } scope :by_name, ->(name) { by_type(integration_name_to_type(name)) } scope :external_wikis, -> { by_name(:external_wiki).active } scope :active, -> { where(active: true) } @@ -162,16 +171,14 @@ class Integration < ApplicationRecord class_eval <<~RUBY, __FILE__, __LINE__ + 1 unless method_defined?(arg) def #{arg} - properties['#{arg}'] + properties['#{arg}'] if properties.present? end end def #{arg}=(value) self.properties ||= {} - self.encrypted_properties_tmp = properties updated_properties['#{arg}'] = #{arg} unless #{arg}_changed? - self.properties['#{arg}'] = value - self.encrypted_properties_tmp['#{arg}'] = value + self.properties = self.properties.merge('#{arg}' => value) end def #{arg}_changed? @@ -192,11 +199,13 @@ class Integration < ApplicationRecord # Provide convenient boolean accessor methods for each serialized property. # Also keep track of updated properties in a similar way as ActiveModel::Dirty def self.boolean_accessor(*args) - self.prop_accessor(*args) + prop_accessor(*args) args.each do |arg| class_eval <<~RUBY, __FILE__, __LINE__ + 1 def #{arg} + return if properties.blank? + Gitlab::Utils.to_boolean(properties['#{arg}']) end @@ -315,18 +324,31 @@ class Integration < ApplicationRecord def self.build_from_integration(integration, project_id: nil, group_id: nil) new_integration = integration.dup - if integration.supports_data_fields? - data_fields = integration.data_fields.dup - data_fields.integration = new_integration - end - new_integration.instance = false new_integration.project_id = project_id new_integration.group_id = group_id - new_integration.inherit_from_id = integration.id if integration.instance_level? || integration.group_level? + new_integration.inherit_from_id = integration.id if integration.inheritable? new_integration end + # Duplicating an integration also duplicates the data fields. Duped records have different ciphertexts. + override :dup + def dup + new_integration = super + new_integration.assign_attributes(reencrypt_properties) + + if supports_data_fields? + fields = data_fields.dup + fields.integration = new_integration + end + + new_integration + end + + def inheritable? + instance_level? || group_level? + end + def self.instance_exists_for?(type) exists?(instance: true, type: type) end @@ -350,16 +372,17 @@ class Integration < ApplicationRecord end private_class_method :instance_level_integration - def self.create_from_active_default_integrations(scope, association) - group_ids = sorted_ancestors(scope).select(:id) + # Returns the number of successfully saved integrations + # Duplicate integrations are excluded from this count by their validations. + def self.create_from_active_default_integrations(owner, association) + group_ids = sorted_ancestors(owner).select(:id) array = group_ids.to_sql.present? ? "array(#{group_ids.to_sql})" : 'ARRAY[]' + order = Arel.sql("type_new ASC, array_position(#{array}::bigint[], #{table_name}.group_id), instance DESC") - from_union([ - active.where(instance: true), - active.where(group_id: group_ids, inherit_from_id: nil) - ]).order(Arel.sql("type_new ASC, array_position(#{array}::bigint[], #{table_name}.group_id), instance DESC")).group_by(&:type).each do |type, records| - build_from_integration(records.first, association => scope.id).save - end + from_union([active.where(instance: true), active.where(group_id: group_ids, inherit_from_id: nil)]) + .order(order) + .group_by(&:type) + .count { |type, parents| build_from_integration(parents.first, association => owner.id).save } end def self.inherited_descendants_from_self_or_ancestors_from(integration) @@ -398,13 +421,7 @@ class Integration < ApplicationRecord end def initialize_properties - self.properties = {} if has_attribute?(:properties) && properties.nil? - end - - def copy_properties_to_encrypted_properties - self.encrypted_properties_tmp = properties - rescue ActiveModel::MissingAttributeError - # ignore - in a record built from using a restricted select list + self.properties = {} if has_attribute?(:encrypted_properties) && encrypted_properties.nil? end def title @@ -428,7 +445,9 @@ class Integration < ApplicationRecord [] end - def password_fields + # TODO: Once all integrations use `Integrations::Field` we can + # use `#secret?` here. + def secret_fields fields.select { |f| f[:type] == 'password' }.pluck(:name) end @@ -439,21 +458,26 @@ class Integration < ApplicationRecord %w[active] end + # properties is always nil - ignore it. + override :attributes + def attributes + super.except('properties') + end + # return a hash of columns => values suitable for passing to insert_all def to_integration_hash column = self.class.attribute_aliases.fetch('type', 'type') - copy_properties_to_encrypted_properties - as_json(except: %w[id instance project_id group_id encrypted_properties_tmp]) + as_json(except: %w[id instance project_id group_id]) .merge(column => type) .merge(reencrypt_properties) end def reencrypt_properties unless properties.nil? || properties.empty? - alg = self.class.encrypted_attributes[:encrypted_properties_tmp][:algorithm] + alg = self.class.encrypted_attributes[:properties][:algorithm] iv = generate_iv(alg) - ep = self.class.encrypt(:encrypted_properties_tmp, properties, { iv: iv }) + ep = self.class.encrypt(:properties, properties, { iv: iv }) end { 'encrypted_properties' => ep, 'encrypted_properties_iv' => iv } diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index d5b6357cb66..54bd595892f 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -35,8 +35,9 @@ module Integrations validates :labels_to_be_notified_behavior, inclusion: { in: LABEL_NOTIFICATION_BEHAVIOURS }, allow_blank: true def initialize_properties - if properties.nil? - self.properties = {} + super + + if properties.empty? self.notify_only_broken_pipelines = true self.branches_to_be_notified = "default" self.labels_to_be_notified_behavior = MATCH_ANY_LABEL diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index 458d0199e7a..bffe87c21ee 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -25,12 +25,15 @@ module Integrations def handle_properties # this has been moved from initialize_properties and should be improved # as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - return unless properties + return unless properties.present? + + safe_keys = data_fields.attributes.keys.grep_v(/encrypted/) - %w[id service_id created_at] @legacy_properties_data = properties.dup - data_values = properties.slice!('title', 'description') + + data_values = properties.slice(*safe_keys) data_values.reject! { |key| data_fields.changed.include?(key) } - data_values.slice!(*data_fields.attributes.keys) + data_fields.assign_attributes(data_values) if data_values.present? self.properties = {} @@ -68,10 +71,6 @@ module Integrations issue_url(iid) end - def initialize_properties - {} - end - # Initialize with default properties values def set_default_data return unless issues_tracker.present? diff --git a/app/models/integrations/base_third_party_wiki.rb b/app/models/integrations/base_third_party_wiki.rb new file mode 100644 index 00000000000..24f5bec93cf --- /dev/null +++ b/app/models/integrations/base_third_party_wiki.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Integrations + class BaseThirdPartyWiki < Integration + default_value_for :category, 'third_party_wiki' + + validate :only_one_third_party_wiki, if: :activated?, on: :manual_change + + after_commit :cache_project_has_integration + + def self.supported_events + %w() + end + + private + + def only_one_third_party_wiki + return unless project_level? + + if project.integrations.third_party_wikis.id_not_in(id).any? + errors.add(:base, _('Another third-party wiki is already in use. '\ + 'Only one third-party wiki integration can be active at a time')) + end + end + + def cache_project_has_integration + return unless project && !project.destroyed? + + project_setting = project.project_setting + + project_setting.public_send("#{project_settings_cache_key}=", active?) # rubocop:disable GitlabSecurity/PublicSend + project_setting.save! + end + + def project_settings_cache_key + "has_#{self.class.to_param}" + end + end +end diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb index 90593d78a5d..b816f90ef52 100644 --- a/app/models/integrations/buildkite.rb +++ b/app/models/integrations/buildkite.rb @@ -27,12 +27,12 @@ module Integrations end # Since SSL verification will always be enabled for Buildkite, - # we no longer needs to store the boolean. + # we no longer need to store the boolean. # This is a stub method to work with deprecated API param. # TODO: remove enable_ssl_verification after 14.0 # https://gitlab.com/gitlab-org/gitlab/-/issues/222808 def enable_ssl_verification=(_value) - self.properties.delete('enable_ssl_verification') # Remove unused key + self.properties = properties.except('enable_ssl_verification') # Remove unused key end override :hook_url diff --git a/app/models/integrations/confluence.rb b/app/models/integrations/confluence.rb index 65adce7a8d6..4e1d1993d02 100644 --- a/app/models/integrations/confluence.rb +++ b/app/models/integrations/confluence.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Integrations - class Confluence < Integration + class Confluence < BaseThirdPartyWiki VALID_SCHEME_MATCH = %r{\Ahttps?\Z}.freeze VALID_HOST_MATCH = %r{\A.+\.atlassian\.net\Z}.freeze VALID_PATH_MATCH = %r{\A/wiki(/|\Z)}.freeze @@ -11,16 +11,10 @@ module Integrations validates :confluence_url, presence: true, if: :activated? validate :validate_confluence_url_is_cloud, if: :activated? - after_commit :cache_project_has_confluence - def self.to_param 'confluence' end - def self.supported_events - %w() - end - def title s_('ConfluenceService|Confluence Workspace') end @@ -80,12 +74,5 @@ module Integrations rescue URI::InvalidURIError false end - - def cache_project_has_confluence - return unless project && !project.destroyed? - - project.project_setting.save! unless project.project_setting.persisted? - project.project_setting.update_column(:has_confluence, active?) - end end end diff --git a/app/models/integrations/emails_on_push.rb b/app/models/integrations/emails_on_push.rb index a9cd67550dc..ab458bb2c27 100644 --- a/app/models/integrations/emails_on_push.rb +++ b/app/models/integrations/emails_on_push.rb @@ -13,9 +13,7 @@ module Integrations validate :number_of_recipients_within_limit, if: :validate_recipients? def self.valid_recipients(recipients) - recipients.split.select do |recipient| - recipient.include?('@') - end.uniq(&:downcase) + recipients.split.grep(Devise.email_regexp).uniq(&:downcase) end def title diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb index 49ab97677db..f00c4236a92 100644 --- a/app/models/integrations/field.rb +++ b/app/models/integrations/field.rb @@ -2,7 +2,7 @@ module Integrations class Field - SENSITIVE_NAME = %r/token|key|password|passphrase|secret/.freeze + SECRET_NAME = %r/token|key|password|passphrase|secret/.freeze ATTRIBUTES = %i[ section type placeholder required choices value checkbox_label @@ -17,7 +17,7 @@ module Integrations def initialize(name:, type: 'text', api_only: false, **attributes) @name = name.to_s.freeze - attributes[:type] = SENSITIVE_NAME.match?(@name) ? 'password' : type + attributes[:type] = SECRET_NAME.match?(@name) ? 'password' : type attributes[:api_only] = api_only @attributes = attributes.freeze end @@ -31,7 +31,7 @@ module Integrations value end - def sensitive? + def secret? @attributes[:type] == 'password' end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 74ece57000f..a800b9e5baa 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -94,10 +94,6 @@ module Integrations !!URI(url).hostname&.end_with?(JIRA_CLOUD_HOST) end - def initialize_properties - {} - end - def data_fields jira_tracker_data || self.build_jira_tracker_data end @@ -106,7 +102,7 @@ module Integrations return unless reset_password? data_fields.password = nil - properties.delete('password') if properties + self.properties = properties.except('password') end def set_default_data @@ -143,7 +139,7 @@ module Integrations end def help - jira_doc_link_start = '<a href="%{url}">'.html_safe % { url: help_page_url('integration/jira/index.html') } + jira_doc_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_path('integration/jira/index') } s_("JiraService|You must configure Jira before enabling this integration. %{jira_doc_link_start}Learn more.%{link_end}") % { jira_doc_link_start: jira_doc_link_start, link_end: '</a>'.html_safe } end @@ -160,8 +156,6 @@ module Integrations end def sections - jira_issues_link_start = '<a href="%{url}">'.html_safe % { url: help_page_url('integration/jira/issues.html') } - sections = [ { type: SECTION_TYPE_CONNECTION, @@ -180,7 +174,7 @@ module Integrations sections.push({ type: SECTION_TYPE_JIRA_ISSUES, title: _('Issues'), - description: s_('JiraService|Work on Jira issues without leaving GitLab. Add a Jira menu to access a read-only list of your Jira issues. %{jira_issues_link_start}Learn more.%{link_end}') % { jira_issues_link_start: jira_issues_link_start, link_end: '</a>'.html_safe } + description: jira_issues_section_description }) end @@ -610,6 +604,19 @@ module Integrations data_fields.deployment_server! end end + + def jira_issues_section_description + jira_issues_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_path('integration/jira/issues') } + description = s_('JiraService|Work on Jira issues without leaving GitLab. Add a Jira menu to access a read-only list of your Jira issues. %{jira_issues_link_start}Learn more.%{link_end}') % { jira_issues_link_start: jira_issues_link_start, link_end: '</a>'.html_safe } + + if project&.issues_enabled? + gitlab_issues_link_start = '<a href="%{url}">'.html_safe % { url: edit_project_path(project, anchor: 'js-shared-permissions') } + description += '<br><br>'.html_safe + description += s_("JiraService|Displaying Jira issues while leaving GitLab issues also enabled might be confusing. Consider %{gitlab_issues_link_start}disabling GitLab issues%{link_end} if they won't otherwise be used.") % { gitlab_issues_link_start: gitlab_issues_link_start, link_end: '</a>'.html_safe } + end + + description + end end end diff --git a/app/models/integrations/pipelines_email.rb b/app/models/integrations/pipelines_email.rb index 6dc41958daa..f15482dc2e1 100644 --- a/app/models/integrations/pipelines_email.rb +++ b/app/models/integrations/pipelines_email.rb @@ -12,8 +12,9 @@ module Integrations validate :number_of_recipients_within_limit, if: :validate_recipients? def initialize_properties - if properties.nil? - self.properties = {} + super + + if properties.blank? self.notify_only_broken_pipelines = true self.branches_to_be_notified = "default" elsif !self.notify_only_default_branch.nil? diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index 2e275dab91b..d6aafe45ae9 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -32,12 +32,6 @@ module Integrations scope :preload_project, -> { preload(:project) } scope :with_clusters_with_cilium, -> { joins(project: [:clusters]).merge(Clusters::Cluster.with_available_cilium) } - def initialize_properties - if properties.nil? - self.properties = {} - end - end - def show_active_box? false end diff --git a/app/models/integrations/shimo.rb b/app/models/integrations/shimo.rb index 0e1023bb7a7..dd25a0bc558 100644 --- a/app/models/integrations/shimo.rb +++ b/app/models/integrations/shimo.rb @@ -1,12 +1,10 @@ # frozen_string_literal: true module Integrations - class Shimo < Integration + class Shimo < BaseThirdPartyWiki prop_accessor :external_wiki_url validates :external_wiki_url, presence: true, public_url: true, if: :activated? - after_commit :cache_project_has_shimo - def render? return false unless Feature.enabled?(:shimo_integration, project) @@ -33,10 +31,6 @@ module Integrations nil end - def self.supported_events - %w() - end - def fields [ { @@ -47,14 +41,5 @@ module Integrations } ] end - - private - - def cache_project_has_shimo - return unless project && !project.destroyed? - - project.project_setting.save! unless project.project_setting.persisted? - project.project_setting.update_column(:has_shimo, activated?) - end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 75727fff2cd..c2b8b457049 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -118,13 +118,15 @@ class Issue < ApplicationRecord scope :not_authored_by, ->(user) { where.not(author_id: user) } - scope :order_due_date_asc, -> { reorder(::Gitlab::Database.nulls_last_order('due_date', 'ASC')) } - scope :order_due_date_desc, -> { reorder(::Gitlab::Database.nulls_last_order('due_date', 'DESC')) } + scope :order_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } + scope :order_due_date_desc, -> { reorder(arel_table[:due_date].desc.nulls_last) } scope :order_closest_future_date, -> { reorder(Arel.sql('CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC')) } scope :order_closed_date_desc, -> { reorder(closed_at: :desc) } scope :order_created_at_desc, -> { reorder(created_at: :desc) } scope :order_severity_asc, -> { includes(:issuable_severity).order('issuable_severities.severity ASC NULLS FIRST') } scope :order_severity_desc, -> { includes(:issuable_severity).order('issuable_severities.severity DESC NULLS LAST') } + scope :order_escalation_status_asc, -> { includes(:incident_management_issuable_escalation_status).order(IncidentManagement::IssuableEscalationStatus.arel_table[:status].asc.nulls_last).references(:incident_management_issuable_escalation_status) } + scope :order_escalation_status_desc, -> { includes(:incident_management_issuable_escalation_status).order(IncidentManagement::IssuableEscalationStatus.arel_table[:status].desc.nulls_last).references(:incident_management_issuable_escalation_status) } scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } @@ -133,7 +135,7 @@ class Issue < ApplicationRecord scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } scope :with_self_managed_prometheus_alert_events, -> { joins(:issues_self_managed_prometheus_alert_events) } scope :with_api_entity_associations, -> { - preload(:timelogs, :closed_by, :assignees, :author, :labels, + preload(:timelogs, :closed_by, :assignees, :author, :labels, :issuable_severity, milestone: { project: [:route, { namespace: :route }] }, project: [:route, { namespace: :route }]) } @@ -327,6 +329,8 @@ class Issue < ApplicationRecord when 'relative_position', 'relative_position_asc' then order_by_relative_position when 'severity_asc' then order_severity_asc.with_order_id_desc when 'severity_desc' then order_severity_desc.with_order_id_desc + when 'escalation_status_asc' then order_escalation_status_asc.with_order_id_desc + when 'escalation_status_desc' then order_escalation_status_desc.with_order_id_desc else super end @@ -340,8 +344,8 @@ class Issue < ApplicationRecord Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( attribute_name: 'relative_position', column_expression: arel_table[:relative_position], - order_expression: Gitlab::Database.nulls_last_order('issues.relative_position', 'ASC'), - reversed_order_expression: Gitlab::Database.nulls_last_order('issues.relative_position', 'DESC'), + order_expression: Issue.arel_table[:relative_position].asc.nulls_last, + reversed_order_expression: Issue.arel_table[:relative_position].desc.nulls_last, order_direction: :asc, nullable: :nulls_last, distinct: false @@ -382,10 +386,6 @@ class Issue < ApplicationRecord resource_parent.root_namespace&.issue_repositioning_disabled? end - def hook_attrs - Gitlab::HookData::IssueBuilder.new(self).build - end - # `from` argument can be a Namespace or Project. def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" @@ -526,10 +526,6 @@ class Issue < ApplicationRecord ::MergeRequestsClosingIssues.count_for_issue(self.id, user) end - def labels_hook_attrs - labels.map(&:hook_attrs) - end - def previous_updated_at previous_changes['updated_at']&.first || updated_at end diff --git a/app/models/key.rb b/app/models/key.rb index 4a4e792c074..42ea0f29171 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -26,7 +26,13 @@ class Key < ApplicationRecord validates :fingerprint, uniqueness: true, - presence: { message: 'cannot be generated' } + presence: { message: 'cannot be generated' }, + unless: -> { Gitlab::FIPS.enabled? } + + validates :fingerprint_sha256, + uniqueness: true, + presence: { message: 'cannot be generated' }, + if: -> { Gitlab::FIPS.enabled? } validate :key_meets_restrictions @@ -43,7 +49,7 @@ class Key < ApplicationRecord scope :preload_users, -> { preload(:user) } scope :for_user, -> (user) { where(user: user) } - scope :order_last_used_at_desc, -> { reorder(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } + scope :order_last_used_at_desc, -> { reorder(arel_table[:last_used_at].desc.nulls_last) } # Date is set specifically in this scope to improve query time. scope :expired_today_and_not_notified, -> { where(["date(expires_at AT TIME ZONE 'UTC') = CURRENT_DATE AND expiry_notification_delivered_at IS NULL"]) } @@ -129,7 +135,7 @@ class Key < ApplicationRecord return unless public_key.valid? - self.fingerprint_md5 = public_key.fingerprint + self.fingerprint_md5 = public_key.fingerprint unless Gitlab::FIPS.enabled? self.fingerprint_sha256 = public_key.fingerprint_sha256.gsub("SHA256:", "") end diff --git a/app/models/member.rb b/app/models/member.rb index 528c6855d9c..18ad2785d6e 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -22,6 +22,7 @@ class Member < ApplicationRecord STATE_AWAITING = 1 attr_accessor :raw_invite_token + attr_writer :blocking_refresh belongs_to :created_by, class_name: "User" belongs_to :user @@ -65,10 +66,10 @@ class Member < ApplicationRecord scope :in_hierarchy, ->(source) do groups = source.root_ancestor.self_and_descendants - group_members = Member.default_scoped.where(source: groups) + group_members = Member.default_scoped.where(source: groups).select(*Member.cached_column_list) projects = source.root_ancestor.all_projects - project_members = Member.default_scoped.where(source: projects) + project_members = Member.default_scoped.where(source: projects).select(*Member.cached_column_list) Member.default_scoped.from_union([ group_members, @@ -177,10 +178,14 @@ class Member < ApplicationRecord unscoped.from(distinct_members, :members) end - scope :order_name_asc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'ASC')) } - scope :order_name_desc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'DESC')) } - scope :order_recent_sign_in, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.last_sign_in_at', 'DESC')) } - scope :order_oldest_sign_in, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.last_sign_in_at', 'ASC')) } + scope :order_name_asc, -> { left_join_users.reorder(User.arel_table[:name].asc.nulls_last) } + scope :order_name_desc, -> { left_join_users.reorder(User.arel_table[:name].desc.nulls_last) } + scope :order_recent_sign_in, -> { left_join_users.reorder(User.arel_table[:last_sign_in_at].desc.nulls_last) } + scope :order_oldest_sign_in, -> { left_join_users.reorder(User.arel_table[:last_sign_in_at].asc.nulls_last) } + scope :order_recent_last_activity, -> { left_join_users.reorder(User.arel_table[:last_activity_on].desc.nulls_last) } + scope :order_oldest_last_activity, -> { left_join_users.reorder(User.arel_table[:last_activity_on].asc.nulls_first) } + scope :order_recent_created_user, -> { left_join_users.reorder(User.arel_table[:created_at].desc.nulls_last) } + scope :order_oldest_created_user, -> { left_join_users.reorder(User.arel_table[:created_at].asc.nulls_first) } scope :on_project_and_ancestors, ->(project) { where(source: [project] + project.ancestors) } @@ -197,7 +202,7 @@ class Member < ApplicationRecord after_save :log_invitation_token_cleanup after_commit on: [:create, :update], unless: :importing? do - refresh_member_authorized_projects(blocking: true) + refresh_member_authorized_projects(blocking: blocking_refresh) end after_commit on: [:destroy], unless: :importing? do @@ -232,6 +237,10 @@ class Member < ApplicationRecord when 'access_level_desc' then reorder(access_level: :desc) when 'recent_sign_in' then order_recent_sign_in when 'oldest_sign_in' then order_oldest_sign_in + when 'recent_created_user' then order_recent_created_user + when 'oldest_created_user' then order_oldest_created_user + when 'recent_last_activity' then order_recent_last_activity + when 'oldest_last_activity' then order_oldest_last_activity when 'last_joined' then order_created_desc when 'oldest_joined' then order_created_asc else @@ -505,6 +514,13 @@ class Member < ApplicationRecord error = StandardError.new("Invitation token is present but invite was already accepted!") Gitlab::ErrorTracking.track_exception(error, attributes.slice(%w["invite_accepted_at created_at source_type source_id user_id id"])) end + + def blocking_refresh + return true unless Feature.enabled?(:allow_non_blocking_member_refresh, default_enabled: :yaml) + return true if @blocking_refresh.nil? + + @blocking_refresh + end end Member.prepend_mod_with('Member') diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 3e19f294253..995c26d7221 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -82,10 +82,6 @@ class ProjectMember < Member source end - def owner? - project.owner == user - end - def notifiable_options { project: project } end @@ -132,7 +128,10 @@ class ProjectMember < Member end def post_create_hook - unless owner? + # The creator of a personal project gets added as a `ProjectMember` + # with `OWNER` access during creation of a personal project, + # but we do not want to trigger notifications to the same person who created the personal project. + unless project.personal_namespace_holder?(user) event_service.join_project(self.project, self.user) run_after_commit_or_now { notification_service.new_project_member(self) } end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 854325e1fcd..4c6ed399bf9 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -329,15 +329,15 @@ class MergeRequest < ApplicationRecord end scope :by_target_branch, ->(branch_name) { where(target_branch: branch_name) } scope :order_by_metric, ->(metric, direction) do - reverse_direction = { 'ASC' => 'DESC', 'DESC' => 'ASC' } - reversed_direction = reverse_direction[direction] || raise("Unknown sort direction was given: #{direction}") + column_expression = MergeRequest::Metrics.arel_table[metric] + column_expression_with_direction = direction == 'ASC' ? column_expression.asc : column_expression.desc order = Gitlab::Pagination::Keyset::Order.build([ Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( attribute_name: "merge_request_metrics_#{metric}", - column_expression: MergeRequest::Metrics.arel_table[metric], - order_expression: Gitlab::Database.nulls_last_order("merge_request_metrics.#{metric}", direction), - reversed_order_expression: Gitlab::Database.nulls_first_order("merge_request_metrics.#{metric}", reversed_direction), + column_expression: column_expression, + order_expression: column_expression_with_direction.nulls_last, + reversed_order_expression: column_expression_with_direction.reverse.nulls_first, order_direction: direction, nullable: :nulls_last, distinct: false, @@ -1409,9 +1409,7 @@ class MergeRequest < ApplicationRecord def has_ci? return false if has_no_commits? - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336891') do - !!(head_pipeline_id || all_pipelines.any? || source_project&.ci_integration) - end + !!(head_pipeline_id || all_pipelines.any? || source_project&.ci_integration) end def branch_missing? @@ -1444,7 +1442,7 @@ class MergeRequest < ApplicationRecord # This method is for looking for active environments which created via pipelines for merge requests. # Since deployments run on a merge request ref (e.g. `refs/merge-requests/:iid/head`), # we cannot look up environments with source branch name. - def environments + def legacy_environments return Environment.none unless actual_head_pipeline&.merge_request? build_for_actual_head_pipeline = Ci::Build.latest.where(pipeline: actual_head_pipeline) @@ -1458,6 +1456,14 @@ class MergeRequest < ApplicationRecord Environment.where(project: project, name: environments) end + def environments_in_head_pipeline(deployment_status: nil) + if ::Feature.enabled?(:fix_related_environments_for_merge_requests, target_project, default_enabled: :yaml) + actual_head_pipeline&.environments_in_self_and_descendants(deployment_status: deployment_status) || Environment.none + else + legacy_environments + end + end + def fetch_ref! target_project.repository.fetch_source_branch!(source_project.repository, source_branch, ref_path) end @@ -1904,9 +1910,7 @@ class MergeRequest < ApplicationRecord end def find_actual_head_pipeline - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336891') do - all_pipelines.for_sha_or_source_sha(diff_head_sha).first - end + all_pipelines.for_sha_or_source_sha(diff_head_sha).first end def etag_caching_enabled? diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 86da29dd27a..ff4fadb0f13 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -31,7 +31,7 @@ class Milestone < ApplicationRecord end scope :order_by_name_asc, -> { order(Arel::Nodes::Ascending.new(arel_table[:title].lower)) } - scope :reorder_by_due_date_asc, -> { reorder(Gitlab::Database.nulls_last_order('due_date', 'ASC')) } + scope :reorder_by_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } scope :with_api_entity_associations, -> { preload(project: [:project_feature, :route, namespace: :route]) } scope :order_by_dates_and_title, -> { order(due_date: :asc, start_date: :asc, title: :asc) } @@ -116,15 +116,15 @@ class Milestone < ApplicationRecord when 'due_date_asc' reorder_by_due_date_asc when 'due_date_desc' - reorder(Gitlab::Database.nulls_last_order('due_date', 'DESC')) + reorder(arel_table[:due_date].desc.nulls_last) when 'name_asc' reorder(Arel::Nodes::Ascending.new(arel_table[:title].lower)) when 'name_desc' reorder(Arel::Nodes::Descending.new(arel_table[:title].lower)) when 'start_date_asc' - reorder(Gitlab::Database.nulls_last_order('start_date', 'ASC')) + reorder(arel_table[:start_date].asc.nulls_last) when 'start_date_desc' - reorder(Gitlab::Database.nulls_last_order('start_date', 'DESC')) + reorder(arel_table[:start_date].desc.nulls_last) else order_by(method) end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index ffaeb2071f6..3b75b6d163a 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -15,6 +15,7 @@ class Namespace < ApplicationRecord include Namespaces::Traversal::Recursive include Namespaces::Traversal::Linear include EachBatch + include BlocksUnsafeSerialization # Temporary column used for back-filling project namespaces. # Remove it once the back-filling of all project namespaces is done. @@ -131,7 +132,7 @@ class Namespace < ApplicationRecord scope :user_namespaces, -> { where(type: Namespaces::UserNamespace.sti_name) } scope :without_project_namespaces, -> { where(Namespace.arel_table[:type].not_eq(Namespaces::ProjectNamespace.sti_name)) } - scope :sort_by_type, -> { order(Gitlab::Database.nulls_first_order(:type)) } + scope :sort_by_type, -> { order(arel_table[:type].asc.nulls_first) } scope :include_route, -> { includes(:route) } scope :by_parent, -> (parent) { where(parent_id: parent) } scope :filter_by_path, -> (query) { where('lower(path) = :query', query: query.downcase) } @@ -372,7 +373,7 @@ class Namespace < ApplicationRecord end # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. - def feature_available?(feature) + def feature_available?(feature, _user = nil) licensed_feature_available?(feature) end diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index ee04ec39b1e..96715863892 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -23,6 +23,14 @@ class Namespace::RootStorageStatistics < ApplicationRecord delegate :all_projects, to: :namespace + enum notification_level: { + storage_remaining: 100, + caution: 30, + warning: 15, + danger: 5, + exceeded: 0 + }, _prefix: true + def recalculate! update!(merged_attributes) end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 1963745cf4d..6320e0bc39d 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -49,6 +49,33 @@ module Namespaces before_commit :sync_traversal_ids, on: [:create], if: -> { sync_traversal_ids? } end + class_methods do + # This method looks into a list of namespaces trying to optimise a returned traversal_ids + # into a list of shortest prefixes, due to fact that the shortest prefixes include all childrens. + # Example: + # INPUT: [[4909902], [4909902,51065789], [4909902,51065793], [7135830], [15599674, 1], [15599674, 1, 3], [15599674, 2]] + # RESULT: [[4909902], [7135830], [15599674, 1], [15599674, 2]] + def shortest_traversal_ids_prefixes + raise ArgumentError, 'Feature not supported since the `:use_traversal_ids` is disabled' unless use_traversal_ids? + + prefixes = [] + + # The array needs to be sorted (O(nlogn)) to ensure shortest elements are always first + # This allows to do O(n) search of shortest prefixes + all_traversal_ids = all.order('namespaces.traversal_ids').pluck('namespaces.traversal_ids') + last_prefix = [nil] + + all_traversal_ids.each do |traversal_ids| + next if last_prefix == traversal_ids[0..(last_prefix.count - 1)] + + last_prefix = traversal_ids + prefixes << traversal_ids + end + + prefixes + end + end + def sync_traversal_ids? Feature.enabled?(:sync_traversal_ids, root_ancestor, default_enabled: :yaml) end diff --git a/app/models/note.rb b/app/models/note.rb index 4f2e7ebe2c5..3d2ac69a2ab 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -35,6 +35,8 @@ class Note < ApplicationRecord contact: :read_crm_contact }.freeze + NON_DIFF_NOTE_TYPES = ['Note', 'DiscussionNote', nil].freeze + # Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with notes. # See https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/10392/diffs#note_28719102 alias_attribute :last_edited_by, :updated_by @@ -97,6 +99,11 @@ class Note < ApplicationRecord validates :author, presence: true validates :discussion_id, presence: true, format: { with: /\A\h{40}\z/ } + validate :ensure_confidentiality_discussion_compliance + validate :ensure_noteable_can_have_confidential_note + validate :ensure_note_type_can_be_confidential + validate :ensure_confidentiality_not_changed, on: :update + validate unless: [:for_commit?, :importing?, :skip_project_check?] do |note| unless note.noteable.try(:project) == note.project errors.add(:project, 'does not match noteable project') @@ -121,6 +128,7 @@ class Note < ApplicationRecord scope :with_discussion_ids, ->(discussion_ids) { where(discussion_id: discussion_ids) } scope :with_suggestions, -> { joins(:suggestions) } scope :inc_author, -> { includes(:author) } + scope :inc_note_diff_file, -> { includes(:note_diff_file) } scope :with_api_entity_associations, -> { preload(:note_diff_file, :author) } scope :inc_relations_for_view, -> do includes({ project: :group }, { author: :status }, :updated_by, :resolved_by, :award_emoji, @@ -140,7 +148,7 @@ class Note < ApplicationRecord scope :diff_notes, -> { where(type: %w(LegacyDiffNote DiffNote)) } scope :new_diff_notes, -> { where(type: 'DiffNote') } - scope :non_diff_notes, -> { where(type: ['Note', 'DiscussionNote', nil]) } + scope :non_diff_notes, -> { where(type: NON_DIFF_NOTE_TYPES) } scope :with_associations, -> do # FYI noteable cannot be loaded for LegacyDiffNote for commits @@ -457,7 +465,7 @@ class Note < ApplicationRecord # and all its notes and if we don't care about the discussion's resolvability status. def discussion strong_memoize(:discussion) do - full_discussion = self.noteable.notes.find_discussion(self.discussion_id) if part_of_discussion? + full_discussion = self.noteable.notes.find_discussion(self.discussion_id) if self.noteable && part_of_discussion? full_discussion || to_discussion end end @@ -501,7 +509,15 @@ class Note < ApplicationRecord # Instead of calling touch which is throttled via ThrottledTouch concern, # we bump the updated_at column directly. This also prevents executing # after_commit callbacks that we don't need. - update_column(:updated_at, Time.current) + attributes_to_update = { updated_at: Time.current } + + # Notes that were edited before the `last_edited_at` column was added, fall back to `updated_at` for the edit time. + # We copy this over to the correct column so we don't erroneously change the edit timestamp. + if updated_by_id.present? && read_attribute(:last_edited_at).blank? + attributes_to_update[:last_edited_at] = updated_at + end + + update_columns(attributes_to_update) end def expire_etag_cache @@ -717,6 +733,42 @@ class Note < ApplicationRecord def noteable_label_url_method for_merge_request? ? :project_merge_requests_url : :project_issues_url end + + def ensure_confidentiality_not_changed + return unless will_save_change_to_attribute?(:confidential) + return unless attribute_change_to_be_saved(:confidential).include?(true) + + errors.add(:confidential, _('can not be changed for existing notes')) + end + + def ensure_confidentiality_discussion_compliance + return if start_of_discussion? + + if discussion.first_note.confidential? != confidential? + errors.add(:confidential, _('reply should have same confidentiality as top-level note')) + end + + ensure + clear_memoization(:discussion) + end + + def ensure_noteable_can_have_confidential_note + return unless confidential? + return if noteable_can_have_confidential_note? + + errors.add(:confidential, _('can not be set for this resource')) + end + + def ensure_note_type_can_be_confidential + return unless confidential? + return if NON_DIFF_NOTE_TYPES.include?(type) + + errors.add(:confidential, _('can not be set for this type of note')) + end + + def noteable_can_have_confidential_note? + for_issue? + end end Note.prepend_mod_with('Note') diff --git a/app/models/onboarding_progress.rb b/app/models/onboarding_progress.rb index 58b7848f7e2..e5851c5cfc5 100644 --- a/app/models/onboarding_progress.rb +++ b/app/models/onboarding_progress.rb @@ -27,7 +27,8 @@ class OnboardingProgress < ApplicationRecord :secure_secret_detection_run, :secure_coverage_fuzzing_run, :secure_api_fuzzing_run, - :secure_cluster_image_scanning_run + :secure_cluster_image_scanning_run, + :license_scanning_run ].freeze scope :incomplete_actions, -> (actions) do diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index c76473c9438..7744e578df5 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -228,8 +228,8 @@ class Packages::Package < ApplicationRecord def self.keyset_pagination_order(join_class:, column_name:, direction: :asc) join_table = join_class.table_name - asc_order_expression = Gitlab::Database.nulls_last_order("#{join_table}.#{column_name}", :asc) - desc_order_expression = Gitlab::Database.nulls_first_order("#{join_table}.#{column_name}", :desc) + asc_order_expression = join_class.arel_table[column_name].asc.nulls_last + desc_order_expression = join_class.arel_table[column_name].desc.nulls_first order_direction = direction == :asc ? asc_order_expression : desc_order_expression reverse_order_direction = direction == :asc ? desc_order_expression : asc_order_expression arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index ad8140ac684..b49e04f481c 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -34,7 +34,7 @@ class Packages::PackageFile < ApplicationRecord validates :file, presence: true validates :file_name, presence: true - validates :file_name, uniqueness: { scope: :package }, if: -> { package&.pypi? } + validates :file_name, uniqueness: { scope: :package }, if: -> { !pending_destruction? && package&.pypi? } scope :recent, -> { order(id: :desc) } scope :limit_recent, ->(limit) { recent.limit(limit) } diff --git a/app/models/preloaders/group_root_ancestor_preloader.rb b/app/models/preloaders/group_root_ancestor_preloader.rb new file mode 100644 index 00000000000..3ca713d9635 --- /dev/null +++ b/app/models/preloaders/group_root_ancestor_preloader.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Preloaders + class GroupRootAncestorPreloader + def initialize(groups, root_ancestor_preloads = []) + @groups = groups + @root_ancestor_preloads = root_ancestor_preloads + end + + def execute + return unless ::Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + + # type == 'Group' condition located on subquery to prevent a filter in the query + root_query = Namespace.joins("INNER JOIN (#{join_sql}) as root_query ON root_query.root_id = namespaces.id") + .select('namespaces.*, root_query.id as source_id') + + root_query = root_query.preload(*@root_ancestor_preloads) if @root_ancestor_preloads.any? + + root_ancestors_by_id = root_query.group_by(&:source_id) + + @groups.each do |group| + group.root_ancestor = root_ancestors_by_id[group.id].first + end + end + + private + + def join_sql + Group.select('id, traversal_ids[1] as root_id').where(id: @groups.map(&:id)).to_sql + end + end +end diff --git a/app/models/programming_language.rb b/app/models/programming_language.rb index 375fbe9b5a9..06e3034e56a 100644 --- a/app/models/programming_language.rb +++ b/app/models/programming_language.rb @@ -4,9 +4,10 @@ class ProgrammingLanguage < ApplicationRecord validates :name, presence: true validates :color, allow_blank: false, color: true - # Returns all programming languages which match the given name (case + # Returns all programming languages which match any of the given names (case # insensitively). - scope :with_name_case_insensitive, ->(name) do - where(arel_table[:name].matches(sanitize_sql_like(name))) + scope :with_name_case_insensitive, ->(*names) do + sanitized_names = names.map(&method(:sanitize_sql_like)) + where(arel_table[:name].matches_any(sanitized_names)) end end diff --git a/app/models/project.rb b/app/models/project.rb index 155ebe88d33..f7182d1645c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -37,6 +37,7 @@ class Project < ApplicationRecord include EachBatch include GitlabRoutingHelper include BulkMemberAccessLoad + include BulkUsersByEmailLoad include RunnerTokenExpirationInterval include BlocksUnsafeSerialization @@ -382,7 +383,7 @@ class Project < ApplicationRecord has_many :source_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :project_id has_many :import_failures, inverse_of: :project - has_many :jira_imports, -> { order 'jira_imports.created_at' }, class_name: 'JiraImportState', inverse_of: :project + has_many :jira_imports, -> { order(JiraImportState.arel_table[:created_at].asc) }, class_name: 'JiraImportState', inverse_of: :project has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult' has_many :ci_feature_usages, class_name: 'Projects::CiFeatureUsage' @@ -545,8 +546,8 @@ class Project < ApplicationRecord .or(arel_table[:storage_version].eq(nil))) end - # last_activity_at is throttled every minute, but last_repository_updated_at is updated with every push - scope :sorted_by_activity, -> { reorder(Arel.sql("GREATEST(COALESCE(last_activity_at, '1970-01-01'), COALESCE(last_repository_updated_at, '1970-01-01')) DESC")) } + scope :sorted_by_updated_asc, -> { reorder(self.arel_table['updated_at'].asc) } + scope :sorted_by_updated_desc, -> { reorder(self.arel_table['updated_at'].desc) } scope :sorted_by_stars_desc, -> { reorder(self.arel_table['star_count'].desc) } scope :sorted_by_stars_asc, -> { reorder(self.arel_table['star_count'].asc) } # Sometimes queries (e.g. using CTEs) require explicit disambiguation with table name @@ -655,7 +656,9 @@ class Project < ApplicationRecord preload(:project_feature, :route, namespace: [:route, :owner]) } + scope :created_by, -> (user) { where(creator: user) } scope :imported_from, -> (type) { where(import_type: type) } + scope :imported, -> { where.not(import_type: nil) } scope :with_tracing_enabled, -> { joins(:tracing_setting) } scope :with_enabled_error_tracking, -> { joins(:error_tracking_setting).where(project_error_tracking_settings: { enabled: true }) } @@ -780,9 +783,9 @@ class Project < ApplicationRecord # pass a string to avoid AR adding the table name reorder('project_statistics.storage_size DESC, projects.id DESC') when 'latest_activity_desc' - reorder(self.arel_table['last_activity_at'].desc) + sorted_by_updated_desc when 'latest_activity_asc' - reorder(self.arel_table['last_activity_at'].asc) + sorted_by_updated_asc when 'stars_desc' sorted_by_stars_desc when 'stars_asc' @@ -896,6 +899,18 @@ class Project < ApplicationRecord association(:namespace).loaded? end + def personal_namespace_holder?(user) + return false unless personal? + return false unless user + + # We do not want to use a check like `project.team.owner?(user)` + # here because that would depend upon the state of the `project_authorizations` cache, + # and also perform the check across multiple `owners` of the project, but our intention + # is to check if the user is the "holder" of the personal namespace, so need to make this + # check against only a single user (ie, namespace.owner). + namespace.owner == user + end + def project_setting super.presence || build_project_setting end @@ -1048,6 +1063,17 @@ class Project < ApplicationRecord end end + def container_repositories_size + strong_memoize(:container_repositories_size) do + next unless Gitlab.com? + next 0 if container_repositories.empty? + next unless container_repositories.all_migrated? + next unless ContainerRegistry::GitlabApiClient.supports_gitlab_api? + + ContainerRegistry::GitlabApiClient.deduplicated_size(full_path) + end + end + def has_container_registry_tags? return @images if defined?(@images) @@ -1401,7 +1427,7 @@ class Project < ApplicationRecord end def last_activity_date - [last_activity_at, last_repository_updated_at, updated_at].compact.max + updated_at end def project_id @@ -1469,7 +1495,7 @@ class Project < ApplicationRecord end def find_or_initialize_integration(name) - return if disabled_integrations.include?(name) + return if disabled_integrations.include?(name) || Integration.available_integration_names.exclude?(name) find_integration(integrations, name) || build_from_instance(name) || build_integration(name) end @@ -1920,6 +1946,10 @@ class Project < ApplicationRecord Gitlab.config.pages.enabled end + def pages_show_onboarding? + !(pages_metadatum&.onboarding_complete || pages_metadatum&.deployed) + end + def remove_private_deploy_keys exclude_keys_linked_to_other_projects = <<-SQL NOT EXISTS ( @@ -1935,6 +1965,10 @@ class Project < ApplicationRecord .delete_all end + def mark_pages_onboarding_complete + ensure_pages_metadatum.update!(onboarding_complete: true) + end + def mark_pages_as_deployed ensure_pages_metadatum.update!(deployed: true) end @@ -1974,13 +2008,15 @@ class Project < ApplicationRecord ProjectCacheWorker.perform_async(self.id, [], [:repository_size]) AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(id) + enqueue_record_project_target_platforms + # The import assigns iid values on its own, e.g. by re-using GitHub ids. # Flush existing InternalId records for this project for consistency reasons. # Those records are going to be recreated with the next normal creation # of a model instance (e.g. an Issue). InternalId.flush_records!(project: self) - import_state.finish + import_state&.finish update_project_counter_caches after_create_default_branch join_pool_repository @@ -2829,6 +2865,22 @@ class Project < ApplicationRecord pending_delete? || hidden? end + def work_items_feature_flag_enabled? + group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self, default_enabled: :yaml) + end + + def enqueue_record_project_target_platforms + return unless Gitlab.com? + return unless Feature.enabled?(:record_projects_target_platforms, self, default_enabled: :yaml) + + Projects::RecordTargetPlatformsWorker.perform_async(id) + end + + def inactive? + (statistics || build_statistics).storage_size > ::Gitlab::CurrentSettings.inactive_projects_min_size_mb.megabytes && + last_activity_at < ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months.months.ago + end + private # overridden in EE diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 0d3e50837ab..33783d31355 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -3,6 +3,7 @@ class ProjectFeature < ApplicationRecord include Featurable extend Gitlab::ConfigHelper + extend ::Gitlab::Utils::Override # When updating this array, make sure to update rubocop/cop/gitlab/feature_available_usage.rb as well. FEATURES = %i[ @@ -155,31 +156,14 @@ class ProjectFeature < ApplicationRecord %i(merge_requests_access_level builds_access_level).each(&validator) end - def get_permission(user, feature) - case access_level(feature) - when DISABLED - false - when PRIVATE - team_access?(user, feature) - when ENABLED - true - when PUBLIC - true - else - true - end + def feature_validation_exclusion + %i(pages) end - def team_access?(user, feature) - return unless user - return true if user.can_read_all_resources? - + override :resource_member? + def resource_member?(user, feature) project.team.member?(user, ProjectFeature.required_minimum_access_level(feature)) end - - def feature_validation_exclusion - %i(pages) - end end ProjectFeature.prepend_mod_with('ProjectFeature') diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index 8394ebe1df4..2ba3c74df5b 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -18,6 +18,7 @@ class ProjectGroupLink < ApplicationRecord scope :in_group, -> (group_ids) { where(group_id: group_ids) } alias_method :shared_with_group, :group + alias_method :shared_from, :project def self.access_options Gitlab::Access.options diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index 0f04eb7d4af..fabbd5b49cb 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -6,6 +6,8 @@ class ProjectImportState < ApplicationRecord self.table_name = "project_mirror_data" + after_commit :expire_etag_cache + belongs_to :project, inverse_of: :import_state validates :project, presence: true @@ -58,9 +60,7 @@ class ProjectImportState < ApplicationRecord end after_transition any => :failed do |state, _| - if Feature.enabled?(:remove_import_data_on_failure, state.project, default_enabled: :yaml) - state.project.remove_import_data - end + state.project.remove_import_data end after_transition started: :finished do |state, _| @@ -78,6 +78,23 @@ class ProjectImportState < ApplicationRecord end end + def expire_etag_cache + if realtime_changes_path + Gitlab::EtagCaching::Store.new.tap do |store| + store.touch(realtime_changes_path) + rescue Gitlab::EtagCaching::Store::InvalidKeyError + # no-op: not every realtime changes endpoint is using etag caching + end + end + end + + def realtime_changes_path + Gitlab::Routing.url_helpers.polymorphic_path([:realtime_changes_import, project.import_type.to_sym], format: :json) + rescue NoMethodError + # polymorphic_path throws NoMethodError when no such path exists + nil + end + def relation_hard_failures(limit:) project.import_failures.hard_failures_by_correlation_id(correlation_id).limit(limit) end diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index ae3d7038a88..6cd6eee2616 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -1,9 +1,7 @@ # frozen_string_literal: true class ProjectSetting < ApplicationRecord - include IgnorableColumns - - ignore_column :show_diff_preview_in_email, remove_with: '14.10', remove_after: '2022-03-22' + ALLOWED_TARGET_PLATFORMS = %w(ios osx tvos watchos).freeze belongs_to :project, inverse_of: :project_setting @@ -18,6 +16,9 @@ class ProjectSetting < ApplicationRecord validates :merge_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH } validates :squash_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH } + validates :target_platforms, inclusion: { in: ALLOWED_TARGET_PLATFORMS } + + validate :validates_mr_default_target_self default_value_for(:legacy_open_source_license_available) do Feature.enabled?(:legacy_open_source_license_available, default_enabled: :yaml, type: :ops) @@ -31,7 +32,9 @@ class ProjectSetting < ApplicationRecord %w[always never].include?(squash_option) end - validate :validates_mr_default_target_self + def target_platforms=(val) + super(val&.map(&:to_s)&.sort) + end private diff --git a/app/models/projects/build_artifacts_size_refresh.rb b/app/models/projects/build_artifacts_size_refresh.rb index afb67b79f0d..959f486a50a 100644 --- a/app/models/projects/build_artifacts_size_refresh.rb +++ b/app/models/projects/build_artifacts_size_refresh.rb @@ -4,7 +4,7 @@ module Projects class BuildArtifactsSizeRefresh < ApplicationRecord include BulkInsertSafe - STALE_WINDOW = 3.days + STALE_WINDOW = 2.hours self.table_name = 'project_build_artifacts_size_refreshes' diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb index b42b03f0618..9214a23e259 100644 --- a/app/models/projects/topic.rb +++ b/app/models/projects/topic.rb @@ -23,6 +23,10 @@ module Projects end class << self + def find_by_name_case_insensitive(name) + find_by('LOWER(name) = ?', name.downcase) + end + def search(query) fuzzy_search(query, [:name]) end diff --git a/app/models/repository.rb b/app/models/repository.rb index 346478b6689..dc0b5b54fb0 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -2,6 +2,12 @@ require 'securerandom' +# Explicitly require licensee/license file in order to use Licensee::InvalidLicense class defined in +# https://github.com/licensee/licensee/blob/v9.14.1/lib/licensee/license.rb#L6 +# The problem is that nested classes are not automatically preloaded which may lead to +# uninitialized constant exception being raised: https://gitlab.com/gitlab-org/gitlab/-/issues/356658 +require 'licensee/license' + class Repository REF_MERGE_REQUEST = 'merge-requests' REF_KEEP_AROUND = 'keep-around' @@ -789,6 +795,12 @@ class Repository def create_file(user, path, content, **options) options[:actions] = [{ action: :create, file_path: path, content: content }] + execute_filemode = options.delete(:execute_filemode) + + unless execute_filemode.nil? + options[:actions].push({ action: :chmod, file_path: path, execute_filemode: execute_filemode }) + end + multi_action(user, **options) end @@ -798,6 +810,12 @@ class Repository options[:actions] = [{ action: action, file_path: path, previous_path: previous_path, content: content }] + execute_filemode = options.delete(:execute_filemode) + + unless execute_filemode.nil? + options[:actions].push({ action: :chmod, file_path: path, execute_filemode: execute_filemode }) + end + multi_action(user, **options) end @@ -941,6 +959,10 @@ class Repository end end + def clone_as_mirror(url, http_authorization_header: "") + import_repository(url, http_authorization_header: http_authorization_header, mirror: true) + end + def fetch_as_mirror(url, forced: false, refmap: :all_refs, prune: true, http_authorization_header: "") fetch_remote(url, refmap: refmap, forced: forced, prune: prune, http_authorization_header: http_authorization_header) end diff --git a/app/models/repository_language.rb b/app/models/repository_language.rb index 2816aa4cc5b..60aaa1f932a 100644 --- a/app/models/repository_language.rb +++ b/app/models/repository_language.rb @@ -8,8 +8,8 @@ class RepositoryLanguage < ApplicationRecord default_scope { includes(:programming_language) } # rubocop:disable Cop/DefaultScope - scope :with_programming_language, ->(name) do - joins(:programming_language).merge(ProgrammingLanguage.with_name_case_insensitive(name)) + scope :with_programming_language, ->(*names) do + joins(:programming_language).merge(ProgrammingLanguage.with_name_case_insensitive(*names)) end validates :project, presence: true diff --git a/app/models/review.rb b/app/models/review.rb index 5a30e2963c8..c621da3b03c 100644 --- a/app/models/review.rb +++ b/app/models/review.rb @@ -14,6 +14,10 @@ class Review < ApplicationRecord participant :author + def discussion_ids + notes.select(:discussion_id) + end + def all_references(current_user = nil, extractor: nil) ext = super diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 38aaeff5c9a..cf4b83d44c2 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -40,6 +40,7 @@ class Snippet < ApplicationRecord belongs_to :author, class_name: 'User' belongs_to :project + alias_method :resource_parent, :project has_many :notes, as: :noteable, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :user_mentions, class_name: "SnippetUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/suggestion.rb b/app/models/suggestion.rb index f1ca5c23997..ca2ad8bf88c 100644 --- a/app/models/suggestion.rb +++ b/app/models/suggestion.rb @@ -16,10 +16,14 @@ class Suggestion < ApplicationRecord note.latest_diff_file end - def project + def source_project noteable.source_project end + def target_project + noteable.target_project + end + def branch noteable.source_branch end diff --git a/app/models/todo.rb b/app/models/todo.rb index eb5d9965955..45ab770a0f6 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -148,10 +148,10 @@ class Todo < ApplicationRecord target_type_column: "todos.target_type", target_column: "todos.target_id", project_column: "todos.project_id" - ).to_sql + ).arel.as('highest_priority') - select("#{table_name}.*, (#{highest_priority}) AS highest_priority") - .order(Gitlab::Database.nulls_last_order('highest_priority', 'ASC')) + select(arel_table[Arel.star], highest_priority) + .order(Arel.sql('highest_priority').asc.nulls_last) .order('todos.created_at') end diff --git a/app/models/user.rb b/app/models/user.rb index bc02f0ba55e..26d47de4f00 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -21,6 +21,7 @@ class User < ApplicationRecord include OptionallySearch include FromUnion include BatchDestroyDependentAssociations + include BatchNullifyDependentAssociations include HasUniqueInternalUsers include IgnorableColumns include UpdateHighestRole @@ -37,6 +38,9 @@ class User < ApplicationRecord COUNT_CACHE_VALIDITY_PERIOD = 24.hours + OTP_SECRET_LENGTH = 32 + OTP_SECRET_TTL = 2.minutes + MAX_USERNAME_LENGTH = 255 MIN_USERNAME_LENGTH = 2 @@ -46,6 +50,8 @@ class User < ApplicationRecord :public_email ].freeze + FORBIDDEN_SEARCH_STATES = %w(blocked banned ldap_blocked).freeze + add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } add_authentication_token_field :feed_token add_authentication_token_field :static_object_token, encrypted: :optional @@ -184,6 +190,8 @@ class User < ApplicationRecord has_many :snippets, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :issues, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent + has_many :updated_issues, class_name: 'Issue', dependent: :nullify, foreign_key: :updated_by_id # rubocop:disable Cop/ActiveRecordDependent + has_many :closed_issues, class_name: 'Issue', dependent: :nullify, foreign_key: :closed_by_id # rubocop:disable Cop/ActiveRecordDependent has_many :merge_requests, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :events, dependent: :delete_all, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :releases, dependent: :nullify, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent @@ -277,7 +285,7 @@ class User < ApplicationRecord after_update :username_changed_hook, if: :saved_change_to_username? after_destroy :post_destroy_hook after_destroy :remove_key_cache - after_save if: -> { saved_change_to_email? && confirmed? } do + after_save if: -> { (saved_change_to_email? || saved_change_to_confirmed_at?) && confirmed? } do email_to_confirm = self.emails.find_by(email: self.email) if email_to_confirm.present? @@ -322,6 +330,8 @@ class User < ApplicationRecord :setup_for_company, :setup_for_company=, :render_whitespace_in_code, :render_whitespace_in_code=, :markdown_surround_selection, :markdown_surround_selection=, + :diffs_deletion_color, :diffs_deletion_color=, + :diffs_addition_color, :diffs_addition_color=, to: :user_preference delegate :path, to: :namespace, allow_nil: true, prefix: true @@ -460,15 +470,16 @@ class User < ApplicationRecord .where('keys.user_id = users.id') .expiring_soon_and_not_notified) end - scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } - scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } - scope :order_recent_last_activity, -> { reorder(Gitlab::Database.nulls_last_order('last_activity_on', 'DESC')) } - scope :order_oldest_last_activity, -> { reorder(Gitlab::Database.nulls_first_order('last_activity_on', 'ASC')) } + scope :order_recent_sign_in, -> { reorder(arel_table[:current_sign_in_at].desc.nulls_last) } + scope :order_oldest_sign_in, -> { reorder(arel_table[:current_sign_in_at].asc.nulls_last) } + scope :order_recent_last_activity, -> { reorder(arel_table[:last_activity_on].desc.nulls_last) } + scope :order_oldest_last_activity, -> { reorder(arel_table[:last_activity_on].asc.nulls_first) } scope :by_id_and_login, ->(id, login) { where(id: id).where('username = LOWER(:login) OR email = LOWER(:login)', login: login) } scope :dormant, -> { with_state(:active).human_or_service_user.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } scope :with_no_activity, -> { with_state(:active).human_or_service_user.where(last_activity_on: nil) } scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } scope :by_ids_or_usernames, -> (ids, usernames) { where(username: usernames).or(where(id: ids)) } + scope :without_forbidden_states, -> { where.not(state: FORBIDDEN_SEARCH_STATES) } strip_attributes! :name @@ -660,9 +671,9 @@ class User < ApplicationRecord order = <<~SQL CASE - WHEN users.name = :query THEN 0 - WHEN users.username = :query THEN 1 - WHEN users.public_email = :query THEN 2 + WHEN LOWER(users.name) = :query THEN 0 + WHEN LOWER(users.username) = :query THEN 1 + WHEN LOWER(users.public_email) = :query THEN 2 ELSE 3 END SQL @@ -949,6 +960,21 @@ class User < ApplicationRecord (webauthn_registrations.loaded? && webauthn_registrations.any?) || (!webauthn_registrations.loaded? && webauthn_registrations.exists?) end + def needs_new_otp_secret? + !two_factor_enabled? && otp_secret_expired? + end + + def otp_secret_expired? + return true unless otp_secret_expires_at + + otp_secret_expires_at < Time.current + end + + def update_otp_secret! + self.otp_secret = User.generate_otp_secret(OTP_SECRET_LENGTH) + self.otp_secret_expires_at = Time.current + OTP_SECRET_TTL + end + def namespace_move_dir_allowed if namespace&.any_project_has_container_registry_tags? errors.add(:username, _('cannot be changed if a personal project has container registry tags.')) @@ -1709,8 +1735,12 @@ class User < ApplicationRecord end def attention_requested_open_merge_requests_count(force: false) - Rails.cache.fetch(attention_request_cache_key, force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do + if Feature.enabled?(:uncached_mr_attention_requests_count, self, default_enabled: :yaml) MergeRequestsFinder.new(self, attention: self.username, state: 'opened', non_archived: true).execute.count + else + Rails.cache.fetch(attention_request_cache_key, force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do + MergeRequestsFinder.new(self, attention: self.username, state: 'opened', non_archived: true).execute.count + end end end @@ -2121,8 +2151,8 @@ class User < ApplicationRecord def authorized_groups_without_shared_membership Group.from_union([ - groups.select(Namespace.arel_table[Arel.star]), - authorized_projects.joins(:namespace).select(Namespace.arel_table[Arel.star]) + groups.select(*Namespace.cached_column_list), + authorized_projects.joins(:namespace).select(*Namespace.cached_column_list) ]) end @@ -2237,33 +2267,66 @@ class User < ApplicationRecord end def ci_owned_project_runners_from_project_members - Ci::RunnerProject - .select('ci_runners.*') - .joins(:runner) - .where(project: project_members.where('access_level >= ?', Gitlab::Access::MAINTAINER).pluck(:source_id)) + project_ids = project_members.where('access_level >= ?', Gitlab::Access::MAINTAINER).pluck(:source_id) + + Ci::Runner + .joins(:runner_projects) + .where(runner_projects: { project: project_ids }) end def ci_owned_project_runners_from_group_members - Ci::RunnerProject - .select('ci_runners.*') - .joins(:runner) - .joins('JOIN ci_project_mirrors ON ci_project_mirrors.project_id = ci_runner_projects.project_id') - .joins('JOIN ci_namespace_mirrors ON ci_namespace_mirrors.namespace_id = ci_project_mirrors.namespace_id') - .merge(ci_namespace_mirrors_for_group_members(Gitlab::Access::MAINTAINER)) + cte_namespace_ids = Gitlab::SQL::CTE.new( + :cte_namespace_ids, + ci_namespace_mirrors_for_group_members(Gitlab::Access::MAINTAINER).select(:namespace_id) + ) + + cte_project_ids = Gitlab::SQL::CTE.new( + :cte_project_ids, + Ci::ProjectMirror + .select(:project_id) + .where('ci_project_mirrors.namespace_id IN (SELECT namespace_id FROM cte_namespace_ids)') + ) + + Ci::Runner + .with(cte_namespace_ids.to_arel) + .with(cte_project_ids.to_arel) + .joins(:runner_projects) + .where('ci_runner_projects.project_id IN (SELECT project_id FROM cte_project_ids)') end def ci_owned_group_runners - Ci::RunnerNamespace - .select('ci_runners.*') - .joins(:runner) - .joins('JOIN ci_namespace_mirrors ON ci_namespace_mirrors.namespace_id = ci_runner_namespaces.namespace_id') - .merge(ci_namespace_mirrors_for_group_members(Gitlab::Access::OWNER)) + cte_namespace_ids = Gitlab::SQL::CTE.new( + :cte_namespace_ids, + ci_namespace_mirrors_for_group_members(Gitlab::Access::OWNER).select(:namespace_id) + ) + + Ci::Runner + .with(cte_namespace_ids.to_arel) + .joins(:runner_namespaces) + .where('ci_runner_namespaces.namespace_id IN (SELECT namespace_id FROM cte_namespace_ids)') end def ci_namespace_mirrors_for_group_members(level) - Ci::NamespaceMirror.contains_any_of_namespaces( - group_members.where('access_level >= ?', level).pluck(:source_id) - ) + search_members = group_members.where('access_level >= ?', level) + + # This reduces searched prefixes to only shortest ones + # to avoid querying descendants since they are already covered + # by ancestor namespaces. If the FF is not available fallback to + # inefficient search: https://gitlab.com/gitlab-org/gitlab/-/issues/336436 + unless Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) + return Ci::NamespaceMirror.contains_any_of_namespaces(search_members.pluck(:source_id)) + end + + traversal_ids = Group.joins(:all_group_members) + .merge(search_members) + .shortest_traversal_ids_prefixes + + # Use efficient btree index to perform search + if Feature.enabled?(:ci_owned_runners_unnest_index, self, default_enabled: :yaml) + Ci::NamespaceMirror.contains_traversal_ids(traversal_ids) + else + Ci::NamespaceMirror.contains_any_of_namespaces(traversal_ids.map(&:last)) + end end end diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb index 727975c3f6e..62614a851c1 100644 --- a/app/models/user_custom_attribute.rb +++ b/app/models/user_custom_attribute.rb @@ -5,4 +5,14 @@ class UserCustomAttribute < ApplicationRecord validates :user_id, :key, :value, presence: true validates :key, uniqueness: { scope: [:user_id] } + + def self.upsert_custom_attributes(custom_attributes) + created_at = DateTime.now + updated_at = DateTime.now + + custom_attributes.map! do |custom_attribute| + custom_attribute.merge({ created_at: created_at, updated_at: updated_at }) + end + upsert_all(custom_attributes, unique_by: [:user_id, :key]) + end end diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index 7687430cfd1..9b4c0a2527a 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -19,6 +19,9 @@ class UserPreference < ApplicationRecord greater_than_or_equal_to: Gitlab::TabWidth::MIN, less_than_or_equal_to: Gitlab::TabWidth::MAX } + validates :diffs_deletion_color, :diffs_addition_color, + format: { with: ColorsHelper::HEX_COLOR_PATTERN }, + allow_blank: true ignore_columns :experience_level, remove_with: '14.10', remove_after: '2021-03-22' diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 0922323e12b..a91a3406b22 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -48,7 +48,8 @@ module Users storage_enforcement_banner_third_enforcement_threshold: 45, storage_enforcement_banner_fourth_enforcement_threshold: 46, attention_requests_top_nav: 47, - attention_requests_side_nav: 48 + attention_requests_side_nav: 48, + minute_limit_banner: 49 } validates :feature_name, diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index 839be8d2a48..373bc30889f 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -14,7 +14,9 @@ module Users storage_enforcement_banner_first_enforcement_threshold: 3, storage_enforcement_banner_second_enforcement_threshold: 4, storage_enforcement_banner_third_enforcement_threshold: 5, - storage_enforcement_banner_fourth_enforcement_threshold: 6 + storage_enforcement_banner_fourth_enforcement_threshold: 6, + preview_user_over_limit_free_plan_alert: 7, # EE-only + user_reached_limit_free_plan_alert: 8 # EE-only } validates :group, presence: true diff --git a/app/models/users/in_product_marketing_email.rb b/app/models/users/in_product_marketing_email.rb index 1f1eaacfe5c..f2f1d18339e 100644 --- a/app/models/users/in_product_marketing_email.rb +++ b/app/models/users/in_product_marketing_email.rb @@ -26,12 +26,17 @@ module Users invite_team: 8 }, _suffix: true + # Tracks we don't send emails for (e.g. unsuccessful experiment). These + # are kept since we already have DB records that use the enum value. + INACTIVE_TRACK_NAMES = %w(invite_team).freeze + ACTIVE_TRACKS = tracks.except(*INACTIVE_TRACK_NAMES) + scope :without_track_and_series, -> (track, series) do users = User.arel_table product_emails = arel_table join_condition = users[:id].eq(product_emails[:user_id]) - .and(product_emails[:track]).eq(tracks[track]) + .and(product_emails[:track]).eq(ACTIVE_TRACKS[track]) .and(product_emails[:series]).eq(series) arel_join = users.join(product_emails, Arel::Nodes::OuterJoin).on(join_condition) diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb index a5881e80e88..8bb598ee316 100644 --- a/app/models/vulnerability.rb +++ b/app/models/vulnerability.rb @@ -5,6 +5,8 @@ class Vulnerability < ApplicationRecord include EachBatch include IgnorableColumns + alias_attribute :vulnerability_id, :id + def self.link_reference_pattern nil end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index 622070abd88..b3f09b20463 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -10,12 +10,46 @@ class Wiki extend ActiveModel::Naming MARKUPS = { # rubocop:disable Style/MultilineIfModifier - 'Markdown' => :markdown, - 'RDoc' => :rdoc, - 'AsciiDoc' => :asciidoc, - 'Org' => :org + markdown: { + name: 'Markdown', + default_extension: :md, + created_by_user: true + }, + rdoc: { + name: 'RDoc', + default_extension: :rdoc, + created_by_user: true + }, + asciidoc: { + name: 'AsciiDoc', + default_extension: :asciidoc, + created_by_user: true + }, + org: { + name: 'Org', + default_extension: :org, + created_by_user: true + }, + textile: { + name: 'Textile', + default_extension: :textile + }, + creole: { + name: 'Creole', + default_extension: :creole + }, + rest: { + name: 'reStructuredText', + default_extension: :rst + }, + mediawiki: { + name: 'MediaWiki', + default_extension: :mediawiki + } }.freeze unless defined?(MARKUPS) + VALID_USER_MARKUPS = MARKUPS.select { |_, v| v[:created_by_user] }.freeze unless defined?(VALID_USER_MARKUPS) + CouldNotCreateWikiError = Class.new(StandardError) HOMEPAGE = 'home' @@ -184,12 +218,37 @@ class Wiki end def update_page(page, content:, title: nil, format: :markdown, message: nil) - commit = commit_details(:updated, message, page.title) + if Feature.enabled?(:gitaly_replace_wiki_update_page, container, default_enabled: :yaml) + with_valid_format(format) do |default_extension| + title = title.presence || Pathname(page.path).sub_ext('').to_s - wiki.update_page(page.path, title || page.name, format.to_sym, content, commit) - after_wiki_activity + # If the format is the same we keep the former extension. This check is for formats + # that can have more than one extension like Markdown (.md, .markdown) + # If we don't do this we will override the existing extension. + extension = page.format != format.to_sym ? default_extension : File.extname(page.path).downcase[1..] - true + capture_git_error(:updated) do + repository.update_file( + user, + sluggified_full_path(title, extension), + content, + previous_path: page.path, + **multi_commit_options(:updated, message, title)) + + after_wiki_activity + + true + end + end + else + commit = commit_details(:updated, message, page.title) + + wiki.update_page(page.path, title || page.name, format.to_sym, content, commit) + + after_wiki_activity + + true + end end def delete_page(page, message = nil) @@ -296,7 +355,7 @@ class Wiki git_user = Gitlab::Git::User.from_gitlab(user) { - branch_name: repository.root_ref, + branch_name: repository.root_ref || default_branch, message: commit_message, author_email: git_user.email, author_name: git_user.name @@ -321,6 +380,26 @@ class Wiki def default_message(action, title) "#{user.username} #{action} page: #{title}" end + + def with_valid_format(format, &block) + default_extension = Wiki::VALID_USER_MARKUPS.dig(format.to_sym, :default_extension).to_s + + if default_extension.blank? + @error_message = _('Invalid format selected') + + return false + end + + yield default_extension + end + + def sluggified_full_path(title, extension) + sluggified_title(title) + '.' + extension + end + + def sluggified_title(title) + Gitlab::EncodingHelper.encode_utf8_no_detect(title).tr(' ', '-') + end end Wiki.prepend_mod_with('Wiki') diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 803b9781ac4..647b4e787c6 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -185,7 +185,7 @@ class WikiPage # :content - The raw markup content. # :format - Optional symbol representing the # content format. Can be any type - # listed in the Wiki::MARKUPS + # listed in the Wiki::VALID_USER_MARKUPS # Hash. # :message - Optional commit message to set on # the new page. @@ -205,7 +205,7 @@ class WikiPage # attrs - Hash of attributes to be updated on the page. # :content - The raw markup content to replace the existing. # :format - Optional symbol representing the content format. - # See Wiki::MARKUPS Hash for available formats. + # See Wiki::VALID_USER_MARKUPS Hash for available formats. # :message - Optional commit message to set on the new version. # :last_commit_sha - Optional last commit sha to validate the page unchanged. # :title - The Title (optionally including dir) to replace existing title @@ -222,7 +222,7 @@ class WikiPage update_attributes(attrs) - if title.present? && title_changed? && wiki.find_page(title).present? + if title.present? && title_changed? && wiki.find_page(title, load_content: false).present? attributes[:title] = page.title raise PageRenameError, s_('WikiEdit|There is already a page with the same title in that path.') end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index 080513b28e9..e2d38dc9903 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -37,7 +37,7 @@ module WorkItems validates :icon_name, length: { maximum: 255 } scope :default, -> { where(namespace: nil) } - scope :order_by_name_asc, -> { order('LOWER(name)') } + scope :order_by_name_asc, -> { order(arel_table[:name].lower.asc) } scope :by_type, ->(base_type) { where(base_type: base_type) } def self.default_by_type(type) |