diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 20:02:30 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 20:02:30 +0000 |
| commit | 41fe97390ceddf945f3d967b8fdb3de4c66b7dea (patch) | |
| tree | 9c8d89a8624828992f06d892cd2f43818ff5dcc8 /app/models | |
| parent | 0804d2dc31052fb45a1efecedc8e06ce9bc32862 (diff) | |
| download | gitlab-ce-41fe97390ceddf945f3d967b8fdb3de4c66b7dea.tar.gz | |
Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42
Diffstat (limited to 'app/models')
120 files changed, 1637 insertions, 585 deletions
diff --git a/app/models/analytics/cycle_analytics/aggregation.rb b/app/models/analytics/cycle_analytics/aggregation.rb new file mode 100644 index 00000000000..44d2dc369f7 --- /dev/null +++ b/app/models/analytics/cycle_analytics/aggregation.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +class Analytics::CycleAnalytics::Aggregation < ApplicationRecord + include FromUnion + + belongs_to :group, optional: false + + validates :incremental_runtimes_in_seconds, :incremental_processed_records, :last_full_run_runtimes_in_seconds, :last_full_run_processed_records, presence: true, length: { maximum: 10 }, allow_blank: true + + scope :priority_order, -> (column_to_sort = :last_incremental_run_at) { order(arel_table[column_to_sort].asc.nulls_first) } + scope :enabled, -> { where('enabled IS TRUE') } + + def estimated_next_run_at + return unless enabled + return if last_incremental_run_at.nil? + + estimation = duration_until_the_next_aggregation_job + + average_aggregation_duration + + (last_incremental_run_at - earliest_last_run_at) + + estimation < 1 ? nil : estimation.from_now + end + + def self.safe_create_for_group(group) + top_level_group = group.root_ancestor + aggregation = find_by(group_id: top_level_group.id) + return aggregation if aggregation.present? + + insert({ group_id: top_level_group.id }, unique_by: :group_id) + find_by(group_id: top_level_group.id) + end + + private + + # The aggregation job is scheduled every 10 minutes: */10 * * * * + def duration_until_the_next_aggregation_job + (10 - (DateTime.current.minute % 10)).minutes.seconds + end + + def average_aggregation_duration + return 0.seconds if incremental_runtimes_in_seconds.empty? + + average = incremental_runtimes_in_seconds.sum.fdiv(incremental_runtimes_in_seconds.size) + average.seconds + end + + def earliest_last_run_at + max = self.class.select(:last_incremental_run_at) + .where(enabled: true) + .where.not(last_incremental_run_at: nil) + .priority_order + .limit(1) + .to_sql + + connection.select_value("(#{max})") + end + + def self.load_batch(last_run_at, column_to_query = :last_incremental_run_at, batch_size = 100) + last_run_at_not_set = Analytics::CycleAnalytics::Aggregation + .enabled + .where(column_to_query => nil) + .priority_order(column_to_query) + .limit(batch_size) + + last_run_at_before = Analytics::CycleAnalytics::Aggregation + .enabled + .where(arel_table[column_to_query].lt(last_run_at)) + .priority_order(column_to_query) + .limit(batch_size) + + Analytics::CycleAnalytics::Aggregation + .from_union([last_run_at_not_set, last_run_at_before], remove_order: false, remove_duplicates: false) + .limit(batch_size) + end +end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 06ff18ca409..198a3653cd3 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -5,6 +5,7 @@ class ApplicationRecord < ActiveRecord::Base include Transactions include LegacyBulkInsert include CrossDatabaseModification + include SensitiveSerializableHash self.abstract_class = true @@ -60,8 +61,10 @@ class ApplicationRecord < ActiveRecord::Base end # Start a new transaction with a shorter-than-usual statement timeout. This is - # currently one third of the default 15-second timeout - def self.with_fast_read_statement_timeout(timeout_ms = 5000) + # currently one third of the default 15-second timeout with a 500ms buffer + # to allow callers gracefully handling the errors to still complete within + # the 5s target duration of a low urgency request. + def self.with_fast_read_statement_timeout(timeout_ms = 4500) ::Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions connection.exec_query("SET LOCAL statement_timeout = #{timeout_ms}") @@ -99,6 +102,10 @@ class ApplicationRecord < ActiveRecord::Base where('EXISTS (?)', query.select(1)) end + def self.where_not_exists(query) + where('NOT EXISTS (?)', query.select(1)) + end + def self.declarative_enum(enum_mod) enum(enum_mod.key => enum_mod.values) end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 02fbf0f855e..c7aad7ff861 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -11,6 +11,7 @@ class ApplicationSetting < ApplicationRecord ignore_columns %i[elasticsearch_shards elasticsearch_replicas], remove_with: '14.4', remove_after: '2021-09-22' ignore_columns %i[static_objects_external_storage_auth_token], remove_with: '14.9', remove_after: '2022-03-22' ignore_column %i[max_package_files_for_package_destruction], remove_with: '14.9', remove_after: '2022-03-22' + ignore_column :user_email_lookup_limit, remove_with: '15.0', remove_after: '2022-04-18' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -362,6 +363,9 @@ class ApplicationSetting < ApplicationRecord :container_registry_expiration_policies_worker_capacity, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :container_registry_expiration_policies_caching, + inclusion: { in: [true, false], message: _('must be a boolean value') } + validates :container_registry_import_max_tags_count, :container_registry_import_max_retries, :container_registry_import_start_max_retries, @@ -516,9 +520,12 @@ class ApplicationSetting < ApplicationRecord validates :notes_create_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } - validates :user_email_lookup_limit, + validates :search_rate_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :search_rate_limit_unauthenticated, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :notes_create_limit_allowlist, length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, allow_nil: false @@ -650,7 +657,17 @@ class ApplicationSetting < ApplicationRecord users_count >= INSTANCE_REVIEW_MIN_USERS end + Recursion = Class.new(RuntimeError) + def self.create_from_defaults + # this is posssible if calls to create the record depend on application + # settings themselves. This was seen in the case of a feature flag called by + # `transaction` that ended up requiring application settings to determine metrics behavior. + # If something like that happens, we break the loop here, and let the caller decide how to manage it. + raise Recursion if Thread.current[:application_setting_create_from_defaults] + + Thread.current[:application_setting_create_from_defaults] = true + check_schema! transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions @@ -659,6 +676,8 @@ class ApplicationSetting < ApplicationRecord rescue ActiveRecord::RecordNotUnique # We already have an ApplicationSetting record, so just return it. current_without_cache + ensure + Thread.current[:application_setting_create_from_defaults] = nil end def self.find_or_create_without_cache diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 415f0b35f3a..42049713883 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -218,7 +218,9 @@ module ApplicationSettingImplementation valid_runner_registrars: VALID_RUNNER_REGISTRAR_TYPES, wiki_page_max_content_bytes: 50.megabytes, container_registry_delete_tags_service_timeout: 250, - container_registry_expiration_policies_worker_capacity: 0, + container_registry_expiration_policies_worker_capacity: 4, + container_registry_cleanup_tags_service_max_list_size: 200, + container_registry_expiration_policies_caching: true, container_registry_import_max_tags_count: 100, container_registry_import_max_retries: 3, container_registry_import_start_max_retries: 50, @@ -231,7 +233,8 @@ module ApplicationSettingImplementation rate_limiting_response_text: nil, whats_new_variant: 0, user_deactivation_emails_enabled: true, - user_email_lookup_limit: 60, + search_rate_limit: 30, + search_rate_limit_unauthenticated: 10, users_get_by_id_limit: 300, users_get_by_id_limit_allowlist: [] } @@ -402,7 +405,7 @@ module ApplicationSettingImplementation def normalized_repository_storage_weights strong_memoize(:normalized_repository_storage_weights) do repository_storages_weights = repository_storages_weighted.slice(*Gitlab.config.repositories.storages.keys) - weights_total = repository_storages_weights.values.reduce(:+) + weights_total = repository_storages_weights.values.sum repository_storages_weights.transform_values do |w| next w if weights_total == 0 diff --git a/app/models/blobs/notebook.rb b/app/models/blobs/notebook.rb new file mode 100644 index 00000000000..bdb438cccd9 --- /dev/null +++ b/app/models/blobs/notebook.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Blobs + class Notebook < ::Blob + attr_reader :data + + def initialize(blob, data) + super(blob.__getobj__, blob.container) + @data = data + end + end +end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 1ee5c081840..949902fbb77 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -4,12 +4,21 @@ class BroadcastMessage < ApplicationRecord include CacheMarkdownField include Sortable + ALLOWED_TARGET_ACCESS_LEVELS = [ + Gitlab::Access::GUEST, + Gitlab::Access::REPORTER, + Gitlab::Access::DEVELOPER, + Gitlab::Access::MAINTAINER, + Gitlab::Access::OWNER + ].freeze + cache_markdown_field :message, pipeline: :broadcast_message, whitelisted: true validates :message, presence: true validates :starts_at, presence: true validates :ends_at, presence: true validates :broadcast_type, presence: true + validates :target_access_levels, inclusion: { in: ALLOWED_TARGET_ACCESS_LEVELS } validates :color, allow_blank: true, color: true validates :font, allow_blank: true, color: true @@ -29,20 +38,20 @@ class BroadcastMessage < ApplicationRecord } class << self - def current_banner_messages(current_path = nil) - fetch_messages BANNER_CACHE_KEY, current_path do + def current_banner_messages(current_path: nil, user_access_level: nil) + fetch_messages BANNER_CACHE_KEY, current_path, user_access_level do current_and_future_messages.banner end end - def current_notification_messages(current_path = nil) - fetch_messages NOTIFICATION_CACHE_KEY, current_path do + def current_notification_messages(current_path: nil, user_access_level: nil) + fetch_messages NOTIFICATION_CACHE_KEY, current_path, user_access_level do current_and_future_messages.notification end end - def current(current_path = nil) - fetch_messages CACHE_KEY, current_path do + def current(current_path: nil, user_access_level: nil) + fetch_messages CACHE_KEY, current_path, user_access_level do current_and_future_messages end end @@ -53,7 +62,7 @@ class BroadcastMessage < ApplicationRecord def cache ::Gitlab::SafeRequestStore.fetch(:broadcast_message_json_cache) do - Gitlab::JsonCache.new(cache_key_with_version: false) + Gitlab::JsonCache.new end end @@ -63,7 +72,7 @@ class BroadcastMessage < ApplicationRecord private - def fetch_messages(cache_key, current_path) + def fetch_messages(cache_key, current_path, user_access_level) messages = cache.fetch(cache_key, as: BroadcastMessage, expires_in: cache_expires_in) do yield end @@ -74,7 +83,13 @@ class BroadcastMessage < ApplicationRecord # displaying we'll refresh the cache so we don't need to keep filtering. cache.expire(cache_key) if now_or_future != messages - now_or_future.select(&:now?).select { |message| message.matches_current_path(current_path) } + messages = now_or_future.select(&:now?) + messages = messages.select do |message| + message.matches_current_user_access_level?(user_access_level) + end + messages.select do |message| + message.matches_current_path(current_path) + end end end @@ -102,6 +117,13 @@ class BroadcastMessage < ApplicationRecord now? || future? end + def matches_current_user_access_level?(user_access_level) + return false if target_access_levels.present? && Feature.disabled?(:role_targeted_broadcast_messages, default_enabled: :yaml) + return true unless target_access_levels.present? + + target_access_levels.include? user_access_level + end + def matches_current_path(current_path) return false if current_path.blank? && target_path.present? return true if current_path.blank? || target_path.blank? diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 38b7da76306..a7e1384641c 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -20,6 +20,8 @@ class BulkImports::Entity < ApplicationRecord self.table_name = 'bulk_import_entities' + FailedError = Class.new(StandardError) + belongs_to :bulk_import, optional: false belongs_to :parent, class_name: 'BulkImports::Entity', optional: true diff --git a/app/models/bulk_imports/export_status.rb b/app/models/bulk_imports/export_status.rb index abf064adaae..cae6aad27da 100644 --- a/app/models/bulk_imports/export_status.rb +++ b/app/models/bulk_imports/export_status.rb @@ -30,7 +30,12 @@ module BulkImports def export_status strong_memoize(:export_status) do - fetch_export_status.find { |item| item['relation'] == relation } + status = fetch_export_status + + # Consider empty response as failed export + raise StandardError, 'Empty export status response' unless status&.present? + + status.find { |item| item['relation'] == relation } end rescue StandardError => e { 'status' => Export::FAILED, 'error' => e.message } diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 50bda64d537..2ff777bfc89 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -11,6 +11,11 @@ module Ci InvalidBridgeTypeError = Class.new(StandardError) InvalidTransitionError = Class.new(StandardError) + FORWARD_DEFAULTS = { + yaml_variables: true, + pipeline_variables: false + }.freeze + belongs_to :project belongs_to :trigger_request has_many :sourced_pipelines, class_name: "::Ci::Sources::Pipeline", @@ -199,12 +204,13 @@ module Ci end def downstream_variables - variables = scoped_variables.concat(pipeline.persisted_variables) - - variables.to_runner_variables.yield_self do |all_variables| - yaml_variables.to_a.map do |hash| - { key: hash[:key], value: ::ExpandVariables.expand(hash[:value], all_variables) } - end + if ::Feature.enabled?(:ci_trigger_forward_variables, project, default_enabled: :yaml) + calculate_downstream_variables + .reverse # variables priority + .uniq { |var| var[:key] } # only one variable key to pass + .reverse + else + legacy_downstream_variables end end @@ -250,6 +256,58 @@ module Ci } } end + + def legacy_downstream_variables + variables = scoped_variables.concat(pipeline.persisted_variables) + + variables.to_runner_variables.yield_self do |all_variables| + yaml_variables.to_a.map do |hash| + { key: hash[:key], value: ::ExpandVariables.expand(hash[:value], all_variables) } + end + end + end + + def calculate_downstream_variables + expand_variables = scoped_variables + .concat(pipeline.persisted_variables) + .to_runner_variables + + # The order of this list refers to the priority of the variables + downstream_yaml_variables(expand_variables) + + downstream_pipeline_variables(expand_variables) + end + + def downstream_yaml_variables(expand_variables) + return [] unless forward_yaml_variables? + + yaml_variables.to_a.map do |hash| + { key: hash[:key], value: ::ExpandVariables.expand(hash[:value], expand_variables) } + end + end + + def downstream_pipeline_variables(expand_variables) + return [] unless forward_pipeline_variables? + + pipeline.variables.to_a.map do |variable| + { key: variable.key, value: ::ExpandVariables.expand(variable.value, expand_variables) } + end + end + + def forward_yaml_variables? + strong_memoize(:forward_yaml_variables) do + result = options&.dig(:trigger, :forward, :yaml_variables) + + result.nil? ? FORWARD_DEFAULTS[:yaml_variables] : result + end + end + + def forward_pipeline_variables? + strong_memoize(:forward_pipeline_variables) do + result = options&.dig(:trigger, :forward, :pipeline_variables) + + result.nil? ? FORWARD_DEFAULTS[:pipeline_variables] : result + end + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index c4d1a2c740b..68ec196a9ee 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -10,6 +10,8 @@ module Ci include Presentable include Importable include Ci::HasRef + include HasDeploymentName + extend ::Gitlab::Utils::Override BuildArchivedError = Class.new(StandardError) @@ -35,6 +37,8 @@ module Ci DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' RUNNERS_STATUS_CACHE_EXPIRATION = 1.minute + DEPLOYMENT_NAMES = %w[deploy release rollout].freeze + has_one :deployment, as: :deployable, class_name: 'Deployment' has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id @@ -68,6 +72,7 @@ module Ci delegate :terminal_specification, to: :runner_session, allow_nil: true delegate :service_specification, to: :runner_session, allow_nil: true delegate :gitlab_deploy_token, to: :project + delegate :harbor_integration, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true ## @@ -579,6 +584,7 @@ module Ci .append(key: 'CI_REGISTRY_PASSWORD', value: token.to_s, public: false, masked: true) .append(key: 'CI_REPOSITORY_URL', value: repo_url.to_s, public: false) .concat(deploy_token_variables) + .concat(harbor_variables) end end @@ -615,6 +621,12 @@ module Ci end end + def harbor_variables + return [] unless harbor_integration.try(:activated?) + + Gitlab::Ci::Variables::Collection.new(harbor_integration.ci_variables) + end + def features { trace_sections: true, @@ -1123,6 +1135,10 @@ module Ci .include?(exit_code) end + def track_deployment_usage + Gitlab::Utils::UsageData.track_usage_event('ci_users_executing_deployment_job', user_id) if user_id.present? && count_user_deployment? + end + protected def run_status_commit_hooks! diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 165bee5c54d..0af5533613f 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -18,5 +18,6 @@ module Ci scope :unprotected, -> { where(protected: false) } scope :by_environment_scope, -> (environment_scope) { where(environment_scope: environment_scope) } + scope :for_groups, ->(group_ids) { where(group_id: group_ids) } end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index a1311b8555f..ae3ea7aa03f 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -25,6 +25,7 @@ module Ci }.freeze CONFIG_EXTENSION = '.gitlab-ci.yml' DEFAULT_CONFIG_PATH = CONFIG_EXTENSION + CANCELABLE_STATUSES = (Ci::HasStatus::CANCELABLE_STATUSES + ['manual']).freeze BridgeStatusError = Class.new(StandardError) @@ -421,9 +422,7 @@ module Ci sql = sql.where(ref: ref) if ref - sql.each_with_object({}) do |pipeline, hash| - hash[pipeline.sha] = pipeline - end + sql.index_by(&:sha) end def self.latest_successful_ids_per_project @@ -653,7 +652,7 @@ module Ci def coverage coverage_array = latest_statuses.map(&:coverage).compact if coverage_array.size >= 1 - coverage_array.reduce(:+) / coverage_array.size + coverage_array.sum / coverage_array.size end end @@ -1165,11 +1164,7 @@ module Ci end def merge_request? - if Feature.enabled?(:ci_pipeline_merge_request_presence_check, default_enabled: :yaml) - merge_request_id.present? && merge_request - else - merge_request_id.present? - end + merge_request_id.present? && merge_request.present? end def external_pull_request? diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index b915495ac38..96e5567e85e 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -66,6 +66,18 @@ module Ci project.actual_limits.limit_for(:ci_daily_pipeline_schedule_triggers) end + def ref_for_display + return unless ref.present? + + ref.gsub(%r{^refs/(heads|tags)/}, '') + end + + def for_tag? + return false unless ref.present? + + ref.start_with? 'refs/tags/' + end + private def worker_cron_expression diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 372df8cc264..4d119706a43 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -16,7 +16,7 @@ module Ci scope :with_needs, -> (names = nil) do needs = Ci::BuildNeed.scoped_build.select(1) needs = needs.where(name: names) if names - where('EXISTS (?)', needs).preload(:needs) + where('EXISTS (?)', needs) end scope :without_needs, -> (names = nil) do diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 11150e839a3..4228da279a4 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -59,7 +59,7 @@ module Ci AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze - AVAILABLE_STATUSES = %w[active paused online offline not_connected never_contacted stale].freeze # TODO: Remove in %15.0: active, paused, not_connected. Relevant issues: https://gitlab.com/gitlab-org/gitlab/-/issues/347303, https://gitlab.com/gitlab-org/gitlab/-/issues/347305, https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + AVAILABLE_STATUSES = %w[active paused online offline not_connected never_contacted stale].freeze # TODO: Remove in %15.0: not_connected. In %16.0: active, paused. Relevant issues: https://gitlab.com/gitlab-org/gitlab/-/issues/347303, https://gitlab.com/gitlab-org/gitlab/-/issues/347305, https://gitlab.com/gitlab-org/gitlab/-/issues/344648 AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze @@ -200,7 +200,7 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } - validates :maintenance_note, length: { maximum: 255 } + validates :maintenance_note, length: { maximum: 1024 } alias_attribute :maintenance_note, :maintainer_note @@ -329,9 +329,9 @@ module Ci end # DEPRECATED - # TODO Remove in %15.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + # TODO Remove in %16.0 in favor of `status` for REST calls def deprecated_rest_status - if contacted_at.nil? + if contacted_at.nil? # TODO Remove in %15.0, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 :not_connected elsif active? online? ? :online : :offline diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 56f632b6232..18f0093ea41 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -3,10 +3,14 @@ module Ci class SecureFile < Ci::ApplicationRecord include FileStoreMounter + include Limitable FILE_SIZE_LIMIT = 5.megabytes.freeze CHECKSUM_ALGORITHM = 'sha256' + self.limit_scope = :project + self.limit_name = 'project_ci_secure_files' + belongs_to :project, optional: false validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } diff --git a/app/models/concerns/blocks_json_serialization.rb b/app/models/concerns/blocks_json_serialization.rb deleted file mode 100644 index 18c00532d78..00000000000 --- a/app/models/concerns/blocks_json_serialization.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# Overrides `as_json` and `to_json` to raise an exception when called in order -# to prevent accidentally exposing attributes -# -# Not that would ever happen... but just in case. -module BlocksJsonSerialization - extend ActiveSupport::Concern - - JsonSerializationError = Class.new(StandardError) - - def to_json(*) - raise JsonSerializationError, - "JSON serialization has been disabled on #{self.class.name}" - end - - alias_method :as_json, :to_json -end diff --git a/app/models/concerns/blocks_unsafe_serialization.rb b/app/models/concerns/blocks_unsafe_serialization.rb new file mode 100644 index 00000000000..72adbe70f15 --- /dev/null +++ b/app/models/concerns/blocks_unsafe_serialization.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# Overrides `#serializable_hash` to raise an exception when called without the `only` option +# in order to prevent accidentally exposing attributes. +# +# An `unsafe: true` option can also be passed in to bypass this check. +# +# `#serializable_hash` is used by ActiveModel serializers like `ActiveModel::Serializers::JSON` +# which overrides `#as_json` and `#to_json`. +# +module BlocksUnsafeSerialization + extend ActiveSupport::Concern + extend ::Gitlab::Utils::Override + + UnsafeSerializationError = Class.new(StandardError) + + override :serializable_hash + def serializable_hash(options = nil) + return super if allow_serialization?(options) + + raise UnsafeSerializationError, + "Serialization has been disabled on #{self.class.name}" + end + + private + + def allow_serialization?(options = nil) + return false unless options + + !!(options[:only] || options[:unsafe]) + end +end diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb index 927d6ccb28f..efc65e55e40 100644 --- a/app/models/concerns/bulk_member_access_load.rb +++ b/app/models/concerns/bulk_member_access_load.rb @@ -1,61 +1,19 @@ # frozen_string_literal: true -# Returns and caches in thread max member access for a resource -# module BulkMemberAccessLoad extend ActiveSupport::Concern included do - # Determine the maximum access level for a group of resources in bulk. - # - # Returns a Hash mapping resource ID -> maximum access level. - def max_member_access_for_resource_ids(resource_klass, resource_ids, &block) - raise 'Block is mandatory' unless block_given? - - memoization_index = self.id - memoization_class = self.class - - resource_ids = resource_ids.uniq - memo_id = "#{memoization_class}:#{memoization_index}" - access = load_access_hash(resource_klass, memo_id) - - # Look up only the IDs we need - resource_ids -= access.keys - - return access if resource_ids.empty? - - resource_access = yield(resource_ids) - - access.merge!(resource_access) - - missing_resource_ids = resource_ids - resource_access.keys - - missing_resource_ids.each do |resource_id| - access[resource_id] = Gitlab::Access::NO_ACCESS - end - - access - end - def merge_value_to_request_store(resource_klass, resource_id, value) - max_member_access_for_resource_ids(resource_klass, [resource_id]) do + Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(resource_klass), + resource_ids: [resource_id], + default_value: Gitlab::Access::NO_ACCESS) do { resource_id => value } end end - private - - def max_member_access_for_resource_key(klass, memoization_index) - "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}" - end - - def load_access_hash(resource_klass, memo_id) - return {} unless Gitlab::SafeRequestStore.active? - - key = max_member_access_for_resource_key(resource_klass, memo_id) - Gitlab::SafeRequestStore[key] ||= {} - - Gitlab::SafeRequestStore[key] + def max_member_access_for_resource_key(klass) + "max_member_access_for_#{klass.name.underscore.pluralize}:#{self.class}:#{self.id}" end end end diff --git a/app/models/concerns/ci/has_deployment_name.rb b/app/models/concerns/ci/has_deployment_name.rb new file mode 100644 index 00000000000..fe288134872 --- /dev/null +++ b/app/models/concerns/ci/has_deployment_name.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Ci + module HasDeploymentName + extend ActiveSupport::Concern + + def count_user_deployment? + Feature.enabled?(:job_deployment_count) && deployment_name? + end + + def deployment_name? + self.class::DEPLOYMENT_NAMES.any? { |n| name.downcase.include?(n) } + end + end +end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index ccaccec3b6b..313c767e59f 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -7,12 +7,16 @@ module Ci DEFAULT_STATUS = 'created' BLOCKED_STATUS = %w[manual scheduled].freeze AVAILABLE_STATUSES = %w[created waiting_for_resource preparing pending running success failed canceled skipped manual scheduled].freeze + # TODO: replace STARTED_STATUSES with data from BUILD_STARTED_RUNNING_STATUSES in https://gitlab.com/gitlab-org/gitlab/-/issues/273378 + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82149#note_865508501 + BUILD_STARTED_RUNNING_STATUSES = %w[running success failed].freeze STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze ACTIVE_STATUSES = %w[waiting_for_resource preparing pending running].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze + CANCELABLE_STATUSES = %w[running waiting_for_resource preparing pending created scheduled].freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7, scheduled: 8, preparing: 9, waiting_for_resource: 10 }.freeze @@ -85,7 +89,7 @@ module Ci scope :waiting_for_resource_or_upcoming, -> { with_status(:created, :scheduled, :waiting_for_resource) } scope :cancelable, -> do - where(status: [:running, :waiting_for_resource, :preparing, :pending, :created, :scheduled]) + where(status: klass::CANCELABLE_STATUSES) end scope :without_statuses, -> (names) do diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 4bfeba338d2..b41b1ba6008 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -102,9 +102,7 @@ module CounterAttribute run_after_commit_or_now do if counter_attribute_enabled?(attribute) - redis_state do |redis| - redis.incrby(counter_key(attribute), increment) - end + increment_counter(attribute, increment) FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, self.class.name, self.id, attribute) else @@ -115,6 +113,28 @@ module CounterAttribute true end + def increment_counter(attribute, increment) + if counter_attribute_enabled?(attribute) + redis_state do |redis| + redis.incrby(counter_key(attribute), increment) + end + end + end + + def clear_counter!(attribute) + if counter_attribute_enabled?(attribute) + redis_state { |redis| redis.del(counter_key(attribute)) } + end + end + + def get_counter_value(attribute) + if counter_attribute_enabled?(attribute) + redis_state do |redis| + redis.get(counter_key(attribute)).to_i + end + end + end + def counter_key(attribute) "project:{#{project_id}}:counters:#{self.class}:#{id}:#{attribute}" end diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index b6245e29746..d9c622f247a 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -3,6 +3,8 @@ module DeploymentPlatform # rubocop:disable Gitlab/ModuleWithInstanceVariables def deployment_platform(environment: nil) + return if Feature.disabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + @deployment_platform ||= {} @deployment_platform[environment] ||= find_deployment_platform(environment) diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb index 28ee54afaa9..ad070090dd5 100644 --- a/app/models/concerns/has_user_type.rb +++ b/app/models/concerns/has_user_type.rb @@ -46,4 +46,17 @@ module HasUserType def internal? ghost? || (bot? && !project_bot?) end + + def redacted_name(viewing_user) + return self.name unless self.project_bot? + + return self.name if self.groups.any? && viewing_user&.can?(:read_group, self.groups.first) + + return self.name if viewing_user&.can?(:read_project, self.projects.first) + + # If the requester does not have permission to read the project bot name, + # the API returns an arbitrary string. UI changes will be addressed in a follow up issue: + # https://gitlab.com/gitlab-org/gitlab/-/issues/346058 + '****' + end end diff --git a/app/models/concerns/integrations/has_issue_tracker_fields.rb b/app/models/concerns/integrations/has_issue_tracker_fields.rb new file mode 100644 index 00000000000..b1def38d019 --- /dev/null +++ b/app/models/concerns/integrations/has_issue_tracker_fields.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Integrations + module HasIssueTrackerFields + extend ActiveSupport::Concern + + included do + field :project_url, + required: true, + storage: :data_fields, + title: -> { _('Project URL') }, + help: -> { s_('IssueTracker|The URL to the project in the external issue tracker.') } + + field :issues_url, + required: true, + storage: :data_fields, + title: -> { s_('IssueTracker|Issue URL') }, + help: -> do + format s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.'), + colon_id: '<code>:id</code>'.html_safe + end + + field :new_issue_url, + required: true, + storage: :data_fields, + title: -> { s_('IssueTracker|New issue URL') }, + help: -> { s_('IssueTracker|The URL to create an issue in the external issue tracker.') } + end + end +end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 0138c0ad20f..1eb30e88f16 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -74,6 +74,7 @@ module Issuable end has_many :note_authors, -> { distinct }, through: :notes, source: :author + has_many :user_note_authors, -> { distinct.where("notes.system = false") }, through: :notes, source: :author has_many :label_links, as: :target, inverse_of: :target has_many :labels, through: :label_links @@ -464,37 +465,54 @@ module Issuable false end - def to_hook_data(user, old_associations: {}) - changes = previous_changes + def hook_association_changes(old_associations) + changes = {} - if old_associations - old_labels = old_associations.fetch(:labels, labels) - old_assignees = old_associations.fetch(:assignees, assignees) - old_severity = old_associations.fetch(:severity, severity) + old_labels = old_associations.fetch(:labels, labels) + old_assignees = old_associations.fetch(:assignees, assignees) + old_severity = old_associations.fetch(:severity, severity) - if old_labels != labels - changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)] - end + if old_labels != labels + changes[:labels] = [old_labels.map(&:hook_attrs), labels.map(&:hook_attrs)] + end - if old_assignees != assignees - changes[:assignees] = [old_assignees.map(&:hook_attrs), assignees.map(&:hook_attrs)] - end + if old_assignees != assignees + changes[:assignees] = [old_assignees.map(&:hook_attrs), assignees.map(&:hook_attrs)] + end + + if supports_severity? && old_severity != severity + changes[:severity] = [old_severity, severity] + end + + if supports_escalation? && escalation_status + current_escalation_status = escalation_status.status_name + old_escalation_status = old_associations.fetch(:escalation_status, current_escalation_status) - if supports_severity? && old_severity != severity - changes[:severity] = [old_severity, severity] + if old_escalation_status != current_escalation_status + changes[:escalation_status] = [old_escalation_status, current_escalation_status] end + end - if self.respond_to?(:total_time_spent) - old_total_time_spent = old_associations.fetch(:total_time_spent, total_time_spent) - old_time_change = old_associations.fetch(:time_change, time_change) + if self.respond_to?(:total_time_spent) + old_total_time_spent = old_associations.fetch(:total_time_spent, total_time_spent) + old_time_change = old_associations.fetch(:time_change, time_change) - if old_total_time_spent != total_time_spent - changes[:total_time_spent] = [old_total_time_spent, total_time_spent] - changes[:time_change] = [old_time_change, time_change] - end + if old_total_time_spent != total_time_spent + changes[:total_time_spent] = [old_total_time_spent, total_time_spent] + changes[:time_change] = [old_time_change, time_change] end end + changes + end + + def to_hook_data(user, old_associations: {}) + changes = previous_changes + + if old_associations.present? + changes.merge!(hook_association_changes(old_associations)) + end + Gitlab::HookData::IssuableBuilder.new(self).build(user: user, changes: changes) end diff --git a/app/models/concerns/issuable_link.rb b/app/models/concerns/issuable_link.rb new file mode 100644 index 00000000000..3e14507bc70 --- /dev/null +++ b/app/models/concerns/issuable_link.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# == IssuableLink concern +# +# Contains common functionality shared between related Issues and related Epics +# +# Used by IssueLink, Epic::RelatedEpicLink +# +module IssuableLink + extend ActiveSupport::Concern + + TYPE_RELATES_TO = 'relates_to' + TYPE_BLOCKS = 'blocks' ## EE-only. Kept here to be used on link_type enum. + + class_methods do + def inverse_link_type(type) + type + end + + def issuable_type + raise NotImplementedError + end + end + + included do + validates :source, presence: true + validates :target, presence: true + validates :source, uniqueness: { scope: :target_id, message: 'is already related' } + validate :check_self_relation + validate :check_opposite_relation + + enum link_type: { TYPE_RELATES_TO => 0, TYPE_BLOCKS => 1 } + + private + + def check_self_relation + return unless source && target + + if source == target + errors.add(:source, 'cannot be related to itself') + end + end + + def check_opposite_relation + return unless source && target + + if self.class.base_class.find_by(source: target, target: source) + errors.add(:source, "is already related to this #{self.class.issuable_type}") + end + end + end +end + +IssuableLink.prepend_mod_with('IssuableLink') +IssuableLink::ClassMethods.prepend_mod_with('IssuableLink::ClassMethods') diff --git a/app/models/concerns/issue_resource_event.rb b/app/models/concerns/issue_resource_event.rb index 1c24032dbbb..5cbc937e465 100644 --- a/app/models/concerns/issue_resource_event.rb +++ b/app/models/concerns/issue_resource_event.rb @@ -8,6 +8,10 @@ module IssueResourceEvent scope :by_issue, ->(issue) { where(issue_id: issue.id) } - scope :by_issue_ids_and_created_at_earlier_or_equal_to, ->(issue_ids, time) { where(issue_id: issue_ids).where('created_at <= ?', time) } + scope :by_created_at_earlier_or_equal_to, ->(time) { where('created_at <= ?', time) } + scope :by_issue_ids, ->(issue_ids) do + table = self.klass.arel_table + where(table[:issue_id].in(issue_ids)) + end end end diff --git a/app/models/concerns/merge_request_reviewer_state.rb b/app/models/concerns/merge_request_reviewer_state.rb index 5859f43a70c..893d06b4da8 100644 --- a/app/models/concerns/merge_request_reviewer_state.rb +++ b/app/models/concerns/merge_request_reviewer_state.rb @@ -14,6 +14,14 @@ module MergeRequestReviewerState presence: true, inclusion: { in: self.states.keys } + belongs_to :updated_state_by, class_name: 'User', foreign_key: :updated_state_by_user_id + after_initialize :set_state, unless: :persisted? + + def attention_requested_by + return unless attention_requested? + + updated_state_by + end end end diff --git a/app/models/concerns/pg_full_text_searchable.rb b/app/models/concerns/pg_full_text_searchable.rb new file mode 100644 index 00000000000..68357c44300 --- /dev/null +++ b/app/models/concerns/pg_full_text_searchable.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +# This module adds PG full-text search capabilities to a model. +# A `search_data` association with a `search_vector` column is required. +# +# Declare the fields that will be part of the search vector with their +# corresponding weights. Possible values for weight are A, B, C, or D. +# For example: +# +# include PgFullTextSearchable +# pg_full_text_searchable columns: [{ name: 'title', weight: 'A' }, { name: 'description', weight: 'B' }] +# +# This module sets up an after_commit hook that updates the search data +# when the searchable columns are changed. You will need to implement the +# `#persist_pg_full_text_search_vector` method that does the actual insert or update. +# +# This also adds a `pg_full_text_search` scope so you can do: +# +# Model.pg_full_text_search("some search term") + +module PgFullTextSearchable + extend ActiveSupport::Concern + + LONG_WORDS_REGEX = %r([A-Za-z0-9+/@]{50,}).freeze + TSVECTOR_MAX_LENGTH = 1.megabyte.freeze + TEXT_SEARCH_DICTIONARY = 'english' + + def update_search_data! + tsvector_sql_nodes = self.class.pg_full_text_searchable_columns.map do |column, weight| + tsvector_arel_node(column, weight)&.to_sql + end + + persist_pg_full_text_search_vector(Arel.sql(tsvector_sql_nodes.compact.join(' || '))) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + Gitlab::AppJsonLogger.error( + message: 'Error updating search data: string is too long for tsvector', + class: self.class.name, + model_id: self.id + ) + end + + private + + def persist_pg_full_text_search_vector(search_vector) + raise NotImplementedError + end + + def tsvector_arel_node(column, weight) + return if self[column].blank? + + column_text = self[column].gsub(LONG_WORDS_REGEX, ' ') + column_text = column_text[0..(TSVECTOR_MAX_LENGTH - 1)] + column_text = ActiveSupport::Inflector.transliterate(column_text) + + Arel::Nodes::NamedFunction.new( + 'setweight', + [ + Arel::Nodes::NamedFunction.new( + 'to_tsvector', + [Arel::Nodes.build_quoted(TEXT_SEARCH_DICTIONARY), Arel::Nodes.build_quoted(column_text)] + ), + Arel::Nodes.build_quoted(weight) + ] + ) + end + + included do + cattr_reader :pg_full_text_searchable_columns do + {} + end + end + + class_methods do + def pg_full_text_searchable(columns:) + raise 'Full text search columns already defined!' if pg_full_text_searchable_columns.present? + + columns.each do |column| + pg_full_text_searchable_columns[column[:name]] = column[:weight] + end + + # We update this outside the transaction because this could raise an error if the resulting tsvector + # is too long. When that happens, we still persist the create / update but the model will not have a + # search data record. This is fine in most cases because this is a very rare occurrence and only happens + # with strings that are most likely unsearchable anyway. + # + # We also do not want to use a subtransaction here due to: https://gitlab.com/groups/gitlab-org/-/epics/6540 + after_save_commit do + next unless pg_full_text_searchable_columns.keys.any? { |f| saved_changes.has_key?(f) } + + update_search_data! + end + end + + def pg_full_text_search(search_term) + search_data_table = reflect_on_association(:search_data).klass.arel_table + + joins(:search_data).where( + Arel::Nodes::InfixOperation.new( + '@@', + search_data_table[:search_vector], + Arel::Nodes::NamedFunction.new( + 'websearch_to_tsquery', + [Arel::Nodes.build_quoted(TEXT_SEARCH_DICTIONARY), Arel::Nodes.build_quoted(search_term)] + ) + ) + ) + end + end +end diff --git a/app/models/concerns/runners_token_prefixable.rb b/app/models/concerns/runners_token_prefixable.rb index 1aea874337e..99bbbece7c7 100644 --- a/app/models/concerns/runners_token_prefixable.rb +++ b/app/models/concerns/runners_token_prefixable.rb @@ -1,14 +1,8 @@ # frozen_string_literal: true module RunnersTokenPrefixable - extend ActiveSupport::Concern - # Prefix for runners_token which can be used to invalidate existing tokens. # The value chosen here is GR (for Gitlab Runner) combined with the rotation # date (20220225) decimal to hex encoded. RUNNERS_TOKEN_PREFIX = 'GR1348941' - - def runners_token_prefix - RUNNERS_TOKEN_PREFIX - end end diff --git a/app/models/concerns/select_for_project_authorization.rb b/app/models/concerns/select_for_project_authorization.rb index 49342e30db6..5a7e16eb2c4 100644 --- a/app/models/concerns/select_for_project_authorization.rb +++ b/app/models/concerns/select_for_project_authorization.rb @@ -8,8 +8,10 @@ module SelectForProjectAuthorization select("projects.id AS project_id", "members.access_level") end - def select_as_maintainer_for_project_authorization - select(["projects.id AS project_id", "#{Gitlab::Access::MAINTAINER} AS access_level"]) + # workaround until we migrate Project#owners to have membership with + # OWNER access level + def select_project_owner_for_project_authorization + select(["projects.id AS project_id", "#{Gitlab::Access::OWNER} AS access_level"]) end end end diff --git a/app/models/concerns/sensitive_serializable_hash.rb b/app/models/concerns/sensitive_serializable_hash.rb new file mode 100644 index 00000000000..725ec60e9b6 --- /dev/null +++ b/app/models/concerns/sensitive_serializable_hash.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module SensitiveSerializableHash + extend ActiveSupport::Concern + + included do + class_attribute :attributes_exempt_from_serializable_hash, default: [] + end + + class_methods do + def prevent_from_serialization(*keys) + self.attributes_exempt_from_serializable_hash ||= [] + self.attributes_exempt_from_serializable_hash.concat keys + end + end + + # Override serializable_hash to exclude sensitive attributes by default + # + # In general, prefer NOT to use serializable_hash / to_json / as_json in favor + # of serializers / entities instead which has an allowlist of attributes + def serializable_hash(options = nil) + return super unless prevent_sensitive_fields_from_serializable_hash? + return super if options && options[:unsafe_serialization_hash] + + options = options.try(:dup) || {} + options[:except] = Array(options[:except]).dup + + options[:except].concat self.class.attributes_exempt_from_serializable_hash + + if self.class.respond_to?(:encrypted_attributes) + options[:except].concat self.class.encrypted_attributes.keys + + # Per https://github.com/attr-encrypted/attr_encrypted/blob/a96693e9a2a25f4f910bf915e29b0f364f277032/lib/attr_encrypted.rb#L413 + options[:except].concat self.class.encrypted_attributes.values.map { |v| v[:attribute] } + options[:except].concat self.class.encrypted_attributes.values.map { |v| "#{v[:attribute]}_iv" } + end + + super(options) + end + + private + + def prevent_sensitive_fields_from_serializable_hash? + Feature.enabled?(:prevent_sensitive_fields_from_serializable_hash, default_enabled: :yaml) + end +end diff --git a/app/models/concerns/spammable.rb b/app/models/concerns/spammable.rb index 4901cd832ff..b475eb79aa3 100644 --- a/app/models/concerns/spammable.rb +++ b/app/models/concerns/spammable.rb @@ -12,7 +12,7 @@ module Spammable included do has_one :user_agent_detail, as: :subject, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - attr_accessor :spam + attr_writer :spam attr_accessor :needs_recaptcha attr_accessor :spam_log @@ -29,6 +29,10 @@ module Spammable delegate :ip_address, :user_agent, to: :user_agent_detail, allow_nil: true end + def spam + !!@spam # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + def submittable_as_spam_by?(current_user) current_user && current_user.admin? && submittable_as_spam? end @@ -74,8 +78,9 @@ module Spammable end def recaptcha_error! - self.errors.add(:base, "Your #{spammable_entity_type} has been recognized as spam. "\ - "Please, change the content or solve the reCAPTCHA to proceed.") + self.errors.add(:base, _("Your %{spammable_entity_type} has been recognized as spam. "\ + "Please, change the content or solve the reCAPTCHA to proceed.") \ + % { spammable_entity_type: spammable_entity_type }) end def unrecoverable_spam_error! diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index 943ef3fa59f..d53594eb5af 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -44,7 +44,6 @@ module Timebox validates :group, presence: true, unless: :project validates :project, presence: true, unless: :group - validates :title, presence: true validate :timebox_type_check validate :start_date_should_be_less_than_due_date, if: proc { |m| m.start_date.present? && m.due_date.present? } diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index f44ad8ebe90..d91ec161b84 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -8,6 +8,10 @@ module TokenAuthenticatable @encrypted_token_authenticatable_fields ||= [] end + def token_authenticatable_fields + @token_authenticatable_fields ||= [] + end + private def add_authentication_token_field(token_field, options = {}) @@ -23,6 +27,8 @@ module TokenAuthenticatable strategy = TokenAuthenticatableStrategies::Base .fabricate(self, token_field, options) + prevent_from_serialization(*strategy.token_fields) if respond_to?(:prevent_from_serialization) + if options.fetch(:unique, true) define_singleton_method("find_by_#{token_field}") do |token| strategy.find_token_authenticatable(token) @@ -82,9 +88,5 @@ module TokenAuthenticatable @token_authenticatable_module ||= const_set(:TokenAuthenticatable, Module.new).tap(&method(:include)) end - - def token_authenticatable_fields - @token_authenticatable_fields ||= [] - end end end diff --git a/app/models/concerns/token_authenticatable_strategies/base.rb b/app/models/concerns/token_authenticatable_strategies/base.rb index 2cec4ab460e..2b677f37c89 100644 --- a/app/models/concerns/token_authenticatable_strategies/base.rb +++ b/app/models/concerns/token_authenticatable_strategies/base.rb @@ -23,6 +23,14 @@ module TokenAuthenticatableStrategies raise NotImplementedError end + def token_fields + result = [token_field] + + result << @expires_at_field if expirable? + + result + end + # Default implementation returns the token as-is def format_token(instance, token) instance.send("format_#{@token_field}", token) # rubocop:disable GitlabSecurity/PublicSend diff --git a/app/models/concerns/token_authenticatable_strategies/digest.rb b/app/models/concerns/token_authenticatable_strategies/digest.rb index 9926662ed66..5c94f25949f 100644 --- a/app/models/concerns/token_authenticatable_strategies/digest.rb +++ b/app/models/concerns/token_authenticatable_strategies/digest.rb @@ -2,6 +2,10 @@ module TokenAuthenticatableStrategies class Digest < Base + def token_fields + super + [token_field_name] + end + def find_token_authenticatable(token, unscoped = false) return unless token diff --git a/app/models/concerns/token_authenticatable_strategies/encrypted.rb b/app/models/concerns/token_authenticatable_strategies/encrypted.rb index e957d09fbc6..1db88c27181 100644 --- a/app/models/concerns/token_authenticatable_strategies/encrypted.rb +++ b/app/models/concerns/token_authenticatable_strategies/encrypted.rb @@ -2,6 +2,10 @@ module TokenAuthenticatableStrategies class Encrypted < Base + def token_fields + super + [encrypted_field] + end + def find_token_authenticatable(token, unscoped = false) return if token.blank? diff --git a/app/models/concerns/update_namespace_statistics.rb b/app/models/concerns/update_namespace_statistics.rb new file mode 100644 index 00000000000..26d6fc10228 --- /dev/null +++ b/app/models/concerns/update_namespace_statistics.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This module provides helpers for updating `NamespaceStatistics` with `after_save` and +# `after_destroy` hooks. +# +# Models including this module must respond to and return a `namespace` +# +# Example: +# +# class DependencyProxy::Manifest +# include UpdateNamespaceStatistics +# +# belongs_to :group +# alias_attribute :namespace, :group +# +# update_namespace_statistics namespace_statistics_name: :dependency_proxy_size +# end +module UpdateNamespaceStatistics + extend ActiveSupport::Concern + include AfterCommitQueue + + class_methods do + attr_reader :namespace_statistics_name, :statistic_attribute + + # Configure the model to update `namespace_statistics_name` on NamespaceStatistics, + # when `statistic_attribute` changes + # + # - namespace_statistics_name: A column of `NamespaceStatistics` to update + # - statistic_attribute: An attribute of the current model, default to `size` + def update_namespace_statistics(namespace_statistics_name:, statistic_attribute: :size) + @namespace_statistics_name = namespace_statistics_name + @statistic_attribute = statistic_attribute + + after_save(:schedule_namespace_statistics_refresh, if: :update_namespace_statistics?) + after_destroy(:schedule_namespace_statistics_refresh) + end + + private :update_namespace_statistics + end + + included do + private + + def update_namespace_statistics? + saved_change_to_attribute?(self.class.statistic_attribute) + end + + def schedule_namespace_statistics_refresh + run_after_commit do + Groups::UpdateStatisticsWorker.perform_async(namespace.id, [self.class.namespace_statistics_name]) + end + end + end +end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 1f123cb0244..fa03d73646d 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -14,6 +14,8 @@ class ContainerRepository < ApplicationRecord ABORTABLE_MIGRATION_STATES = (ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze MIGRATION_STATES = (IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze + MIGRATION_PHASE_1_STARTED_AT = Date.new(2021, 11, 4).freeze + TooManyImportsError = Class.new(StandardError) NativeImportError = Class.new(StandardError) @@ -64,7 +66,7 @@ class ContainerRepository < ApplicationRecord # feature flag since it is only accessed in this query. # https://gitlab.com/gitlab-org/gitlab/-/issues/350543 tracks the rollout and # removal of this feature flag. - joins(:project).where( + joins(project: [:namespace]).where( migration_state: [:default], created_at: ...ContainerRegistry::Migration.created_before ).with_target_import_tier @@ -74,7 +76,7 @@ class ContainerRepository < ApplicationRecord FROM feature_gates WHERE feature_gates.feature_key = 'container_registry_phase_2_deny_list' AND feature_gates.key = 'actors' - AND feature_gates.value = concat('Group:', projects.namespace_id) + AND feature_gates.value = concat('Group:', namespaces.traversal_ids[1]) )" ) end @@ -408,6 +410,16 @@ class ContainerRepository < ApplicationRecord update!(expiration_policy_started_at: Time.zone.now) end + def size + strong_memoize(:size) do + next unless Gitlab.com? + next if self.created_at.before?(MIGRATION_PHASE_1_STARTED_AT) + next unless gitlab_api_client.supports_gitlab_api? + + gitlab_api_client.repository_details(self.path, with_size: true)['size_bytes'] + end + end + def migration_in_active_state? migration_state.in?(ACTIVE_MIGRATION_STATES) end diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index a981351f4a0..4fa2c3fb8cf 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -23,8 +23,9 @@ class CustomerRelations::Contact < ApplicationRecord validates :last_name, presence: true, length: { maximum: 255 } validates :email, length: { maximum: 255 } validates :description, length: { maximum: 1024 } + validates :email, uniqueness: { scope: :group_id } validate :validate_email_format - validate :unique_email_for_group_hierarchy + validate :validate_root_group def self.reference_prefix '[contact:' @@ -41,14 +42,13 @@ class CustomerRelations::Contact < ApplicationRecord def self.find_ids_by_emails(group, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK - where(group_id: group.self_and_ancestor_ids, email: emails) - .pluck(:id) + where(group: group, email: emails).pluck(:id) end def self.exists_for_group?(group) return false unless group - exists?(group_id: group.self_and_ancestor_ids) + exists?(group: group) end private @@ -59,13 +59,9 @@ class CustomerRelations::Contact < ApplicationRecord self.errors.add(:email, I18n.t(:invalid, scope: 'valid_email.validations.email')) unless ValidateEmail.valid?(self.email) end - def unique_email_for_group_hierarchy - return unless group - return unless email + def validate_root_group + return if group&.root? - duplicate_email_exists = CustomerRelations::Contact - .where(group_id: group.self_and_hierarchy.pluck(:id), email: email) - .where.not(id: id).exists? - self.errors.add(:email, _('contact with same email already exists in group hierarchy')) if duplicate_email_exists + self.errors.add(:base, _('contacts can only be added to root groups')) end end diff --git a/app/models/customer_relations/issue_contact.rb b/app/models/customer_relations/issue_contact.rb index 3e9d1e97c8c..dc7a3fd87bc 100644 --- a/app/models/customer_relations/issue_contact.rb +++ b/app/models/customer_relations/issue_contact.rb @@ -6,7 +6,7 @@ class CustomerRelations::IssueContact < ApplicationRecord belongs_to :issue, optional: false, inverse_of: :customer_relations_contacts belongs_to :contact, optional: false, inverse_of: :issue_contacts - validate :contact_belongs_to_issue_group_or_ancestor + validate :contact_belongs_to_root_group def self.find_contact_ids_by_emails(issue_id, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK @@ -24,11 +24,11 @@ class CustomerRelations::IssueContact < ApplicationRecord private - def contact_belongs_to_issue_group_or_ancestor + def contact_belongs_to_root_group return unless contact&.group_id return unless issue&.project&.namespace_id - return if issue.project.group&.self_and_ancestor_ids&.include?(contact.group_id) + return if issue.project.root_ancestor&.id == contact.group_id - errors.add(:base, _('The contact does not belong to the issue group or an ancestor')) + errors.add(:base, _("The contact does not belong to the issue group's root ancestor")) end end diff --git a/app/models/customer_relations/organization.rb b/app/models/customer_relations/organization.rb index c206d1e05f5..a23b9d8fe28 100644 --- a/app/models/customer_relations/organization.rb +++ b/app/models/customer_relations/organization.rb @@ -19,9 +19,18 @@ class CustomerRelations::Organization < ApplicationRecord validates :name, uniqueness: { case_sensitive: false, scope: [:group_id] } validates :name, length: { maximum: 255 } validates :description, length: { maximum: 1024 } + validate :validate_root_group def self.find_by_name(group_id, name) where(group: group_id) .where('LOWER(name) = LOWER(?)', name) end + + private + + def validate_root_group + return if group&.root? + + self.errors.add(:base, _('organizations can only be added to root groups')) + end end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb index f7b08f1d077..dc40ff62adb 100644 --- a/app/models/dependency_proxy/blob.rb +++ b/app/models/dependency_proxy/blob.rb @@ -5,8 +5,10 @@ class DependencyProxy::Blob < ApplicationRecord include TtlExpirable include Packages::Destructible include EachBatch + include UpdateNamespaceStatistics belongs_to :group + alias_attribute :namespace, :group MAX_FILE_SIZE = 5.gigabytes.freeze @@ -17,6 +19,7 @@ class DependencyProxy::Blob < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) } mount_file_store_uploader DependencyProxy::FileUploader + update_namespace_statistics namespace_statistics_name: :dependency_proxy_size def self.total_size sum(:size) diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index c2587ffac9d..5ad746e4cd1 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -5,8 +5,10 @@ class DependencyProxy::Manifest < ApplicationRecord include TtlExpirable include Packages::Destructible include EachBatch + include UpdateNamespaceStatistics belongs_to :group + alias_attribute :namespace, :group MAX_FILE_SIZE = 10.megabytes.freeze DIGEST_HEADER = 'Docker-Content-Digest' @@ -20,6 +22,7 @@ class DependencyProxy::Manifest < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) } mount_file_store_uploader DependencyProxy::FileUploader + update_namespace_statistics namespace_statistics_name: :dependency_proxy_size def self.find_by_file_name_or_digest(file_name:, digest:) find_by(file_name: file_name) || find_by(digest: digest) diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 46409465209..c06c809538a 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -8,7 +8,6 @@ class Deployment < ApplicationRecord include Importable include Gitlab::Utils::StrongMemoize include FastDestroyAll - include FromUnion StatusUpdateError = Class.new(StandardError) StatusSyncError = Class.new(StandardError) diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 6ebac6384bc..02979d5f804 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -145,7 +145,7 @@ class DiffNote < Note end def fetch_diff_file - return note_diff_file.raw_diff_file if note_diff_file + return note_diff_file.raw_diff_file if note_diff_file && !note_diff_file.raw_diff_file.has_renderable? if created_at_diff?(noteable.diff_refs) # We're able to use the already persisted diffs (Postgres) if we're diff --git a/app/models/environment.rb b/app/models/environment.rb index 51a9024721b..450ed6206d5 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -461,11 +461,16 @@ class Environment < ApplicationRecord # See https://en.wikipedia.org/wiki/Deployment_environment for industry standard deployment environments def guess_tier case name - when %r{dev|review|trunk}i then self.class.tiers[:development] - when %r{test|qc}i then self.class.tiers[:testing] - when %r{st(a|)g|mod(e|)l|pre|demo}i then self.class.tiers[:staging] - when %r{pr(o|)d|live}i then self.class.tiers[:production] - else self.class.tiers[:other] + when /(dev|review|trunk)/i + self.class.tiers[:development] + when /(test|tst|int|ac(ce|)pt|qa|qc|control|quality)/i + self.class.tiers[:testing] + when /(st(a|)g|mod(e|)l|pre|demo)/i + self.class.tiers[:staging] + when /(pr(o|)d|live)/i + self.class.tiers[:production] + else + self.class.tiers[:other] end end end diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index 25f812645b1..0a429bb7afd 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -59,6 +59,10 @@ module ErrorTracking integrated end + def integrated_enabled? + enabled? && integrated_client? + end + def gitlab_dsn strong_memoize(:gitlab_dsn) do client_key&.sentry_dsn diff --git a/app/models/event_collection.rb b/app/models/event_collection.rb index f799377a15f..fc093894847 100644 --- a/app/models/event_collection.rb +++ b/app/models/event_collection.rb @@ -44,31 +44,31 @@ class EventCollection private def project_events - relation_with_join_lateral('project_id', projects) + in_operator_optimized_relation('project_id', projects) end - def project_and_group_events - group_events = relation_with_join_lateral('group_id', groups) + def group_events + in_operator_optimized_relation('group_id', groups) + end + def project_and_group_events Event.from_union([project_events, group_events]).recent end - # This relation is built using JOIN LATERAL, producing faster queries than a - # regular LIMIT + OFFSET approach. - def relation_with_join_lateral(parent_column, parents) - parents_for_lateral = parents.select(:id).to_sql - - lateral = filtered_events - # Applying the limit here (before we filter (permissions) means we may get less than limit) - .limit(limit_for_join_lateral) - .where("events.#{parent_column} = parents_for_lateral.id") # rubocop:disable GitlabSecurity/SqlInjection - .to_sql - - # The outer query does not need to re-apply the filters since the JOIN - # LATERAL body already takes care of this. - base_relation - .from("(#{parents_for_lateral}) parents_for_lateral") - .joins("JOIN LATERAL (#{lateral}) AS #{Event.table_name} ON true") + def in_operator_optimized_relation(parent_column, parents) + scope = filtered_events + array_scope = parents.select(:id) + array_mapping_scope = -> (parent_id_expression) { Event.where(Event.arel_table[parent_column].eq(parent_id_expression)).reorder(id: :desc) } + finder_query = -> (id_expression) { Event.where(Event.arel_table[:id].eq(id_expression)) } + + Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder + .new( + scope: scope, + array_scope: array_scope, + array_mapping_scope: array_mapping_scope, + finder_query: finder_query + ) + .execute end def filtered_events @@ -85,16 +85,6 @@ class EventCollection Event.unscoped.recent end - def limit_for_join_lateral - # Applying the OFFSET on the inside of a JOIN LATERAL leads to incorrect - # results. To work around this we need to increase the inner limit for every - # page. - # - # This means that on page 1 we use LIMIT 20, and an outer OFFSET of 0. On - # page 2 we use LIMIT 40 and an outer OFFSET of 20. - @limit + @offset - end - def current_page (@offset / @limit) + 1 end diff --git a/app/models/group.rb b/app/models/group.rb index 1d6a3a14450..14d088dd38b 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -19,7 +19,6 @@ class Group < Namespace include BulkMemberAccessLoad include ChronicDurationAttribute include RunnerTokenExpirationInterval - include RunnersTokenPrefixable extend ::Gitlab::Utils::Override @@ -120,7 +119,7 @@ class Group < Namespace add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption, default_enabled: true) ? :optional : :required }, - prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX after_create :post_create_hook after_destroy :post_destroy_hook @@ -676,7 +675,7 @@ class Group < Namespace override :format_runners_token def format_runners_token(token) - "#{runners_token_prefix}#{token}" + "#{RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX}#{token}" end def project_creation_level @@ -817,7 +816,9 @@ class Group < Namespace private def max_member_access(user_ids) - max_member_access_for_resource_ids(User, user_ids) do |user_ids| + Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(User), + resource_ids: user_ids, + default_value: Gitlab::Access::NO_ACCESS) do |user_ids| members_with_parents.where(user_id: user_ids).group(:user_id).maximum(:access_level) end end @@ -892,6 +893,7 @@ class Group < Namespace .where(group_member_table[:requested_at].eq(nil)) .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) .where(group_member_table[:source_type].eq('Namespace')) + .where(group_member_table[:state].eq(::Member::STATE_ACTIVE)) .non_minimal_access end diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 7e538238cbd..88941df691c 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -37,14 +37,14 @@ class WebHook < ApplicationRecord !temporarily_disabled? && !permanently_disabled? end - def temporarily_disabled? - return false unless web_hooks_disable_failed? + def temporarily_disabled?(ignore_flag: false) + return false unless ignore_flag || web_hooks_disable_failed? disabled_until.present? && disabled_until >= Time.current end - def permanently_disabled? - return false unless web_hooks_disable_failed? + def permanently_disabled?(ignore_flag: false) + return false unless ignore_flag || web_hooks_disable_failed? recent_failures > FAILURE_THRESHOLD end @@ -106,6 +106,13 @@ class WebHook < ApplicationRecord save(validate: false) end + def active_state(ignore_flag: false) + return :permanently_disabled if permanently_disabled?(ignore_flag: ignore_flag) + return :temporarily_disabled if temporarily_disabled?(ignore_flag: ignore_flag) + + :enabled + end + # @return [Boolean] Whether or not the WebHook is currently throttled. def rate_limited? return false unless rate_limit diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 2016024b2f4..00e55d0fd89 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -118,7 +118,8 @@ class InstanceConfiguration group_export_download: application_setting_limit_per_minute(:group_download_export_limit), group_import: application_setting_limit_per_minute(:group_import_limit), raw_blob: application_setting_limit_per_minute(:raw_blob_request_limit), - user_email_lookup: application_setting_limit_per_minute(:user_email_lookup_limit), + search_rate_limit: application_setting_limit_per_minute(:search_rate_limit), + search_rate_limit_unauthenticated: application_setting_limit_per_minute(:search_rate_limit_unauthenticated), users_get_by_id: { enabled: application_settings[:users_get_by_id_limit] > 0, requests_per_period: application_settings[:users_get_by_id_limit], diff --git a/app/models/integration.rb b/app/models/integration.rb index e9cd90649ba..274c16507b7 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -9,10 +9,18 @@ class Integration < ApplicationRecord include Integrations::HasDataFields include FromUnion include EachBatch + include IgnorableColumns + + ignore_column :template, remove_with: '15.0', remove_after: '2022-04-22' + ignore_column :type, remove_with: '15.0', remove_after: '2022-04-22' + + UnknownType = Class.new(StandardError) + + self.inheritance_column = :type_new INTEGRATION_NAMES = %w[ asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord - drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat irker jira + drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat harbor irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao ].freeze @@ -37,9 +45,21 @@ class Integration < ApplicationRecord Integrations::BaseSlashCommands ].freeze + SECTION_TYPE_CONNECTION = 'connection' + serialize :properties, JSON # rubocop:disable Cop/ActiveRecordSerialize - attribute :type, Gitlab::Integrations::StiType.new + attr_encrypted :encrypted_properties_tmp, + attribute: :encrypted_properties, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + + alias_attribute :type, :type_new default_value_for :active, false default_value_for :alert_events, true @@ -57,6 +77,8 @@ class Integration < ApplicationRecord default_value_for :wiki_page_events, true after_initialize :initialize_properties + after_initialize :copy_properties_to_encrypted_properties + before_save :copy_properties_to_encrypted_properties after_commit :reset_updated_properties @@ -74,9 +96,10 @@ class Integration < ApplicationRecord validate :validate_belongs_to_project_or_group scope :external_issue_trackers, -> { where(category: 'issue_tracker').active } - scope :external_wikis, -> { where(type: 'ExternalWikiService').active } + scope :by_name, ->(name) { by_type(integration_name_to_type(name)) } + scope :external_wikis, -> { by_name(:external_wiki).active } scope :active, -> { where(active: true) } - scope :by_type, -> (type) { where(type: type) } + scope :by_type, ->(type) { where(type: type) } # INTERNAL USE ONLY: use by_name instead scope :by_active_flag, -> (flag) { where(active: flag) } scope :inherit_from_id, -> (id) { where(inherit_from_id: id) } scope :with_default_settings, -> { where.not(inherit_from_id: nil) } @@ -99,6 +122,39 @@ class Integration < ApplicationRecord scope :alert_hooks, -> { where(alert_events: true, active: true) } scope :deployment, -> { where(category: 'deployment') } + class << self + private + + attr_writer :field_storage + + def field_storage + @field_storage || :properties + end + end + + # :nocov: Tested on subclasses. + def self.field(name, storage: field_storage, **attrs) + fields << ::Integrations::Field.new(name: name, **attrs) + + case storage + when :properties + prop_accessor(name) + when :data_fields + data_field(name) + else + raise ArgumentError, "Unknown field storage: #{storage}" + end + end + # :nocov: + + def self.fields + @fields ||= [] + end + + def fields + self.class.fields + end + # Provide convenient accessor methods for each serialized property. # Also keep track of updated properties in a similar way as ActiveModel::Dirty def self.prop_accessor(*args) @@ -112,8 +168,10 @@ class Integration < ApplicationRecord def #{arg}=(value) self.properties ||= {} + self.encrypted_properties_tmp = properties updated_properties['#{arg}'] = #{arg} unless #{arg}_changed? self.properties['#{arg}'] = value + self.encrypted_properties_tmp['#{arg}'] = value end def #{arg}_changed? @@ -158,10 +216,6 @@ class Integration < ApplicationRecord self.supported_events.map { |event| IntegrationsHelper.integration_event_field_name(event) } end - def self.supported_event_actions - %w[] - end - def self.supported_events %w[commit push tag_push issue confidential_issue merge_request wiki_page] end @@ -226,7 +280,7 @@ class Integration < ApplicationRecord end # Returns a list of available integration types. - # Example: ["AsanaService", ...] + # Example: ["Integrations::Asana", ...] def self.available_integration_types(include_project_specific: true, include_dev: true) available_integration_names(include_project_specific: include_project_specific, include_dev: include_dev).map do integration_name_to_type(_1) @@ -234,22 +288,27 @@ class Integration < ApplicationRecord end # Returns the model for the given integration name. - # Example: "asana" => Integrations::Asana + # Example: :asana => Integrations::Asana def self.integration_name_to_model(name) type = integration_name_to_type(name) integration_type_to_model(type) end # Returns the STI type for the given integration name. - # Example: "asana" => "AsanaService" + # Example: "asana" => "Integrations::Asana" def self.integration_name_to_type(name) - "#{name}_service".camelize + name = name.to_s + if available_integration_names.exclude?(name) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownType.new(name.inspect)) + else + "Integrations::#{name.camelize}" + end end # Returns the model for the given STI type. - # Example: "AsanaService" => Integrations::Asana + # Example: "Integrations::Asana" => Integrations::Asana def self.integration_type_to_model(type) - Gitlab::Integrations::StiType.new.cast(type).constantize + type.constantize end private_class_method :integration_type_to_model @@ -298,7 +357,7 @@ class Integration < ApplicationRecord from_union([ active.where(instance: true), active.where(group_id: group_ids, inherit_from_id: nil) - ]).order(Arel.sql("type ASC, array_position(#{array}::bigint[], #{table_name}.group_id), instance DESC")).group_by(&:type).each do |type, records| + ]).order(Arel.sql("type_new ASC, array_position(#{array}::bigint[], #{table_name}.group_id), instance DESC")).group_by(&:type).each do |type, records| build_from_integration(records.first, association => scope.id).save end end @@ -330,6 +389,10 @@ class Integration < ApplicationRecord true end + def activate_disabled_reason + nil + end + def category read_attribute(:category).to_sym end @@ -338,6 +401,12 @@ class Integration < ApplicationRecord self.properties = {} if has_attribute?(:properties) && properties.nil? end + def copy_properties_to_encrypted_properties + self.encrypted_properties_tmp = properties + rescue ActiveModel::MissingAttributeError + # ignore - in a record built from using a restricted select list + end + def title # implement inside child end @@ -355,8 +424,7 @@ class Integration < ApplicationRecord self.class.to_param end - def fields - # implement inside child + def sections [] end @@ -371,8 +439,24 @@ class Integration < ApplicationRecord %w[active] end + # return a hash of columns => values suitable for passing to insert_all def to_integration_hash - as_json(methods: :type, except: %w[id instance project_id group_id]) + column = self.class.attribute_aliases.fetch('type', 'type') + copy_properties_to_encrypted_properties + + as_json(except: %w[id instance project_id group_id encrypted_properties_tmp]) + .merge(column => type) + .merge(reencrypt_properties) + end + + def reencrypt_properties + unless properties.nil? || properties.empty? + alg = self.class.encrypted_attributes[:encrypted_properties_tmp][:algorithm] + iv = generate_iv(alg) + ep = self.class.encrypt(:encrypted_properties_tmp, properties, { iv: iv }) + end + + { 'encrypted_properties' => ep, 'encrypted_properties_iv' => iv } end def to_data_fields_hash @@ -392,7 +476,10 @@ class Integration < ApplicationRecord end def api_field_names - fields.pluck(:name).grep_v(/password|token|key|title|description/) + fields + .reject { _1[:type] == 'password' } + .pluck(:name) + .grep_v(/password|token|key/) end def global_fields @@ -410,10 +497,6 @@ class Integration < ApplicationRecord end end - def configurable_event_actions - self.class.supported_event_actions - end - def supported_events self.class.supported_events end diff --git a/app/models/integrations/asana.rb b/app/models/integrations/asana.rb index 7949563a1dc..054f0606dd2 100644 --- a/app/models/integrations/asana.rb +++ b/app/models/integrations/asana.rb @@ -4,8 +4,6 @@ require 'asana' module Integrations class Asana < Integration - include ActionView::Helpers::UrlHelper - prop_accessor :api_key, :restrict_to_branch validates :api_key, presence: true, if: :activated? @@ -18,7 +16,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/asana'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/asana'), target: '_blank', rel: 'noopener noreferrer' s_('Add commit messages as comments to Asana tasks. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index 57767c63cf4..c614a9415ab 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -2,7 +2,6 @@ module Integrations class Bamboo < BaseCi - include ActionView::Helpers::UrlHelper include ReactivelyCached prepend EnableSslVerification @@ -36,7 +35,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bamboo'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bamboo'), target: '_blank', rel: 'noopener noreferrer' s_('BambooService|Run CI/CD pipelines with Atlassian Bamboo. You must set up automatic revision labeling and a repository trigger in Bamboo. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index d0d54a92021..d5b6357cb66 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -241,7 +241,6 @@ module Integrations def notify_for_ref?(data) return true if data[:object_kind] == 'tag_push' - return true if data[:object_kind] == 'deployment' && !Feature.enabled?(:chat_notification_deployment_protected_branch_filter, project) ref = data[:ref] || data.dig(:object_attributes, :ref) return true if ref.blank? # No need to check protected branches when there is no ref diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index 42a6a3a19c8..458d0199e7a 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -4,10 +4,6 @@ module Integrations class BaseIssueTracker < Integration validate :one_issue_tracker, if: :activated?, on: :manual_change - # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :project_url, :issues_url, :new_issue_url - default_value_for :category, 'issue_tracker' before_validation :handle_properties @@ -72,14 +68,6 @@ module Integrations issue_url(iid) end - def fields - [ - { type: 'text', name: 'project_url', title: _('Project URL'), help: s_('IssueTracker|The URL to the project in the external issue tracker.'), required: true }, - { type: 'text', name: 'issues_url', title: s_('IssueTracker|Issue URL'), help: s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') % { colon_id: '<code>:id</code>'.html_safe }, required: true }, - { type: 'text', name: 'new_issue_url', title: s_('IssueTracker|New issue URL'), help: s_('IssueTracker|The URL to create an issue in the external issue tracker.'), required: true } - ] - end - def initialize_properties {} end @@ -132,8 +120,18 @@ module Integrations # implement inside child end + def activate_disabled_reason + { trackers: other_external_issue_trackers } if other_external_issue_trackers.any? + end + private + def other_external_issue_trackers + return [] unless project_level? + + @other_external_issue_trackers ||= project.integrations.external_issue_trackers.where.not(id: id) + end + def enabled_in_gitlab_config Gitlab.config.issues_tracker && Gitlab.config.issues_tracker.values.any? && @@ -145,10 +143,10 @@ module Integrations end def one_issue_tracker - return if template? || instance? + return if instance? return if project.blank? - if project.integrations.external_issue_trackers.where.not(id: id).any? + if other_external_issue_trackers.any? errors.add(:base, _('Another issue tracker is already in use. Only one issue tracker service can be active at a time')) end end diff --git a/app/models/integrations/bugzilla.rb b/app/models/integrations/bugzilla.rb index 9251015acb8..74e282f6848 100644 --- a/app/models/integrations/bugzilla.rb +++ b/app/models/integrations/bugzilla.rb @@ -2,7 +2,7 @@ module Integrations class Bugzilla < BaseIssueTracker - include ActionView::Helpers::UrlHelper + include Integrations::HasIssueTrackerFields validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? @@ -15,7 +15,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bugzilla'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/bugzilla'), target: '_blank', rel: 'noopener noreferrer' s_("IssueTracker|Use Bugzilla as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/campfire.rb b/app/models/integrations/campfire.rb index c78fc6eff51..81e6c2411b8 100644 --- a/app/models/integrations/campfire.rb +++ b/app/models/integrations/campfire.rb @@ -2,8 +2,6 @@ module Integrations class Campfire < Integration - include ActionView::Helpers::UrlHelper - prop_accessor :token, :subdomain, :room validates :token, presence: true, if: :activated? @@ -16,7 +14,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('api/services', anchor: 'campfire'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('api/services', anchor: 'campfire'), target: '_blank', rel: 'noopener noreferrer' s_('CampfireService|Send notifications about push events to Campfire chat rooms. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/confluence.rb b/app/models/integrations/confluence.rb index 7f111f482dd..65adce7a8d6 100644 --- a/app/models/integrations/confluence.rb +++ b/app/models/integrations/confluence.rb @@ -2,8 +2,6 @@ module Integrations class Confluence < Integration - include ActionView::Helpers::UrlHelper - VALID_SCHEME_MATCH = %r{\Ahttps?\Z}.freeze VALID_HOST_MATCH = %r{\A.+\.atlassian\.net\Z}.freeze VALID_PATH_MATCH = %r{\A/wiki(/|\Z)}.freeze @@ -39,7 +37,7 @@ module Integrations s_( 'ConfluenceService|Your GitLab wiki is still available at %{wiki_link}. To re-enable the link to the GitLab wiki, disable this integration.' % - { wiki_link: link_to(wiki_url, wiki_url) } + { wiki_link: ActionController::Base.helpers.link_to(wiki_url, wiki_url) } ).html_safe else s_('ConfluenceService|Link to a Confluence Workspace from the sidebar. Enabling this integration replaces the "Wiki" sidebar link with a link to the Confluence Workspace. The GitLab wiki is still available at the original URL.').html_safe diff --git a/app/models/integrations/custom_issue_tracker.rb b/app/models/integrations/custom_issue_tracker.rb index 635a9d093e9..3770e813eaa 100644 --- a/app/models/integrations/custom_issue_tracker.rb +++ b/app/models/integrations/custom_issue_tracker.rb @@ -2,7 +2,8 @@ module Integrations class CustomIssueTracker < BaseIssueTracker - include ActionView::Helpers::UrlHelper + include HasIssueTrackerFields + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def title @@ -14,7 +15,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/custom_issue_tracker'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/custom_issue_tracker'), target: '_blank', rel: 'noopener noreferrer' s_('IssueTracker|Use a custom issue tracker that is not in the integration list. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/discord.rb b/app/models/integrations/discord.rb index 21993dd3c43..790e41e5a2a 100644 --- a/app/models/integrations/discord.rb +++ b/app/models/integrations/discord.rb @@ -4,8 +4,6 @@ require "discordrb/webhooks" module Integrations class Discord < BaseChatNotification - include ActionView::Helpers::UrlHelper - ATTACHMENT_REGEX = /: (?<entry>.*?)\n - (?<name>.*)\n*/.freeze def title @@ -21,7 +19,7 @@ module Integrations end def help - docs_link = link_to _('How do I set up this service?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/discord_notifications'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('How do I set up this service?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/discord_notifications'), target: '_blank', rel: 'noopener noreferrer' s_('Send notifications about project events to a Discord channel. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/ewm.rb b/app/models/integrations/ewm.rb index 24d343b7cb4..1b86ef73c85 100644 --- a/app/models/integrations/ewm.rb +++ b/app/models/integrations/ewm.rb @@ -2,7 +2,7 @@ module Integrations class Ewm < BaseIssueTracker - include ActionView::Helpers::UrlHelper + include HasIssueTrackerFields validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? @@ -19,7 +19,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/ewm'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/ewm'), target: '_blank', rel: 'noopener noreferrer' s_("IssueTracker|Use IBM Engineering Workflow Management as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/external_wiki.rb b/app/models/integrations/external_wiki.rb index 2a8d598117b..18c48411e30 100644 --- a/app/models/integrations/external_wiki.rb +++ b/app/models/integrations/external_wiki.rb @@ -2,8 +2,6 @@ module Integrations class ExternalWiki < Integration - include ActionView::Helpers::UrlHelper - prop_accessor :external_wiki_url validates :external_wiki_url, presence: true, public_url: true, if: :activated? @@ -33,7 +31,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/wiki/index', anchor: 'link-an-external-wiki'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/wiki/index', anchor: 'link-an-external-wiki'), target: '_blank', rel: 'noopener noreferrer' s_('Link an external wiki from the project\'s sidebar. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb new file mode 100644 index 00000000000..49ab97677db --- /dev/null +++ b/app/models/integrations/field.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Integrations + class Field + SENSITIVE_NAME = %r/token|key|password|passphrase|secret/.freeze + + ATTRIBUTES = %i[ + section type placeholder required choices value checkbox_label + title help + non_empty_password_help + non_empty_password_title + api_only + ].freeze + + attr_reader :name + + def initialize(name:, type: 'text', api_only: false, **attributes) + @name = name.to_s.freeze + + attributes[:type] = SENSITIVE_NAME.match?(@name) ? 'password' : type + attributes[:api_only] = api_only + @attributes = attributes.freeze + end + + def [](key) + return name if key == :name + + value = @attributes[key] + return value.call if value.respond_to?(:call) + + value + end + + def sensitive? + @attributes[:type] == 'password' + end + + ATTRIBUTES.each do |name| + define_method(name) { self[name] } + end + end +end diff --git a/app/models/integrations/flowdock.rb b/app/models/integrations/flowdock.rb index 443f61e65dd..476cdc35585 100644 --- a/app/models/integrations/flowdock.rb +++ b/app/models/integrations/flowdock.rb @@ -2,8 +2,6 @@ module Integrations class Flowdock < Integration - include ActionView::Helpers::UrlHelper - prop_accessor :token validates :token, presence: true, if: :activated? @@ -16,7 +14,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('api/services', anchor: 'flowdock'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('api/services', anchor: 'flowdock'), target: '_blank', rel: 'noopener noreferrer' s_('FlowdockService|Send event notifications from GitLab to Flowdock flows. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/hangouts_chat.rb b/app/models/integrations/hangouts_chat.rb index 0d6b9fb1019..8c68c9ff95a 100644 --- a/app/models/integrations/hangouts_chat.rb +++ b/app/models/integrations/hangouts_chat.rb @@ -2,8 +2,6 @@ module Integrations class HangoutsChat < BaseChatNotification - include ActionView::Helpers::UrlHelper - def title 'Google Chat' end @@ -17,7 +15,7 @@ module Integrations end def help - docs_link = link_to _('How do I set up a Google Chat webhook?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/hangouts_chat'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('How do I set up a Google Chat webhook?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/hangouts_chat'), target: '_blank', rel: 'noopener noreferrer' s_('Before enabling this integration, create a webhook for the room in Google Chat where you want to receive notifications from this project. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/harbor.rb b/app/models/integrations/harbor.rb new file mode 100644 index 00000000000..4c76e418886 --- /dev/null +++ b/app/models/integrations/harbor.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Integrations + class Harbor < Integration + prop_accessor :url, :project_name, :username, :password + + validates :url, public_url: true, presence: true, if: :activated? + validates :project_name, presence: true, if: :activated? + validates :username, presence: true, if: :activated? + validates :password, format: { with: ::Ci::Maskable::REGEX }, if: :activated? + + before_validation :reset_username_and_password + + def title + 'Harbor' + end + + def description + s_("HarborIntegration|Use Harbor as this project's container registry.") + end + + def help + s_("HarborIntegration|After the Harbor integration is activated, global variables ‘$HARBOR_USERNAME’, ‘$HARBOR_PASSWORD’, ‘$HARBOR_URL’ and ‘$HARBOR_PROJECT’ will be created for CI/CD use.") + end + + class << self + def to_param + name.demodulize.downcase + end + + def supported_events + [] + end + + def supported_event_actions + [] + end + end + + def test(*_args) + client.ping + end + + def fields + [ + { + type: 'text', + name: 'url', + title: s_('HarborIntegration|Harbor URL'), + placeholder: 'https://demo.goharbor.io', + help: s_('HarborIntegration|Base URL of the Harbor instance.'), + required: true + }, + { + type: 'text', + name: 'project_name', + title: s_('HarborIntegration|Harbor project name'), + help: s_('HarborIntegration|The name of the project in Harbor.') + }, + { + type: 'text', + name: 'username', + title: s_('HarborIntegration|Harbor username'), + required: true + }, + { + type: 'text', + name: 'password', + title: s_('HarborIntegration|Harbor password'), + non_empty_password_title: s_('HarborIntegration|Enter Harbor password'), + non_empty_password_help: s_('HarborIntegration|Password for your Harbor username.'), + required: true + } + ] + end + + def ci_variables + return [] unless activated? + + [ + { key: 'HARBOR_URL', value: url }, + { key: 'HARBOR_PROJECT', value: project_name }, + { key: 'HARBOR_USERNAME', value: username }, + { key: 'HARBOR_PASSWORD', value: password, public: false, masked: true } + ] + end + + private + + def client + @client ||= ::Gitlab::Harbor::Client.new(self) + end + + def reset_username_and_password + if url_changed? && !password_touched? + self.password = nil + end + + if url_changed? && !username_touched? + self.username = nil + end + end + end +end diff --git a/app/models/integrations/irker.rb b/app/models/integrations/irker.rb index cea4aa2038d..116d1fb233d 100644 --- a/app/models/integrations/irker.rb +++ b/app/models/integrations/irker.rb @@ -4,8 +4,6 @@ require 'uri' module Integrations class Irker < Integration - include ActionView::Helpers::UrlHelper - prop_accessor :server_host, :server_port, :default_irc_uri prop_accessor :recipients, :channels boolean_accessor :colorize_messages @@ -44,7 +42,7 @@ module Integrations end def fields - recipients_docs_link = link_to s_('IrkerService|How to enter channels or users?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/irker', anchor: 'enter-irker-recipients'), target: '_blank', rel: 'noopener noreferrer' + recipients_docs_link = ActionController::Base.helpers.link_to s_('IrkerService|How to enter channels or users?'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/irker', anchor: 'enter-irker-recipients'), target: '_blank', rel: 'noopener noreferrer' [ { type: 'text', name: 'server_host', placeholder: 'localhost', title: s_('IrkerService|Server host (optional)'), help: s_('IrkerService|irker daemon hostname (defaults to localhost).') }, @@ -61,7 +59,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/irker', anchor: 'set-up-an-irker-daemon'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/irker', anchor: 'set-up-an-irker-daemon'), target: '_blank', rel: 'noopener noreferrer' s_('IrkerService|Send update messages to an irker server. Before you can use this, you need to set up the irker daemon. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/jenkins.rb b/app/models/integrations/jenkins.rb index 5ea92170c26..32f11ee23eb 100644 --- a/app/models/integrations/jenkins.rb +++ b/app/models/integrations/jenkins.rb @@ -3,7 +3,7 @@ module Integrations class Jenkins < BaseCi include HasWebHook - include ActionView::Helpers::UrlHelper + prepend EnableSslVerification extend Gitlab::Utils::Override @@ -65,7 +65,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('integration/jenkins'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('integration/jenkins'), target: '_blank', rel: 'noopener noreferrer' s_('Run CI/CD pipelines with Jenkins when you push to a repository, or when a merge request is created, updated, or merged. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 966ad07afad..74ece57000f 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -15,6 +15,9 @@ module Integrations ATLASSIAN_REFERRER_GITLAB_COM = { atlOrigin: 'eyJpIjoiY2QyZTJiZDRkNGZhNGZlMWI3NzRkNTBmZmVlNzNiZTkiLCJwIjoianN3LWdpdGxhYi1pbnQifQ' }.freeze ATLASSIAN_REFERRER_SELF_MANAGED = { atlOrigin: 'eyJpIjoiYjM0MTA4MzUyYTYxNDVkY2IwMzVjOGQ3ZWQ3NzMwM2QiLCJwIjoianN3LWdpdGxhYlNNLWludCJ9' }.freeze + SECTION_TYPE_JIRA_TRIGGER = 'jira_trigger' + SECTION_TYPE_JIRA_ISSUES = 'jira_issues' + validates :url, public_url: true, presence: true, if: :activated? validates :api_url, public_url: true, allow_blank: true validates :username, presence: true, if: :activated? @@ -28,11 +31,6 @@ module Integrations # We should use username/password for Jira Server and email/api_token for Jira Cloud, # for more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/49936. - # TODO: we can probably just delegate as part of - # https://gitlab.com/gitlab-org/gitlab/issues/29404 - data_field :username, :password, :url, :api_url, :jira_issue_transition_automatic, :jira_issue_transition_id, :project_key, :issues_enabled, - :vulnerabilities_enabled, :vulnerabilities_issuetype - before_validation :reset_password after_commit :update_deployment_type, on: [:create, :update], if: :update_deployment_type? @@ -41,16 +39,50 @@ module Integrations all_details: 2 } + self.field_storage = :data_fields + + field :url, + section: SECTION_TYPE_CONNECTION, + required: true, + title: -> { s_('JiraService|Web URL') }, + help: -> { s_('JiraService|Base URL of the Jira instance.') }, + placeholder: 'https://jira.example.com' + + field :api_url, + section: SECTION_TYPE_CONNECTION, + title: -> { s_('JiraService|Jira API URL') }, + help: -> { s_('JiraService|If different from Web URL.') } + + field :username, + section: SECTION_TYPE_CONNECTION, + required: true, + title: -> { s_('JiraService|Username or Email') }, + help: -> { s_('JiraService|Use a username for server version and an email for cloud version.') } + + field :password, + section: SECTION_TYPE_CONNECTION, + required: true, + title: -> { s_('JiraService|Password or API token') }, + non_empty_password_title: -> { s_('JiraService|Enter new password or API token') }, + non_empty_password_help: -> { s_('JiraService|Leave blank to use your current password or API token.') }, + help: -> { s_('JiraService|Use a password for server version and an API token for cloud version.') } + + # TODO: we can probably just delegate as part of + # https://gitlab.com/gitlab-org/gitlab/issues/29404 + # These fields are API only, so no field definition is required. + data_field :jira_issue_transition_automatic + data_field :jira_issue_transition_id + data_field :project_key + data_field :issues_enabled + data_field :vulnerabilities_enabled + data_field :vulnerabilities_issuetype + # When these are false GitLab does not create cross reference # comments on Jira except when an issue gets transitioned. def self.supported_events %w(commit merge_request) end - def self.supported_event_actions - %w(comment) - end - # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 def self.reference_pattern(only_long: true) @reference_pattern ||= /(?<issue>\b#{Gitlab::Regex.jira_issue_key_regex})/ @@ -111,8 +143,8 @@ module Integrations end def help - jira_doc_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_url('integration/jira/index.html') } - s_("JiraService|You need to configure Jira before enabling this integration. For more details, read the %{jira_doc_link_start}Jira integration documentation%{link_end}.") % { jira_doc_link_start: jira_doc_link_start, link_end: '</a>'.html_safe } + jira_doc_link_start = '<a href="%{url}">'.html_safe % { url: help_page_url('integration/jira/index.html') } + s_("JiraService|You must configure Jira before enabling this integration. %{jira_doc_link_start}Learn more.%{link_end}") % { jira_doc_link_start: jira_doc_link_start, link_end: '</a>'.html_safe } end def title @@ -127,39 +159,32 @@ module Integrations 'jira' end - def fields - [ - { - type: 'text', - name: 'url', - title: s_('JiraService|Web URL'), - placeholder: 'https://jira.example.com', - help: s_('JiraService|Base URL of the Jira instance.'), - required: true - }, - { - type: 'text', - name: 'api_url', - title: s_('JiraService|Jira API URL'), - help: s_('JiraService|If different from Web URL.') - }, + def sections + jira_issues_link_start = '<a href="%{url}">'.html_safe % { url: help_page_url('integration/jira/issues.html') } + + sections = [ { - type: 'text', - name: 'username', - title: s_('JiraService|Username or Email'), - help: s_('JiraService|Use a username for server version and an email for cloud version.'), - required: true + type: SECTION_TYPE_CONNECTION, + title: s_('Integrations|Connection details'), + description: help }, { - type: 'password', - name: 'password', - title: s_('JiraService|Password or API token'), - non_empty_password_title: s_('JiraService|Enter new password or API token'), - non_empty_password_help: s_('JiraService|Leave blank to use your current password or API token.'), - help: s_('JiraService|Use a password for server version and an API token for cloud version.'), - required: true + type: SECTION_TYPE_JIRA_TRIGGER, + title: _('Trigger'), + description: s_('JiraService|When a Jira issue is mentioned in a commit or merge request, a remote link and comment (if enabled) will be created.') } ] + + # Jira issues is currently only configurable on the project level. + if project_level? + sections.push({ + type: SECTION_TYPE_JIRA_ISSUES, + title: _('Issues'), + description: s_('JiraService|Work on Jira issues without leaving GitLab. Add a Jira menu to access a read-only list of your Jira issues. %{jira_issues_link_start}Learn more.%{link_end}') % { jira_issues_link_start: jira_issues_link_start, link_end: '</a>'.html_safe } + }) + end + + sections end def web_url(path = nil, **params) @@ -180,17 +205,12 @@ module Integrations url.to_s end - override :project_url - def project_url - web_url - end + alias_method :project_url, :web_url - override :issues_url def issues_url web_url('browse/:id') end - override :new_issue_url def new_issue_url web_url('secure/CreateIssue!default.jspa') end diff --git a/app/models/integrations/mattermost.rb b/app/models/integrations/mattermost.rb index 07a5086b8e9..d9ccbb7ea34 100644 --- a/app/models/integrations/mattermost.rb +++ b/app/models/integrations/mattermost.rb @@ -3,7 +3,6 @@ module Integrations class Mattermost < BaseChatNotification include SlackMattermostNotifier - include ActionView::Helpers::UrlHelper def title s_('Mattermost notifications') @@ -18,7 +17,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/mattermost'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/mattermost'), target: '_blank', rel: 'noopener noreferrer' s_('Send notifications about project events to Mattermost channels. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/pivotaltracker.rb b/app/models/integrations/pivotaltracker.rb index 24cfd51eb55..5b9ac023b7e 100644 --- a/app/models/integrations/pivotaltracker.rb +++ b/app/models/integrations/pivotaltracker.rb @@ -2,7 +2,6 @@ module Integrations class Pivotaltracker < Integration - include ActionView::Helpers::UrlHelper API_ENDPOINT = 'https://www.pivotaltracker.com/services/v5/source_commits' prop_accessor :token, :restrict_to_branch @@ -17,7 +16,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/pivotal_tracker'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/pivotal_tracker'), target: '_blank', rel: 'noopener noreferrer' s_('Add commit messages as comments to Pivotal Tracker stories. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index 5746343c31c..2e275dab91b 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -115,7 +115,6 @@ module Integrations end def prometheus_available? - return false if template? return false unless project project.all_clusters.enabled.eager_load(:integration_prometheus).any? do |cluster| diff --git a/app/models/integrations/redmine.rb b/app/models/integrations/redmine.rb index 990b538f294..bc2a64b0848 100644 --- a/app/models/integrations/redmine.rb +++ b/app/models/integrations/redmine.rb @@ -2,7 +2,8 @@ module Integrations class Redmine < BaseIssueTracker - include ActionView::Helpers::UrlHelper + include Integrations::HasIssueTrackerFields + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? def title @@ -14,7 +15,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/redmine'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/redmine'), target: '_blank', rel: 'noopener noreferrer' s_('IssueTracker|Use Redmine as the issue tracker. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/webex_teams.rb b/app/models/integrations/webex_teams.rb index 7660eda6f83..345dd98cbc1 100644 --- a/app/models/integrations/webex_teams.rb +++ b/app/models/integrations/webex_teams.rb @@ -2,8 +2,6 @@ module Integrations class WebexTeams < BaseChatNotification - include ActionView::Helpers::UrlHelper - def title s_("WebexTeamsService|Webex Teams") end @@ -17,7 +15,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/webex_teams'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/webex_teams'), target: '_blank', rel: 'noopener noreferrer' s_("WebexTeamsService|Send notifications about project events to a Webex Teams conversation. %{docs_link}") % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/youtrack.rb b/app/models/integrations/youtrack.rb index 10531717f11..ab6e1da27f8 100644 --- a/app/models/integrations/youtrack.rb +++ b/app/models/integrations/youtrack.rb @@ -2,7 +2,7 @@ module Integrations class Youtrack < BaseIssueTracker - include ActionView::Helpers::UrlHelper + include Integrations::HasIssueTrackerFields validates :project_url, :issues_url, presence: true, public_url: true, if: :activated? @@ -24,7 +24,7 @@ module Integrations end def help - docs_link = link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/youtrack'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to _('Learn more.'), Rails.application.routes.url_helpers.help_page_url('user/project/integrations/youtrack'), target: '_blank', rel: 'noopener noreferrer' s_("IssueTracker|Use YouTrack as this project's issue tracker. %{docs_link}").html_safe % { docs_link: docs_link.html_safe } end diff --git a/app/models/integrations/zentao.rb b/app/models/integrations/zentao.rb index 493d42cc40b..c33df465fde 100644 --- a/app/models/integrations/zentao.rb +++ b/app/models/integrations/zentao.rb @@ -11,7 +11,6 @@ module Integrations validates :api_token, presence: true, if: :activated? validates :zentao_product_xid, presence: true, if: :activated? - # License Level: EEP_FEATURES def self.issues_license_available?(project) project&.licensed_feature_available?(:zentao_issues_integration) end @@ -48,10 +47,6 @@ module Integrations %w() end - def self.supported_event_actions - %w() - end - def fields [ { diff --git a/app/models/issue.rb b/app/models/issue.rb index 68ea6cb3abc..75727fff2cd 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -24,6 +24,7 @@ class Issue < ApplicationRecord include Todoable include FromUnion include EachBatch + include PgFullTextSearchable extend ::Gitlab::Utils::Override @@ -77,6 +78,7 @@ class Issue < ApplicationRecord end end + has_one :search_data, class_name: 'Issues::SearchData' has_one :issuable_severity has_one :sentry_issue has_one :alert_management_alert, class_name: 'AlertManagement::Alert' @@ -102,6 +104,8 @@ class Issue < ApplicationRecord alias_attribute :external_author, :service_desk_reply_to + pg_full_text_searchable columns: [{ name: 'title', weight: 'A' }, { name: 'description', weight: 'B' }] + scope :in_projects, ->(project_ids) { where(project_id: project_ids) } scope :not_in_projects, ->(project_ids) { where.not(project_id: project_ids) } @@ -233,6 +237,11 @@ class Issue < ApplicationRecord def order_upvotes_asc reorder(upvotes_count: :asc) end + + override :pg_full_text_search + def pg_full_text_search(search_term) + super.where('issue_search_data.project_id = issues.project_id') + end end def next_object_by_relative_position(ignoring: nil, order: :asc) @@ -611,6 +620,11 @@ class Issue < ApplicationRecord private + override :persist_pg_full_text_search_vector + def persist_pg_full_text_search_vector(search_vector) + Issues::SearchData.upsert({ project_id: project_id, issue_id: id, search_vector: search_vector }, unique_by: %i(project_id issue_id)) + end + def spammable_attribute_changed? title_changed? || description_changed? || diff --git a/app/models/issue_link.rb b/app/models/issue_link.rb index 920586cc1ba..1bd34aa0083 100644 --- a/app/models/issue_link.rb +++ b/app/models/issue_link.rb @@ -2,46 +2,17 @@ class IssueLink < ApplicationRecord include FromUnion + include IssuableLink belongs_to :source, class_name: 'Issue' belongs_to :target, class_name: 'Issue' - validates :source, presence: true - validates :target, presence: true - validates :source, uniqueness: { scope: :target_id, message: 'is already related' } - validate :check_self_relation - validate :check_opposite_relation - scope :for_source_issue, ->(issue) { where(source_id: issue.id) } scope :for_target_issue, ->(issue) { where(target_id: issue.id) } - TYPE_RELATES_TO = 'relates_to' - TYPE_BLOCKS = 'blocks' - # we don't store is_blocked_by in the db but need it for displaying the relation - # from the target (used in IssueLink.inverse_link_type) - TYPE_IS_BLOCKED_BY = 'is_blocked_by' - - enum link_type: { TYPE_RELATES_TO => 0, TYPE_BLOCKS => 1 } - - def self.inverse_link_type(type) - type - end - - private - - def check_self_relation - return unless source && target - - if source == target - errors.add(:source, 'cannot be related to itself') - end - end - - def check_opposite_relation - return unless source && target - - if IssueLink.find_by(source: target, target: source) - errors.add(:source, 'is already related to this issue') + class << self + def issuable_type + :issue end end end diff --git a/app/models/issues/search_data.rb b/app/models/issues/search_data.rb new file mode 100644 index 00000000000..0eda292796d --- /dev/null +++ b/app/models/issues/search_data.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Issues + class SearchData < ApplicationRecord + extend SuppressCompositePrimaryKeyWarning + + self.table_name = 'issue_search_data' + + belongs_to :issue + end +end diff --git a/app/models/label.rb b/app/models/label.rb index 0ebbb5b9bd3..4c9f071f43a 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -12,8 +12,9 @@ class Label < ApplicationRecord cache_markdown_field :description, pipeline: :single_line - DEFAULT_COLOR = '#6699cc' + DEFAULT_COLOR = ::Gitlab::Color.of('#6699cc') + attribute :color, ::Gitlab::Database::Type::Color.new default_value_for :color, DEFAULT_COLOR has_many :lists, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -22,9 +23,9 @@ class Label < ApplicationRecord has_many :issues, through: :label_links, source: :target, source_type: 'Issue' has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' - before_validation :strip_whitespace_from_title_and_color + before_validation :strip_whitespace_from_title - validates :color, color: true, allow_blank: false + validates :color, color: true, presence: true # Don't allow ',' for label titles validates :title, presence: true, format: { with: /\A[^,]+\z/ } @@ -212,15 +213,23 @@ class Label < ApplicationRecord end def text_color - LabelsHelper.text_color_for_bg(self.color) + color.contrast end def title=(value) - write_attribute(:title, sanitize_value(value)) if value.present? + if value.blank? + super + else + write_attribute(:title, sanitize_value(value)) + end end def description=(value) - write_attribute(:description, sanitize_value(value)) if value.present? + if value.blank? + super + else + write_attribute(:description, sanitize_value(value)) + end end ## @@ -285,8 +294,8 @@ class Label < ApplicationRecord CGI.unescapeHTML(Sanitize.clean(value.to_s)) end - def strip_whitespace_from_title_and_color - %w(color title).each { |attr| self[attr] = self[attr]&.strip } + def strip_whitespace_from_title + self[:title] = title&.strip end end diff --git a/app/models/lfs_download_object.rb b/app/models/lfs_download_object.rb index 319499fd1b7..3df6742fbc9 100644 --- a/app/models/lfs_download_object.rb +++ b/app/models/lfs_download_object.rb @@ -4,6 +4,7 @@ class LfsDownloadObject include ActiveModel::Validations attr_accessor :oid, :size, :link, :headers + delegate :sanitized_url, :credentials, to: :sanitized_uri validates :oid, format: { with: /\A\h{64}\z/ } diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 3a449055bc1..3e19f294253 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -94,9 +94,9 @@ class ProjectMember < Member override :access_level_inclusion def access_level_inclusion - return if access_level.in?(Gitlab::Access.values) - - errors.add(:access_level, "is not included in the list") + unless access_level.in?(Gitlab::Access.all_values) + errors.add(:access_level, "is not included in the list") + end end override :refresh_member_authorized_projects diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 29540cbde2f..854325e1fcd 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1016,8 +1016,24 @@ class MergeRequest < ApplicationRecord merge_request_diff.persisted? || create_merge_request_diff end - def create_merge_request_diff + def eager_fetch_ref! + return unless valid? + + # has_internal_id normally attempts to allocate the iid in the + # before_create hook, but we need the iid to be available before + # that to fetch the ref into the target project. + track_target_project_iid! + ensure_target_project_iid! + fetch_ref! + # Prevent the after_create hook from fetching the source branch again. + @skip_fetch_ref = true + end + + def create_merge_request_diff + # Callers such as MergeRequests::BuildService may not call eager_fetch_ref!. Just + # in case they haven't, we fetch the ref. + fetch_ref! unless skip_fetch_ref # n+1: https://gitlab.com/gitlab-org/gitlab/-/issues/19377 Gitlab::GitalyClient.allow_n_plus_1_calls do @@ -1136,15 +1152,20 @@ class MergeRequest < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def mergeable_state?(skip_ci_check: false, skip_discussions_check: false) - return false unless open? - return false if work_in_progress? - return false if broken? - return false unless skip_discussions_check || mergeable_discussions_state? - if Feature.enabled?(:improved_mergeability_checks, self.project, default_enabled: :yaml) - additional_checks = MergeRequests::Mergeability::RunChecksService.new(merge_request: self, params: { skip_ci_check: skip_ci_check }) + additional_checks = MergeRequests::Mergeability::RunChecksService.new( + merge_request: self, + params: { + skip_ci_check: skip_ci_check, + skip_discussions_check: skip_discussions_check + } + ) additional_checks.execute.all?(&:success?) else + return false unless open? + return false if draft? + return false if broken? + return false unless skip_discussions_check || mergeable_discussions_state? return false unless skip_ci_check || mergeable_ci_state? true @@ -1921,10 +1942,18 @@ class MergeRequest < ApplicationRecord merge_request_assignees.find_by(user_id: user.id) end + def merge_request_assignees_with(user_ids) + merge_request_assignees.where(user_id: user_ids) + end + def find_reviewer(user) merge_request_reviewers.find_by(user_id: user.id) end + def merge_request_reviewers_with(user_ids) + merge_request_reviewers.where(user_id: user_ids) + end + def enabled_reports { sast: report_type_enabled?(:sast), @@ -1950,6 +1979,8 @@ class MergeRequest < ApplicationRecord private + attr_accessor :skip_fetch_ref + def set_draft_status self.draft = draft? end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 2c95cc2672c..86da29dd27a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -35,6 +35,7 @@ class Milestone < ApplicationRecord scope :with_api_entity_associations, -> { preload(project: [:project_feature, :route, namespace: :route]) } scope :order_by_dates_and_title, -> { order(due_date: :asc, start_date: :asc, title: :asc) } + validates :title, presence: true validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } validate :uniqueness_of_title, if: :title_changed? diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 5c55f4d3def..ffaeb2071f6 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -117,6 +117,7 @@ class Namespace < ApplicationRecord before_create :sync_share_with_group_lock_with_parent before_update :sync_share_with_group_lock_with_parent, if: :parent_changed? after_update :force_share_with_group_lock_on_descendants, if: -> { saved_change_to_share_with_group_lock? && share_with_group_lock? } + after_update :expire_first_auto_devops_config_cache, if: -> { saved_change_to_auto_devops_enabled? } # Legacy Storage specific hooks @@ -401,7 +402,11 @@ class Namespace < ApplicationRecord return { scope: :group, status: auto_devops_enabled } unless auto_devops_enabled.nil? strong_memoize(:first_auto_devops_config) do - if has_parent? + if has_parent? && cache_first_auto_devops_config? + Rails.cache.fetch(first_auto_devops_config_cache_key_for(id), expires_in: 1.day) do + parent.first_auto_devops_config + end + elsif has_parent? parent.first_auto_devops_config else { scope: :instance, status: Gitlab::CurrentSettings.auto_devops_enabled? } @@ -509,10 +514,6 @@ class Namespace < ApplicationRecord Feature.enabled?(:block_issue_repositioning, self, type: :ops, default_enabled: :yaml) end - def project_namespace_creation_enabled? - Feature.enabled?(:create_project_namespace_on_project_create, self, default_enabled: :yaml) - end - def storage_enforcement_date # should return something like Date.new(2022, 02, 03) # TBD: https://gitlab.com/gitlab-org/gitlab/-/issues/350632 @@ -621,6 +622,20 @@ class Namespace < ApplicationRecord .update_all(share_with_group_lock: true) end + def expire_first_auto_devops_config_cache + return unless cache_first_auto_devops_config? + + descendants_to_expire = self_and_descendants.as_ids + return if descendants_to_expire.load.empty? + + keys = descendants_to_expire.map { |group| first_auto_devops_config_cache_key_for(group.id) } + Rails.cache.delete_multi(keys) + end + + def cache_first_auto_devops_config? + ::Feature.enabled?(:namespaces_cache_first_auto_devops_config, default_enabled: :yaml) + end + def write_projects_repository_config all_projects.find_each do |project| project.set_full_path @@ -638,6 +653,13 @@ class Namespace < ApplicationRecord Namespaces::SyncEvent.enqueue_worker end end + + def first_auto_devops_config_cache_key_for(group_id) + return "namespaces:{first_auto_devops_config}:#{group_id}" unless sync_traversal_ids? + + # Use SHA2 of `traversal_ids` to account for moving a namespace within the same root ancestor hierarchy. + "namespaces:{#{traversal_ids.first}}:first_auto_devops_config:#{group_id}:#{Digest::SHA2.hexdigest(traversal_ids.join(' '))}" + end end Namespace.prepend_mod_with('Namespace') diff --git a/app/models/namespace/traversal_hierarchy.rb b/app/models/namespace/traversal_hierarchy.rb index 34086a8af5d..d2de85b5dd4 100644 --- a/app/models/namespace/traversal_hierarchy.rb +++ b/app/models/namespace/traversal_hierarchy.rb @@ -31,15 +31,16 @@ class Namespace # ActiveRecord. https://github.com/rails/rails/issues/13496 # Ideally it would be: # `incorrect_traversal_ids.update_all('traversal_ids = cte.traversal_ids')` - sql = """ - UPDATE namespaces - SET traversal_ids = cte.traversal_ids - FROM (#{recursive_traversal_ids}) as cte - WHERE namespaces.id = cte.id - AND namespaces.traversal_ids::bigint[] <> cte.traversal_ids - """ + sql = <<-SQL + UPDATE namespaces + SET traversal_ids = cte.traversal_ids + FROM (#{recursive_traversal_ids}) as cte + WHERE namespaces.id = cte.id + AND namespaces.traversal_ids::bigint[] <> cte.traversal_ids + SQL + Namespace.transaction do - @root.lock! + @root.lock!("FOR NO KEY UPDATE") Namespace.connection.exec_query(sql) end rescue ActiveRecord::Deadlocked diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 99a5b8cb063..1963745cf4d 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -44,22 +44,15 @@ module Namespaces included do before_update :lock_both_roots, if: -> { sync_traversal_ids? && parent_id_changed? } after_update :sync_traversal_ids, if: -> { sync_traversal_ids? && saved_change_to_parent_id? } - # sync traversal_ids on namespace create, which can happen quite early within a transaction, thus keeping the lock on root namespace record - # for a relatively long time, e.g. creating the project namespace when a project is being created. - after_create :sync_traversal_ids, if: -> { sync_traversal_ids? && !sync_traversal_ids_before_commit? } # This uses rails internal before_commit API to sync traversal_ids on namespace create, right before transaction is committed. # This helps reduce the time during which the root namespace record is locked to ensure updated traversal_ids are valid - before_commit :sync_traversal_ids, on: [:create], if: -> { sync_traversal_ids? && sync_traversal_ids_before_commit? } + before_commit :sync_traversal_ids, on: [:create], if: -> { sync_traversal_ids? } end def sync_traversal_ids? Feature.enabled?(:sync_traversal_ids, root_ancestor, default_enabled: :yaml) end - def sync_traversal_ids_before_commit? - Feature.enabled?(:sync_traversal_ids_before_commit, root_ancestor, default_enabled: :yaml) - end - def use_traversal_ids? return false unless Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 09d69a5f77a..0cac4c9143a 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -126,36 +126,26 @@ module Namespaces end def self_and_descendants_with_comparison_operators(include_self: true) - base = all.select( - :traversal_ids, - 'LEAD (namespaces.traversal_ids, 1) OVER (ORDER BY namespaces.traversal_ids ASC) next_traversal_ids' - ) + base = all.select(:traversal_ids) base_cte = Gitlab::SQL::CTE.new(:descendants_base_cte, base) namespaces = Arel::Table.new(:namespaces) # Bound the search space to ourselves (optional) and descendants. # - # WHERE (base_cte.next_traversal_ids IS NULL OR base_cte.next_traversal_ids > namespaces.traversal_ids) - # AND next_traversal_ids_sibling(base_cte.traversal_ids) > namespaces.traversal_ids + # WHERE next_traversal_ids_sibling(base_cte.traversal_ids) > namespaces.traversal_ids records = unscoped + .distinct + .with(base_cte.to_arel) .from([base_cte.table, namespaces]) - .where(base_cte.table[:next_traversal_ids].eq(nil).or(base_cte.table[:next_traversal_ids].gt(namespaces[:traversal_ids]))) .where(next_sibling_func(base_cte.table[:traversal_ids]).gt(namespaces[:traversal_ids])) # AND base_cte.traversal_ids <= namespaces.traversal_ids - records = if include_self - records.where(base_cte.table[:traversal_ids].lteq(namespaces[:traversal_ids])) - else - records.where(base_cte.table[:traversal_ids].lt(namespaces[:traversal_ids])) - end - - records_cte = Gitlab::SQL::CTE.new(:descendants_cte, records) - - unscoped - .unscope(where: [:type]) - .with(base_cte.to_arel, records_cte.to_arel) - .from(records_cte.alias_to(namespaces)) + if include_self + records.where(base_cte.table[:traversal_ids].lteq(namespaces[:traversal_ids])) + else + records.where(base_cte.table[:traversal_ids].lt(namespaces[:traversal_ids])) + end end def next_sibling_func(*args) diff --git a/app/models/note.rb b/app/models/note.rb index a84da066968..4f2e7ebe2c5 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -609,7 +609,6 @@ class Note < ApplicationRecord def show_outdated_changes? return false unless for_merge_request? - return false unless Feature.enabled?(:display_outdated_line_diff, noteable.source_project, default_enabled: :yaml) return false unless system? return false unless change_position&.line_range diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index fc7c348dfdb..ad8140ac684 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -49,6 +49,7 @@ class Packages::PackageFile < ApplicationRecord scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) } scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) } scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) } + scope :order_id_asc, -> { order(id: :asc) } scope :for_rubygem_with_file_name, ->(project, file_name) do joins(:package).merge(project.packages.rubygems).with_file_name(file_name) diff --git a/app/models/packages/pypi/metadatum.rb b/app/models/packages/pypi/metadatum.rb index 2e4d61eaf53..ff247fedb59 100644 --- a/app/models/packages/pypi/metadatum.rb +++ b/app/models/packages/pypi/metadatum.rb @@ -6,7 +6,7 @@ class Packages::Pypi::Metadatum < ApplicationRecord belongs_to :package, -> { where(package_type: :pypi) }, inverse_of: :pypi_metadatum validates :package, presence: true - validates :required_python, length: { maximum: 255 }, allow_blank: true + validates :required_python, length: { maximum: 255 }, allow_nil: false validate :pypi_package_type diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 2f515f3443d..021ff789b13 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -34,6 +34,7 @@ class PersonalAccessToken < ApplicationRecord scope :order_expires_at_asc, -> { reorder(expires_at: :asc) } scope :order_expires_at_desc, -> { reorder(expires_at: :desc) } scope :project_access_token, -> { includes(:user).where(user: { user_type: :project_bot }) } + scope :owner_is_human, -> { includes(:user).where(user: { user_type: :human }) } validates :scopes, presence: true validate :validate_scopes diff --git a/app/models/preloaders/environments/deployment_preloader.rb b/app/models/preloaders/environments/deployment_preloader.rb index fcf892698bb..251d1837f19 100644 --- a/app/models/preloaders/environments/deployment_preloader.rb +++ b/app/models/preloaders/environments/deployment_preloader.rb @@ -21,11 +21,13 @@ module Preloaders def load_deployment_association(association_name, association_attributes) return unless environments.present? - union_arg = environments.inject([]) do |result, environment| - result << environment.association(association_name).scope - end - - union_sql = Deployment.from_union(union_arg).to_sql + # Not using Gitlab::SQL::Union as `order_by` in the SQL constructed is ignored. + # See: + # 1) https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/sql/union.rb#L7 + # 2) https://gitlab.com/gitlab-org/gitlab/-/issues/353966#note_860928647 + union_sql = environments.map do |environment| + "(#{environment.association(association_name).scope.to_sql})" + end.join(' UNION ') deployments = Deployment .from("(#{union_sql}) #{::Deployment.table_name}") @@ -34,8 +36,16 @@ module Preloaders deployments_by_environment_id = deployments.index_by(&:environment_id) environments.each do |environment| - environment.association(association_name).target = deployments_by_environment_id[environment.id] + associated_deployment = deployments_by_environment_id[environment.id] + + environment.association(association_name).target = associated_deployment environment.association(association_name).loaded! + + if associated_deployment + # `last?` in DeploymentEntity requires this environment to be loaded + associated_deployment.association(:environment).target = environment + associated_deployment.association(:environment).loaded! + end end end end diff --git a/app/models/project.rb b/app/models/project.rb index f89e616a5ca..155ebe88d33 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -38,7 +38,7 @@ class Project < ApplicationRecord include GitlabRoutingHelper include BulkMemberAccessLoad include RunnerTokenExpirationInterval - include RunnersTokenPrefixable + include BlocksUnsafeSerialization extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -196,6 +196,7 @@ class Project < ApplicationRecord has_one :external_wiki_integration, class_name: 'Integrations::ExternalWiki' has_one :flowdock_integration, class_name: 'Integrations::Flowdock' has_one :hangouts_chat_integration, class_name: 'Integrations::HangoutsChat' + has_one :harbor_integration, class_name: 'Integrations::Harbor' has_one :irker_integration, class_name: 'Integrations::Irker' has_one :jenkins_integration, class_name: 'Integrations::Jenkins' has_one :jira_integration, class_name: 'Integrations::Jira' @@ -344,22 +345,18 @@ class Project < ApplicationRecord has_many :stages, class_name: 'Ci::Stage', inverse_of: :project has_many :ci_refs, class_name: 'Ci::Ref', inverse_of: :project - # Ci::Build objects store data on the file system such as artifact files and - # build traces. Currently there's no efficient way of removing this data in - # bulk that doesn't involve loading the rows into memory. As a result we're - # still using `dependent: :destroy` here. has_many :pending_builds, class_name: 'Ci::PendingBuild' - has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :builds, class_name: 'Ci::Build', inverse_of: :project has_many :processables, class_name: 'Ci::Processable', inverse_of: :project - has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks + has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks, dependent: :restrict_with_error has_many :build_report_results, class_name: 'Ci::BuildReportResult', inverse_of: :project - has_many :job_artifacts, class_name: 'Ci::JobArtifact' - has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :project + has_many :job_artifacts, class_name: 'Ci::JobArtifact', dependent: :restrict_with_error + has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :project, dependent: :restrict_with_error has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' - has_many :secure_files, class_name: 'Ci::SecureFile' + has_many :secure_files, class_name: 'Ci::SecureFile', dependent: :restrict_with_error has_many :environments has_many :environments_for_dashboard, -> { from(with_rank.unfoldered.available, :environments).where('rank <= 3') }, class_name: 'Environment' has_many :deployments @@ -462,7 +459,7 @@ class Project < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team - delegate :add_guest, :add_reporter, :add_developer, :add_maintainer, :add_role, to: :team + delegate :add_guest, :add_reporter, :add_developer, :add_maintainer, :add_owner, :add_role, to: :team delegate :group_runners_enabled, :group_runners_enabled=, to: :ci_cd_settings, allow_nil: true delegate :root_ancestor, to: :namespace, allow_nil: true delegate :last_pipeline, to: :commit, allow_nil: true @@ -501,11 +498,15 @@ class Project < ApplicationRecord presence: true, project_path: true, length: { maximum: 255 } + validates :path, + format: { with: Gitlab::Regex.oci_repository_path_regex, + message: Gitlab::Regex.oci_repository_path_regex_message }, + if: :path_changed? validates :project_feature, presence: true validates :namespace, presence: true - validates :project_namespace, presence: true, on: :create, if: -> { self.namespace && self.root_namespace.project_namespace_creation_enabled? } + validates :project_namespace, presence: true, on: :create, if: -> { self.namespace } validates :project_namespace, presence: true, on: :update, if: -> { self.project_namespace_id_changed?(to: nil) } validates :name, uniqueness: { scope: :namespace_id } validates :import_url, public_url: { schemes: ->(project) { project.persisted? ? VALID_MIRROR_PROTOCOLS : VALID_IMPORT_PROTOCOLS }, @@ -529,6 +530,7 @@ class Project < ApplicationRecord # Scopes scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } + scope :not_hidden, -> { where(hidden: false) } scope :not_aimed_for_deletion, -> { where(marked_for_deletion_at: nil).without_deleted } scope :with_storage_feature, ->(feature) do @@ -1006,10 +1008,6 @@ class Project < ApplicationRecord Feature.enabled?(:unlink_fork_network_upon_visibility_decrease, self, default_enabled: true) end - def context_commits_enabled? - Feature.enabled?(:context_commits, self.group, default_enabled: :yaml) - end - # LFS and hashed repository storage are required for using Design Management. def design_management_enabled? lfs_enabled? && hashed_storage?(:repository) @@ -1565,14 +1563,17 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def execute_hooks(data, hooks_scope = :push_hooks) run_after_commit_or_now do - hooks.hooks_for(hooks_scope).select_active(hooks_scope, data).each do |hook| - hook.async_execute(data, hooks_scope.to_s) - end + triggered_hooks(hooks_scope, data).execute SystemHooksService.new.execute_hooks(data, hooks_scope) end end # rubocop: enable CodeReuse/ServiceClass + def triggered_hooks(hooks_scope, data) + triggered = ::Projects::TriggeredHooks.new(hooks_scope, data) + triggered.add_hooks(hooks) + end + def execute_integrations(data, hooks_scope = :push_hooks) # Call only service hooks that are active for this scope run_after_commit_or_now do @@ -1876,13 +1877,9 @@ class Project < ApplicationRecord ensure_runners_token! end - def runners_token_prefix - RUNNERS_TOKEN_PREFIX - end - override :format_runners_token def format_runners_token(token) - "#{runners_token_prefix}#{token}" + "#{RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX}#{token}" end def pages_deployed? @@ -1938,12 +1935,12 @@ class Project < ApplicationRecord .delete_all end - def mark_pages_as_deployed(artifacts_archive: nil) - ensure_pages_metadatum.update!(deployed: true, artifacts_archive: artifacts_archive) + def mark_pages_as_deployed + ensure_pages_metadatum.update!(deployed: true) end def mark_pages_as_not_deployed - ensure_pages_metadatum.update!(deployed: false, artifacts_archive: nil, pages_deployment: nil) + ensure_pages_metadatum.update!(deployed: false) end def update_pages_deployment!(deployment) @@ -2521,7 +2518,18 @@ class Project < ApplicationRecord end def access_request_approvers_to_be_notified - members.maintainers.connected_to_user.order_recent_sign_in.limit(Member::ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) + # For a personal project: + # The creator is added as a member with `Owner` access level, starting from GitLab 14.8 + # The creator was added as a member with `Maintainer` access level, before GitLab 14.8 + # So, to make sure access requests for all personal projects work as expected, + # we need to filter members with the scope `owners_and_maintainers`. + access_request_approvers = if personal? + members.owners_and_maintainers + else + members.maintainers + end + + access_request_approvers.connected_to_user.order_recent_sign_in.limit(Member::ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end def pages_lookup_path(trim_prefix: nil, domain: nil) @@ -2817,6 +2825,10 @@ class Project < ApplicationRecord end end + def pending_delete_or_hidden? + pending_delete? || hidden? + end + private # overridden in EE @@ -2838,7 +2850,9 @@ class Project < ApplicationRecord if @topic_list != self.topic_list self.topics.delete_all - self.topics = @topic_list.map { |topic| Projects::Topic.find_or_create_by(name: topic) } + self.topics = @topic_list.map do |topic| + Projects::Topic.where('lower(name) = ?', topic.downcase).order(total_projects_count: :desc).first_or_create(name: topic) + end end @topic_list = nil @@ -3010,16 +3024,15 @@ class Project < ApplicationRecord end def ensure_project_namespace_in_sync - # create project_namespace when project is created if create_project_namespace_on_project_create FF is enabled + # create project_namespace when project is created build_project_namespace if project_namespace_creation_enabled? - # regardless of create_project_namespace_on_project_create FF we need - # to keep project and project namespace in sync if there is one + # we need to keep project and project namespace in sync if there is one sync_attributes(project_namespace) if sync_project_namespace? end def project_namespace_creation_enabled? - new_record? && !project_namespace && self.namespace && self.root_namespace.project_namespace_creation_enabled? + new_record? && !project_namespace && self.namespace end def sync_project_namespace? diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index c76332b21cd..5c6fdec16ca 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -9,7 +9,7 @@ class ProjectAuthorization < ApplicationRecord validates :project, presence: true validates :access_level, inclusion: { in: Gitlab::Access.all_values }, presence: true - validates :user, uniqueness: { scope: [:project, :access_level] }, presence: true + validates :user, uniqueness: { scope: :project }, presence: true def self.select_from_union(relations) from_union(relations) diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb index d374ee120d1..3b514d5c5ff 100644 --- a/app/models/project_import_data.rb +++ b/app/models/project_import_data.rb @@ -14,7 +14,12 @@ class ProjectImportData < ApplicationRecord insecure_mode: true, algorithm: 'aes-256-cbc' - serialize :data, JSON # rubocop:disable Cop/ActiveRecordSerialize + # NOTE + # We are serializing a project as `data` in an "unsafe" way here + # because the credentials are necessary for a successful import. + # This is safe because the serialization is only going between rails + # and the database, never to any end users. + serialize :data, Serializers::UnsafeJson # rubocop:disable Cop/ActiveRecordSerialize validates :project, presence: true diff --git a/app/models/project_pages_metadatum.rb b/app/models/project_pages_metadatum.rb index 58dbac9057f..dc1e9319340 100644 --- a/app/models/project_pages_metadatum.rb +++ b/app/models/project_pages_metadatum.rb @@ -4,11 +4,13 @@ class ProjectPagesMetadatum < ApplicationRecord extend SuppressCompositePrimaryKeyWarning include EachBatch + include IgnorableColumns self.primary_key = :project_id + ignore_columns :artifacts_archive_id, remove_with: '15.0', remove_after: '2022-04-22' + belongs_to :project, inverse_of: :pages_metadatum - belongs_to :artifacts_archive, class_name: 'Ci::JobArtifact' belongs_to :pages_deployment scope :deployed, -> { where(deployed: true) } diff --git a/app/models/project_team.rb b/app/models/project_team.rb index c3c7508df9f..4b89d95c1a3 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -23,6 +23,10 @@ class ProjectTeam add_user(user, :maintainer, current_user: current_user) end + def add_owner(user, current_user: nil) + add_user(user, :owner, current_user: current_user) + end + def add_role(user, role, current_user: nil) public_send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend end @@ -103,7 +107,9 @@ class ProjectTeam if group group.owners else - [project.owner] + # workaround until we migrate Project#owners to have membership with + # OWNER access level + Array.wrap(fetch_members(Gitlab::Access::OWNER)) | Array.wrap(project.owner) end end @@ -173,7 +179,9 @@ class ProjectTeam # # Returns a Hash mapping user ID -> maximum access level. def max_member_access_for_user_ids(user_ids) - project.max_member_access_for_resource_ids(User, user_ids) do |user_ids| + Gitlab::SafeRequestLoader.execute(resource_key: project.max_member_access_for_resource_key(User), + resource_ids: user_ids, + default_value: Gitlab::Access::NO_ACCESS) do |user_ids| project.project_authorizations .where(user: user_ids) .group(:user_id) @@ -190,31 +198,15 @@ class ProjectTeam end def contribution_check_for_user_ids(user_ids) - user_ids = user_ids.uniq - key = "contribution_check_for_users:#{project.id}" - - Gitlab::SafeRequestStore[key] ||= {} - contributors = Gitlab::SafeRequestStore[key] || {} - - user_ids -= contributors.keys - - return contributors if user_ids.empty? - - resource_contributors = project.merge_requests - .merged - .where(author_id: user_ids, target_branch: project.default_branch.to_s) - .pluck(:author_id) - .product([true]).to_h - - contributors.merge!(resource_contributors) - - missing_resource_ids = user_ids - resource_contributors.keys - - missing_resource_ids.each do |resource_id| - contributors[resource_id] = false + Gitlab::SafeRequestLoader.execute(resource_key: "contribution_check_for_users:#{project.id}", + resource_ids: user_ids, + default_value: false) do |user_ids| + project.merge_requests + .merged + .where(author_id: user_ids, target_branch: project.default_branch.to_s) + .pluck(:author_id) + .product([true]).to_h end - - contributors end def contributor?(user_id) diff --git a/app/models/projects/build_artifacts_size_refresh.rb b/app/models/projects/build_artifacts_size_refresh.rb new file mode 100644 index 00000000000..afb67b79f0d --- /dev/null +++ b/app/models/projects/build_artifacts_size_refresh.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Projects + class BuildArtifactsSizeRefresh < ApplicationRecord + include BulkInsertSafe + + STALE_WINDOW = 3.days + + self.table_name = 'project_build_artifacts_size_refreshes' + + belongs_to :project + + validates :project, presence: true + + STATES = { + created: 1, + running: 2, + pending: 3 + }.freeze + + state_machine :state, initial: :created do + # created -> running <-> pending + state :created, value: STATES[:created] + state :running, value: STATES[:running] + state :pending, value: STATES[:pending] + + event :process do + transition [:created, :pending, :running] => :running + end + + event :requeue do + transition running: :pending + end + + # set it only the first time we execute the refresh + before_transition created: :running do |refresh| + refresh.reset_project_statistics! + refresh.refresh_started_at = Time.zone.now + end + + before_transition running: any do |refresh, transition| + refresh.updated_at = Time.zone.now + end + + before_transition running: :pending do |refresh, transition| + refresh.last_job_artifact_id = transition.args.first + end + end + + scope :stale, -> { with_state(:running).where('updated_at < ?', STALE_WINDOW.ago) } + scope :remaining, -> { with_state(:created, :pending).or(stale) } + + def self.enqueue_refresh(projects) + now = Time.zone.now + + records = Array(projects).map do |project| + new(project: project, state: STATES[:created], created_at: now, updated_at: now) + end + + bulk_insert!(records, skip_duplicates: true) + end + + def self.process_next_refresh! + next_refresh = nil + + transaction do + next_refresh = remaining + .order(:state, :updated_at) + .lock('FOR UPDATE SKIP LOCKED') + .take + + next_refresh&.process! + end + + next_refresh + end + + def reset_project_statistics! + statistics = project.statistics + statistics.update!(build_artifacts_size: 0) + statistics.clear_counter!(:build_artifacts_size) + end + + def next_batch(limit:) + project.job_artifacts.select(:id, :size) + .where('created_at <= ? AND id > ?', refresh_started_at, last_job_artifact_id.to_i) + .order(:created_at) + .limit(limit) + end + end +end diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb index 78bc2df2e1e..b42b03f0618 100644 --- a/app/models/projects/topic.rb +++ b/app/models/projects/topic.rb @@ -7,18 +7,19 @@ module Projects include Avatarable include Gitlab::SQL::Pattern - validates :name, presence: true, uniqueness: true, length: { maximum: 255 } + validates :name, presence: true, length: { maximum: 255 } + validates :name, uniqueness: { case_sensitive: false }, if: :name_changed? validates :description, length: { maximum: 1024 } has_many :project_topics, class_name: 'Projects::ProjectTopic' has_many :projects, through: :project_topics - scope :order_by_total_projects_count, -> { order(total_projects_count: :desc).order(id: :asc) } + scope :order_by_non_private_projects_count, -> { order(non_private_projects_count: :desc).order(id: :asc) } scope :reorder_by_similarity, -> (search) do order_expression = Gitlab::Database::SimilarityScore.build_expression(search: search, rules: [ { column: arel_table['name'] } ]) - reorder(order_expression.desc, arel_table['total_projects_count'].desc, arel_table['id']) + reorder(order_expression.desc, arel_table['non_private_projects_count'].desc, arel_table['id']) end class << self diff --git a/app/models/projects/triggered_hooks.rb b/app/models/projects/triggered_hooks.rb new file mode 100644 index 00000000000..e3aa3d106b7 --- /dev/null +++ b/app/models/projects/triggered_hooks.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Projects + class TriggeredHooks + def initialize(scope, data) + @scope = scope + @data = data + @relations = [] + end + + def add_hooks(relation) + @relations << relation + self + end + + def execute + # Assumes that the relations implement TriggerableHooks + @relations.each do |hooks| + hooks.hooks_for(@scope).select_active(@scope, @data).each do |hook| + hook.async_execute(@data, @scope.to_s) + end + end + end + end +end diff --git a/app/models/release.rb b/app/models/release.rb index 0fda6940249..c6c0920c4d0 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -5,6 +5,8 @@ class Release < ApplicationRecord include CacheMarkdownField include Importable include Gitlab::Utils::StrongMemoize + include EachBatch + include FromUnion cache_markdown_field :description @@ -24,6 +26,8 @@ class Release < ApplicationRecord before_create :set_released_at validates :project, :tag, presence: true + validates :tag, uniqueness: { scope: :project_id } + validates :description, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT }, if: :description_changed? validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") } validates :links, nested_attributes_duplicates: { scope: :release, child_attributes: %i[name url filepath] } diff --git a/app/models/repository.rb b/app/models/repository.rb index be8e530c650..346478b6689 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -15,6 +15,7 @@ class Repository heads tags replace + #{REF_MERGE_REQUEST} #{REF_ENVIRONMENTS} #{REF_KEEP_AROUND} #{REF_PIPELINES} @@ -1084,10 +1085,10 @@ class Repository blob.data end - def create_if_not_exists + def create_if_not_exists(default_branch = nil) return if exists? - raw.create_repository + raw.create_repository(default_branch) after_create true diff --git a/app/models/snippet.rb b/app/models/snippet.rb index b04fca64c87..38aaeff5c9a 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -350,24 +350,10 @@ class Snippet < ApplicationRecord snippet_repository&.shard_name || Repository.pick_storage_shard end - # Repositories are created with a default branch. This branch - # can be different from the default branch set in the platform. - # This method changes the `HEAD` file to point to the existing - # default branch in case it's different. - def change_head_to_default_branch - return unless repository.exists? - # All snippets must have at least 1 file. Therefore, if - # `HEAD` is empty is because it's pointing to the wrong - # default branch - return unless repository.empty? || list_files('HEAD').empty? - - repository.raw_repository.write_ref('HEAD', "refs/heads/#{default_branch}") - end - def create_repository return if repository_exists? && snippet_repository - repository.create_if_not_exists + repository.create_if_not_exists(default_branch) track_snippet_repository(repository.storage) end diff --git a/app/models/storage/hashed.rb b/app/models/storage/hashed.rb index c61cd3b6b30..05e93f00912 100644 --- a/app/models/storage/hashed.rb +++ b/app/models/storage/hashed.rb @@ -3,6 +3,7 @@ module Storage class Hashed attr_accessor :container + delegate :gitlab_shell, :repository_storage, to: :container REPOSITORY_PATH_PREFIX = '@hashed' diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index 092e5249a3e..0d12a629b8e 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -3,6 +3,7 @@ module Storage class LegacyProject attr_accessor :project + delegate :namespace, :gitlab_shell, :repository_storage, to: :project def initialize(project) diff --git a/app/models/todo.rb b/app/models/todo.rb index dc436570f52..eb5d9965955 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -34,6 +34,8 @@ class Todo < ApplicationRecord ATTENTION_REQUESTED => :attention_requested }.freeze + ACTIONS_MULTIPLE_ALLOWED = [Todo::MENTIONED, Todo::DIRECTLY_ADDRESSED].freeze + belongs_to :author, class_name: "User" belongs_to :note belongs_to :project diff --git a/app/models/user.rb b/app/models/user.rb index 9cd238904ff..b3bdc2c1c42 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -16,7 +16,7 @@ class User < ApplicationRecord include FeatureGate include CreatedAtFilterable include BulkMemberAccessLoad - include BlocksJsonSerialization + include BlocksUnsafeSerialization include WithUploads include OptionallySearch include FromUnion @@ -135,6 +135,7 @@ class User < ApplicationRecord has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :webauthn_registrations has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :saved_replies, class_name: '::Users::SavedReply' has_one :user_synced_attributes_metadata, autosave: true has_one :aws_role, class_name: 'Aws::Role' @@ -276,24 +277,22 @@ class User < ApplicationRecord after_update :username_changed_hook, if: :saved_change_to_username? after_destroy :post_destroy_hook after_destroy :remove_key_cache - after_create :add_primary_email_to_emails!, if: :confirmed? - after_commit(on: :update) do - if previous_changes.key?('email') - # Add the old primary email to Emails if not added already - this should be removed - # after the background migration for MR https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70872/ has completed, - # as the primary email is now added to Emails upon confirmation - # Issue to remove that: https://gitlab.com/gitlab-org/gitlab/-/issues/344134 - previous_confirmed_at = previous_changes.key?('confirmed_at') ? previous_changes['confirmed_at'][0] : confirmed_at - previous_email = previous_changes[:email][0] - if previous_confirmed_at && !emails.exists?(email: previous_email) - # rubocop: disable CodeReuse/ServiceClass - Emails::CreateService.new(self, user: self, email: previous_email).execute(confirmed_at: previous_confirmed_at) - # rubocop: enable CodeReuse/ServiceClass - end + after_save if: -> { saved_change_to_email? && confirmed? } do + email_to_confirm = self.emails.find_by(email: self.email) - update_invalid_gpg_signatures + if email_to_confirm.present? + if skip_confirmation_period_expiry_check + email_to_confirm.force_confirm + else + email_to_confirm.confirm + end + else + add_primary_email_to_emails! end end + after_commit(on: :update) do + update_invalid_gpg_signatures if previous_changes.key?('email') + end after_initialize :set_projects_limit @@ -1692,6 +1691,12 @@ class User < ApplicationRecord end end + def attention_requested_open_merge_requests_count(force: false) + Rails.cache.fetch(attention_request_cache_key, force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do + MergeRequestsFinder.new(self, attention: self.username, state: 'opened', non_archived: true).execute.count + end + end + def assigned_open_issues_count(force: false) Rails.cache.fetch(['users', id, 'assigned_open_issues_count'], force: force, expires_in: COUNT_CACHE_VALIDITY_PERIOD) do IssuesFinder.new(self, assignee_id: self.id, state: 'opened', non_archived: true).execute.count @@ -1735,6 +1740,11 @@ class User < ApplicationRecord def invalidate_merge_request_cache_counts Rails.cache.delete(['users', id, 'assigned_open_merge_requests_count']) Rails.cache.delete(['users', id, 'review_requested_open_merge_requests_count']) + invalidate_attention_requested_count + end + + def invalidate_attention_requested_count + Rails.cache.delete(attention_request_cache_key) end def invalidate_todos_cache_counts @@ -1746,6 +1756,10 @@ class User < ApplicationRecord Rails.cache.delete(['users', id, 'personal_projects_count']) end + def attention_request_cache_key + ['users', id, 'attention_requested_open_merge_requests_count'] + end + # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth # flow means we don't call that automatically (and can't conveniently do so). # @@ -1846,7 +1860,9 @@ class User < ApplicationRecord # # Returns a Hash mapping project ID -> maximum access level. def max_member_access_for_project_ids(project_ids) - max_member_access_for_resource_ids(Project, project_ids) do |project_ids| + Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(Project), + resource_ids: project_ids, + default_value: Gitlab::Access::NO_ACCESS) do |project_ids| project_authorizations.where(project: project_ids) .group(:project_id) .maximum(:access_level) @@ -1861,7 +1877,9 @@ class User < ApplicationRecord # # Returns a Hash mapping project ID -> maximum access level. def max_member_access_for_group_ids(group_ids) - max_member_access_for_resource_ids(Group, group_ids) do |group_ids| + Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(Group), + resource_ids: group_ids, + default_value: Gitlab::Access::NO_ACCESS) do |group_ids| group_members.where(source: group_ids).group(:source_id).maximum(:access_level) end end @@ -1993,29 +2011,6 @@ class User < ApplicationRecord ci_job_token_scope.present? end - # override from Devise::Models::Confirmable - # - # Add the primary email to user.emails (or confirm it if it was already - # present) when the primary email is confirmed. - def confirm(args = {}) - saved = super(args) - return false unless saved - - email_to_confirm = self.emails.find_by(email: self.email) - - if email_to_confirm.present? - if skip_confirmation_period_expiry_check - email_to_confirm.force_confirm(args) - else - email_to_confirm.confirm(args) - end - else - add_primary_email_to_emails! - end - - saved - end - def user_project strong_memoize(:user_project) do personal_projects.find_by(path: username, visibility_level: Gitlab::VisibilityLevel::PUBLIC) @@ -2166,7 +2161,7 @@ class User < ApplicationRecord end def signup_email_invalid_message - self.new_record? ? _('is not allowed for sign-up.') : _('is not allowed.') + self.new_record? ? _('is not allowed for sign-up. Please use your regular email address.') : _('is not allowed. Please use your regular email address.') end def check_username_format diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 5c39e29a128..0922323e12b 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -42,7 +42,13 @@ module Users security_newsletter_callout: 39, verification_reminder: 40, # EE-only ci_deprecation_warning_for_types_keyword: 41, - security_training_feature_promotion: 42 # EE-only + security_training_feature_promotion: 42, # EE-only + storage_enforcement_banner_first_enforcement_threshold: 43, + storage_enforcement_banner_second_enforcement_threshold: 44, + storage_enforcement_banner_third_enforcement_threshold: 45, + storage_enforcement_banner_fourth_enforcement_threshold: 46, + attention_requests_top_nav: 47, + attention_requests_side_nav: 48 } validates :feature_name, diff --git a/app/models/users/credit_card_validation.rb b/app/models/users/credit_card_validation.rb index 556ee03605d..998a5deb0fd 100644 --- a/app/models/users/credit_card_validation.rb +++ b/app/models/users/credit_card_validation.rb @@ -8,7 +8,7 @@ module Users belongs_to :user - validates :holder_name, length: { maximum: 26 } + validates :holder_name, length: { maximum: 50 } validates :network, length: { maximum: 32 } validates :last_digits, allow_nil: true, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 9999 diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index 0dc449719ab..839be8d2a48 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -11,10 +11,10 @@ module Users enum feature_name: { invite_members_banner: 1, approaching_seat_count_threshold: 2, # EE-only - storage_enforcement_banner_first_enforcement_threshold: 43, - storage_enforcement_banner_second_enforcement_threshold: 44, - storage_enforcement_banner_third_enforcement_threshold: 45, - storage_enforcement_banner_fourth_enforcement_threshold: 46 + storage_enforcement_banner_first_enforcement_threshold: 3, + storage_enforcement_banner_second_enforcement_threshold: 4, + storage_enforcement_banner_third_enforcement_threshold: 5, + storage_enforcement_banner_fourth_enforcement_threshold: 6 } validates :group, presence: true diff --git a/app/models/users/saved_reply.rb b/app/models/users/saved_reply.rb new file mode 100644 index 00000000000..7737d826b05 --- /dev/null +++ b/app/models/users/saved_reply.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Users + class SavedReply < ApplicationRecord + self.table_name = 'saved_replies' + + belongs_to :user + + validates :user_id, :name, :content, presence: true + validates :name, + length: { maximum: 255 }, + uniqueness: { scope: [:user_id] }, + format: { + with: Gitlab::Regex.saved_reply_name_regex, + message: Gitlab::Regex.saved_reply_name_regex_message + } + validates :content, length: { maximum: 10000 } + end +end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index e114e30d589..622070abd88 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -87,8 +87,7 @@ class Wiki end def create_wiki_repository - repository.create_if_not_exists - change_head_to_default_branch + repository.create_if_not_exists(default_branch) raise CouldNotCreateWikiError unless repository_exists? rescue StandardError => err @@ -150,10 +149,10 @@ class Wiki # the page. # # Returns an initialized WikiPage instance or nil - def find_page(title, version = nil) + def find_page(title, version = nil, load_content: true) page_title, page_dir = page_title_and_dir(title) - if page = wiki.page(title: page_title, version: version, dir: page_dir) + if page = wiki.page(title: page_title, version: version, dir: page_dir, load_content: load_content) WikiPage.new(self, page) end end @@ -322,16 +321,6 @@ class Wiki def default_message(action, title) "#{user.username} #{action} page: #{title}" end - - def change_head_to_default_branch - # If the wiki has commits in the 'HEAD' branch means that the current - # HEAD is pointing to the right branch. If not, it could mean that either - # the repo has just been created or that 'HEAD' is pointing - # to the wrong branch and we need to rewrite it - return if repository.raw_repository.commit_count('HEAD') != 0 - - repository.raw_repository.write_ref('HEAD', "refs/heads/#{default_branch}") - end end Wiki.prepend_mod_with('Wiki') diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 3dbbbcdfe23..803b9781ac4 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -45,6 +45,7 @@ class WikiPage # The GitLab Wiki instance. attr_reader :wiki + delegate :container, to: :wiki # The raw Gitlab::Git::WikiPage instance. @@ -315,7 +316,6 @@ class WikiPage end def update_front_matter(attrs) - return unless Gitlab::WikiPages::FrontMatterParser.enabled?(container) return unless attrs.has_key?(:front_matter) fm_yaml = serialize_front_matter(attrs[:front_matter]) @@ -326,7 +326,7 @@ class WikiPage def parsed_content strong_memoize(:parsed_content) do - Gitlab::WikiPages::FrontMatterParser.new(raw_content, container).parse + Gitlab::WikiPages::FrontMatterParser.new(raw_content).parse end end @@ -404,3 +404,5 @@ class WikiPage }) end end + +WikiPage.prepend_mod diff --git a/app/models/work_item.rb b/app/models/work_item.rb index 99f05e4a181..557694da35a 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -7,4 +7,12 @@ class WorkItem < Issue def noteable_target_type_name 'issue' end + + private + + def record_create_action + super + + Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter.track_work_item_created_action(author: author) + end end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index 494c4f5abe4..080513b28e9 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -38,6 +38,7 @@ module WorkItems scope :default, -> { where(namespace: nil) } scope :order_by_name_asc, -> { order('LOWER(name)') } + scope :by_type, ->(base_type) { where(base_type: base_type) } def self.default_by_type(type) find_by(namespace_id: nil, base_type: type) |