diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 09:16:11 +0000 |
commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /app/models | |
parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) | |
download | gitlab-ce-edaa33dee2ff2f7ea3fac488d41558eb5f86d68c.tar.gz |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'app/models')
85 files changed, 934 insertions, 447 deletions
diff --git a/app/models/active_session.rb b/app/models/active_session.rb index 0094d98fb73..9f634e70ff4 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -21,7 +21,6 @@ # class ActiveSession include ActiveModel::Model - include ::Gitlab::Redis::SessionsStoreHelper SESSION_BATCH_SIZE = 200 ALLOWED_NUMBER_OF_ACTIVE_SESSIONS = 100 @@ -66,7 +65,7 @@ class ActiveSession end def self.set(user, request) - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| session_private_id = request.session.id.private_id client = DeviceDetector.new(request.user_agent) timestamp = Time.current @@ -107,7 +106,7 @@ class ActiveSession end def self.list(user) - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| cleaned_up_lookup_entries(redis, user).map do |raw_session| load_raw_session(raw_session) end @@ -115,7 +114,7 @@ class ActiveSession end def self.cleanup(user) - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| clean_up_old_sessions(redis, user) cleaned_up_lookup_entries(redis, user) end @@ -138,7 +137,7 @@ class ActiveSession def self.destroy_session(user, session_id) return unless session_id - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| destroy_sessions(redis, user, [session_id].compact) end end @@ -147,7 +146,7 @@ class ActiveSession sessions = not_impersonated(user) sessions.reject! { |session| session.current?(current_rack_session) } if current_rack_session - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| session_ids = sessions.flat_map(&:ids) destroy_sessions(redis, user, session_ids) if session_ids.any? end @@ -182,7 +181,7 @@ class ActiveSession # # Returns an array of strings def self.session_ids_for_user(user_id) - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| redis.smembers(lookup_key_name(user_id)) end end @@ -195,7 +194,7 @@ class ActiveSession def self.sessions_from_ids(session_ids) return [] if session_ids.empty? - redis_store_class.with do |redis| + Gitlab::Redis::Sessions.with do |redis| session_keys = rack_session_keys(session_ids) session_keys.each_slice(SESSION_BATCH_SIZE).flat_map do |session_keys_batch| diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index f40d0cd2fa4..a53fa39c58f 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -78,7 +78,6 @@ module AlertManagement scope :for_environment, -> (environment) { where(environment: environment) } scope :for_assignee_username, -> (assignee_username) { joins(:assignees).merge(User.by_username(assignee_username)) } scope :search, -> (query) { fuzzy_search(query, [:title, :description, :monitoring_tool, :service]) } - scope :open, -> { with_status(open_statuses) } scope :not_resolved, -> { without_status(:resolved) } scope :with_prometheus_alert, -> { includes(:prometheus_alert) } scope :with_threat_monitoring_alerts, -> { where(domain: :threat_monitoring ) } @@ -143,18 +142,6 @@ module AlertManagement reference.to_i > 0 && reference.to_i <= Gitlab::Database::MAX_INT_VALUE end - def self.open_statuses - [:triggered, :acknowledged] - end - - def self.open_status?(status) - open_statuses.include?(status) - end - - def open? - self.class.open_status?(status_name) - end - def prometheus? monitoring_tool == Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 65472615f42..b69c0199c70 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -9,6 +9,7 @@ class ApplicationSetting < ApplicationRecord include Sanitizable ignore_columns %i[elasticsearch_shards elasticsearch_replicas], remove_with: '14.4', remove_after: '2021-09-22' + ignore_columns %i[static_objects_external_storage_auth_token], remove_with: '14.9', remove_after: '2022-03-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -21,7 +22,7 @@ class ApplicationSetting < ApplicationRecord add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption) ? :optional : :required } add_authentication_token_field :health_check_access_token - add_authentication_token_field :static_objects_external_storage_auth_token, encrypted: :optional + add_authentication_token_field :static_objects_external_storage_auth_token, encrypted: :required belongs_to :self_monitoring_project, class_name: "Project", foreign_key: 'instance_administration_project_id' belongs_to :push_rule @@ -77,6 +78,10 @@ class ApplicationSetting < ApplicationRecord chronic_duration_attr_writer :archive_builds_in_human_readable, :archive_builds_in_seconds + chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval + chronic_duration_attr :group_runner_token_expiration_interval_human_readable, :group_runner_token_expiration_interval + chronic_duration_attr :project_runner_token_expiration_interval_human_readable, :project_runner_token_expiration_interval + validates :grafana_url, system_hook_url: { blocked_message: "is blocked: %{exception_message}. " + GRAFANA_URL_ERROR_MESSAGE @@ -352,18 +357,28 @@ class ApplicationSetting < ApplicationRecord validates :hashed_storage_enabled, inclusion: { in: [true], message: _("Hashed storage can't be disabled anymore for new projects") } validates :container_registry_delete_tags_service_timeout, + :container_registry_cleanup_tags_service_max_list_size, + :container_registry_expiration_policies_worker_capacity, numericality: { only_integer: true, greater_than_or_equal_to: 0 } - validates :container_registry_cleanup_tags_service_max_list_size, + validates :container_registry_import_max_tags_count, + :container_registry_import_max_retries, + :container_registry_import_start_max_retries, + :container_registry_import_max_step_duration, + allow_nil: false, numericality: { only_integer: true, greater_than_or_equal_to: 0 } - validates :container_registry_expiration_policies_worker_capacity, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :container_registry_import_target_plan, presence: true + validates :container_registry_import_created_before, presence: true validates :dependency_proxy_ttl_group_policy_worker_capacity, allow_nil: false, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :packages_cleanup_package_file_worker_capacity, + allow_nil: false, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :invisible_captcha_enabled, inclusion: { in: [true, false], message: _('must be a boolean value') } @@ -500,6 +515,9 @@ class ApplicationSetting < ApplicationRecord validates :notes_create_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :user_email_lookup_limit, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :notes_create_limit_allowlist, length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, allow_nil: false diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 5e20aac3b92..25198178f69 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -14,7 +14,7 @@ module ApplicationSettingImplementation # Setting a key restriction to `-1` means that all keys of this type are # forbidden. FORBIDDEN_KEY_VALUE = KeyRestrictionValidator::FORBIDDEN - SUPPORTED_KEY_TYPES = %i[rsa dsa ecdsa ed25519].freeze + SUPPORTED_KEY_TYPES = Gitlab::SSHPublicKey.supported_types VALID_RUNNER_REGISTRAR_TYPES = %w(project group).freeze DEFAULT_PROTECTED_PATHS = [ @@ -217,12 +217,19 @@ module ApplicationSettingImplementation wiki_page_max_content_bytes: 50.megabytes, container_registry_delete_tags_service_timeout: 250, container_registry_expiration_policies_worker_capacity: 0, + container_registry_import_max_tags_count: 100, + container_registry_import_max_retries: 3, + container_registry_import_start_max_retries: 50, + container_registry_import_max_step_duration: 5.minutes, + container_registry_import_target_plan: 'free', + container_registry_import_created_before: '2022-01-23 00:00:00', kroki_enabled: false, kroki_url: nil, kroki_formats: { blockdiag: false, bpmn: false, excalidraw: false }, rate_limiting_response_text: nil, whats_new_variant: 0, - user_deactivation_emails_enabled: true + user_deactivation_emails_enabled: true, + user_email_lookup_limit: 60 } end diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 1a8bd05c42c..35c4e08730e 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class AuditEvent < ApplicationRecord + include AfterCommitQueue include CreatedAtFilterable include BulkInsertSafe include EachBatch diff --git a/app/models/bulk_imports/file_transfer/project_config.rb b/app/models/bulk_imports/file_transfer/project_config.rb index fdfb0dd0186..38884df9fcf 100644 --- a/app/models/bulk_imports/file_transfer/project_config.rb +++ b/app/models/bulk_imports/file_transfer/project_config.rb @@ -8,6 +8,8 @@ module BulkImports group_members ).freeze + LFS_OBJECTS_RELATION = 'lfs_objects' + def import_export_yaml ::Gitlab::ImportExport.config_file end @@ -15,6 +17,10 @@ module BulkImports def skipped_relations SKIPPED_RELATIONS end + + def file_relations + [UPLOADS_RELATION, LFS_OBJECTS_RELATION] + end end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 428e440afba..c4d1a2c740b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -268,6 +268,10 @@ module Ci !build.any_unmet_prerequisites? # If false is returned, it stops the transition end + before_transition on: :enqueue do |build| + !build.waiting_for_deployment_approval? # If false is returned, it stops the transition + end + after_transition created: :scheduled do |build| build.run_after_commit do Ci::BuildScheduleWorker.perform_at(build.scheduled_at, build.id) @@ -393,6 +397,10 @@ module Ci auto_retry.allowed? end + def auto_retry_expected? + failed? && auto_retry_allowed? + end + def detailed_status(current_user) Gitlab::Ci::Status::Build::Factory .new(self.present, current_user) @@ -416,6 +424,10 @@ module Ci true end + def save_tags + super unless Thread.current['ci_bulk_insert_tags'] + end + def archived? return true if degenerated? @@ -424,7 +436,11 @@ module Ci end def playable? - action? && !archived? && (manual? || scheduled? || retryable?) + action? && !archived? && (manual? || scheduled? || retryable?) && !waiting_for_deployment_approval? + end + + def waiting_for_deployment_approval? + manual? && starts_environment? && deployment&.blocked? end def schedulable? @@ -751,9 +767,7 @@ module Ci def any_runners_available? cache_for_available_runners do - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do - project.active_runners.exists? - end + project.active_runners.exists? end end @@ -1021,7 +1035,15 @@ module Ci transaction do update_columns(status: :failed, failure_reason: :data_integrity_failure) all_queuing_entries.delete_all + all_runtime_metadata.delete_all end + + Gitlab::AppLogger.info( + message: 'Build doomed', + class: self.class.name, + build_id: id, + pipeline_id: pipeline_id, + project_id: project_id) end def degradation_threshold @@ -1051,10 +1073,7 @@ module Ci end def drop_with_exit_code!(failure_reason, exit_code) - transaction do - conditionally_allow_failure!(exit_code) - drop!(failure_reason) - end + drop!(::Gitlab::Ci::Build::Status::Reason.new(self, failure_reason, exit_code)) end def exit_codes_defined? @@ -1065,6 +1084,10 @@ module Ci ::Ci::PendingBuild.upsert_from_build!(self) end + def create_runtime_metadata! + ::Ci::RunningBuild.upsert_shared_runner_build!(self) + end + ## # We can have only one queuing entry or running build tracking entry, # because there is a unique index on `build_id` in each table, but we need @@ -1093,6 +1116,13 @@ module Ci end end + def allowed_to_fail_with_code?(exit_code) + options + .dig(:allow_failure_criteria, :exit_codes) + .to_a + .include?(exit_code) + end + protected def run_status_commit_hooks! @@ -1174,27 +1204,15 @@ module Ci break variables unless Feature.enabled?(:ci_job_jwt, project, default_enabled: true) jwt = Gitlab::Ci::Jwt.for_build(self) + jwt_v2 = Gitlab::Ci::JwtV2.for_build(self) variables.append(key: 'CI_JOB_JWT', value: jwt, public: false, masked: true) + variables.append(key: 'CI_JOB_JWT_V1', value: jwt, public: false, masked: true) + variables.append(key: 'CI_JOB_JWT_V2', value: jwt_v2, public: false, masked: true) rescue OpenSSL::PKey::RSAError, Gitlab::Ci::Jwt::NoSigningKeyError => e Gitlab::ErrorTracking.track_exception(e) end end - def conditionally_allow_failure!(exit_code) - return unless exit_code - - if allowed_to_fail_with_code?(exit_code) - update_columns(allow_failure: true) - end - end - - def allowed_to_fail_with_code?(exit_code) - options - .dig(:allow_failure_criteria, :exit_codes) - .to_a - .include?(exit_code) - end - def cache_for_online_runners(&block) Rails.cache.fetch( ['has-online-runners', id], diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index e6dd62fab34..3426c4d5248 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -181,9 +181,7 @@ module Ci end scope :erasable, -> do - types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values - - where(file_type: types) + where(file_type: self.erasable_file_types) end scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } @@ -263,6 +261,10 @@ module Ci [file_type] end + def self.erasable_file_types + self.file_types.keys - NON_ERASABLE_FILE_TYPES + end + def self.total_size self.sum(:size) end @@ -271,10 +273,6 @@ module Ci self.where(project: project).sum(:size) end - def self.distinct_job_ids - distinct.pluck(:job_id) - end - ## # FastDestroyAll concerns # rubocop: disable CodeReuse/ServiceClass @@ -350,9 +348,7 @@ module Ci def store_after_commit? strong_memoize(:store_after_commit) do - trace? && - JobArtifactUploader.direct_upload_enabled? && - Feature.enabled?(:ci_store_trace_outside_transaction, project, default_enabled: :yaml) + trace? && JobArtifactUploader.direct_upload_enabled? end end diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index 8a4be3139e8..ce3faf3546b 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -10,6 +10,12 @@ module Ci where('traversal_ids @> ARRAY[?]::int[]', id) end + scope :contains_any_of_namespaces, -> (ids) do + where('traversal_ids && ARRAY[?]::int[]', ids) + end + + scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) } + class << self def sync!(event) namespace = event.namespace diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index a90bd739741..00d331df4c3 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -69,6 +69,7 @@ module Ci has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus' has_many :job_artifacts, through: :builds + has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' has_many :deployments, through: :builds @@ -130,6 +131,7 @@ module Ci after_create :keep_around_commits, unless: :importing? use_fast_destroy :job_artifacts + use_fast_destroy :build_trace_chunks # We use `Enums::Ci::Pipeline.sources` here so that EE can more easily extend # this `Hash` with new values. @@ -242,11 +244,7 @@ module Ci ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id) end - if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, pipeline.project, default_enabled: :yaml) - Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass - else - ExpirePipelineCacheWorker.perform_async(pipeline.id) - end + Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass end end @@ -466,6 +464,14 @@ module Ci statuses.count(:id) end + def tags_count + ActsAsTaggableOn::Tagging.where(taggable: builds).count + end + + def distinct_tags_count + ActsAsTaggableOn::Tagging.where(taggable: builds).count('distinct(tag_id)') + end + def stages_names statuses.order(:stage_idx).distinct .pluck(:stage, :stage_idx).map(&:first) @@ -1284,6 +1290,12 @@ module Ci end end + def use_variables_builder_definitions? + strong_memoize(:use_variables_builder_definitions) do + ::Feature.enabled?(:ci_use_variables_builder_definitions, project, default_enabled: :yaml) + end + end + private def add_message(severity, content) diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb index d6aaa3f50c1..9000d1791a6 100644 --- a/app/models/ci/project_mirror.rb +++ b/app/models/ci/project_mirror.rb @@ -6,6 +6,9 @@ module Ci class ProjectMirror < ApplicationRecord belongs_to :project + scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) } + scope :by_project_id, -> (project_id) { where(project_id: project_id) } + class << self def sync!(event) upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index a80fd02080f..809c245d2b9 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -95,7 +95,33 @@ module Ci joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } - scope :belonging_to_group, -> (group_id, include_ancestors: false) { + scope :belonging_to_group, -> (group_id) { + joins(:runner_namespaces) + .where(ci_runner_namespaces: { namespace_id: group_id }) + } + + scope :belonging_to_group_or_project_descendants, -> (group_id) { + group_ids = Ci::NamespaceMirror.contains_namespace(group_id).select(:namespace_id) + project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id) + + group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_ids }) + project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_ids }) + + union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql + + from("(#{union_sql}) #{table_name}") + } + + scope :belonging_to_group_and_ancestors, -> (group_id) { + group_self_and_ancestors_ids = ::Group.find_by(id: group_id)&.self_and_ancestor_ids + + joins(:runner_namespaces) + .where(ci_runner_namespaces: { namespace_id: group_self_and_ancestors_ids }) + } + + # deprecated + # split this into: belonging_to_group & belonging_to_group_and_ancestors + scope :legacy_belonging_to_group, -> (group_id, include_ancestors: false) { groups = ::Group.where(id: group_id) groups = groups.self_and_ancestors if include_ancestors @@ -104,7 +130,8 @@ module Ci .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } - scope :belonging_to_group_or_project, -> (group_id, project_id) { + # deprecated + scope :legacy_belonging_to_group_or_project, -> (group_id, project_id) { groups = ::Group.where(id: group_id) group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups }) @@ -117,11 +144,11 @@ module Ci } scope :belonging_to_parent_group_of_project, -> (project_id) { + raise ArgumentError, "only 1 project_id allowed for performance reasons" unless project_id.is_a?(Integer) + project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) - joins(:groups) - .where(namespaces: { id: project_groups.self_and_ancestors.as_ids }) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') + belonging_to_group(project_groups.self_and_ancestors.pluck(:id)) } scope :owned_or_instance_wide, -> (project_id) do @@ -132,7 +159,7 @@ module Ci instance_type ], remove_duplicates: false - ).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') + ) end scope :assignable_for, ->(project) do @@ -183,6 +210,8 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } + validates :maintainer_note, length: { maximum: 255 } + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. @@ -233,18 +262,16 @@ module Ci Arel.sql("(#{arel_tag_names_array.to_sql})") ] - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339621') do - group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| - Gitlab::Ci::Matching::RunnerMatcher.new({ - runner_ids: values[0], - runner_type: values[1], - public_projects_minutes_cost_factor: values[2], - private_projects_minutes_cost_factor: values[3], - run_untagged: values[4], - access_level: values[5], - tag_list: values[6] - }) - end + group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| + Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_ids: values[0], + runner_type: values[1], + public_projects_minutes_cost_factor: values[2], + private_projects_minutes_cost_factor: values[3], + run_untagged: values[4], + access_level: values[5], + tag_list: values[6] + }) end end @@ -441,6 +468,7 @@ module Ci private EXECUTOR_NAME_TO_TYPES = { + 'unknown' => :unknown, 'custom' => :custom, 'shell' => :shell, 'docker' => :docker, @@ -454,6 +482,8 @@ module Ci 'kubernetes' => :kubernetes }.freeze + EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze + def cleanup_runner_queue Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb new file mode 100644 index 00000000000..56f632b6232 --- /dev/null +++ b/app/models/ci/secure_file.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Ci + class SecureFile < Ci::ApplicationRecord + include FileStoreMounter + + FILE_SIZE_LIMIT = 5.megabytes.freeze + CHECKSUM_ALGORITHM = 'sha256' + + belongs_to :project, optional: false + + validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } + validates :checksum, :file_store, :name, :permissions, :project_id, presence: true + + before_validation :assign_checksum + + enum permissions: { read_only: 0, read_write: 1, execute: 2 } + + default_value_for(:file_store) { Ci::SecureFileUploader.default_store } + + mount_file_store_uploader Ci::SecureFileUploader + + def checksum_algorithm + CHECKSUM_ALGORITHM + end + + private + + def assign_checksum + self.checksum = file.checksum if file.present? && file_changed? + end + end +end diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index 98490a13351..79fc2b58237 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -5,6 +5,7 @@ module Clusters self.table_name = 'cluster_agents' INACTIVE_AFTER = 1.hour.freeze + ACTIVITY_EVENT_LIMIT = 200 belongs_to :created_by_user, class_name: 'User', optional: true belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project @@ -36,8 +37,15 @@ module Clusters requested_project == project end - def active? - agent_tokens.where("last_used_at > ?", INACTIVE_AFTER.ago).exists? + def connected? + agent_tokens.active.where("last_used_at > ?", INACTIVE_AFTER.ago).exists? + end + + def activity_event_deletion_cutoff + # Order is defined by the association + activity_events + .offset(ACTIVITY_EVENT_LIMIT - 1) + .pick(:recorded_at) end end end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index 87dba50cd69..691d628524f 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -10,9 +10,6 @@ module Clusters self.table_name = 'cluster_agent_tokens' - # The `UPDATE_USED_COLUMN_EVERY` defines how often the token DB entry can be updated - UPDATE_USED_COLUMN_EVERY = (40.minutes..55.minutes).freeze - belongs_to :agent, class_name: 'Clusters::Agent', optional: false belongs_to :created_by_user, class_name: 'User', optional: true @@ -22,40 +19,11 @@ module Clusters validates :name, presence: true, length: { maximum: 255 } scope :order_last_used_at_desc, -> { order(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) } + scope :with_status, -> (status) { where(status: status) } - def track_usage - track_values = { last_used_at: Time.current.utc } - - cache_attributes(track_values) - - if can_update_track_values? - log_activity_event!(track_values[:last_used_at]) unless agent.active? - - # Use update_column so updated_at is skipped - update_columns(track_values) - end - end - - private - - def can_update_track_values? - # Use a random threshold to prevent beating DB updates. - last_used_at_max_age = Random.rand(UPDATE_USED_COLUMN_EVERY) - - real_last_used_at = read_attribute(:last_used_at) - - # Handle too many updates from high token traffic - real_last_used_at.nil? || - (Time.current - real_last_used_at) >= last_used_at_max_age - end - - def log_activity_event!(recorded_at) - agent.activity_events.create!( - kind: :agent_connected, - level: :info, - recorded_at: recorded_at, - agent_token: self - ) - end + enum status: { + active: 0, + revoked: 1 + } end end diff --git a/app/models/clusters/agents/activity_event.rb b/app/models/clusters/agents/activity_event.rb index 5d9c885c923..ec2bbfde339 100644 --- a/app/models/clusters/agents/activity_event.rb +++ b/app/models/clusters/agents/activity_event.rb @@ -3,6 +3,7 @@ module Clusters module Agents class ActivityEvent < ApplicationRecord + include EachBatch include NullifyIfBlank self.table_name = 'agent_activity_events' @@ -12,6 +13,7 @@ module Clusters belongs_to :agent_token, class_name: 'Clusters::AgentToken' scope :in_timeline_order, -> { order(recorded_at: :desc, id: :desc) } + scope :recorded_before, -> (cutoff) { where('recorded_at < ?', cutoff) } validates :recorded_at, :kind, :level, presence: true diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index b57a24dead0..33cd5de3518 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.35.0' + VERSION = '0.36.0' self.table_name = 'clusters_applications_runners' @@ -41,7 +41,7 @@ module Clusters end def prepare_uninstall - runner&.update!(active: false) + # No op, see https://gitlab.com/gitlab-org/gitlab/-/issues/350180. end def post_uninstall diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d6a2f62ca9b..21e2e21e9b3 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -170,8 +170,11 @@ class CommitStatus < Ci::ApplicationRecord end before_transition any => :failed do |commit_status, transition| - failure_reason = transition.args.first - commit_status.failure_reason = CommitStatus.failure_reasons[failure_reason] + reason = ::Gitlab::Ci::Build::Status::Reason + .fabricate(commit_status, transition.args.first) + + commit_status.failure_reason = reason.failure_reason_enum + commit_status.allow_failure = true if reason.force_allow_failure? end before_transition [:skipped, :manual] => :created do |commit_status, transition| @@ -191,11 +194,7 @@ class CommitStatus < Ci::ApplicationRecord commit_status.run_after_commit do PipelineProcessWorker.perform_async(pipeline_id) unless transition_options[:skip_pipeline_processing] - if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, project, default_enabled: :yaml) - expire_etag_cache! - else - ExpireJobCacheWorker.perform_async(id) - end + expire_etag_cache! end end @@ -221,8 +220,8 @@ class CommitStatus < Ci::ApplicationRecord false end - def self.bulk_insert_tags!(statuses, tag_list_by_build) - Gitlab::Ci::Tags::BulkInsert.new(statuses, tag_list_by_build).insert! + def self.bulk_insert_tags!(statuses) + Gitlab::Ci::Tags::BulkInsert.new(statuses).insert! end def locking_enabled? diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 12ddbc2cc40..ed3b422251f 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -13,6 +13,8 @@ module Ci track_duration do variables = pipeline.variables_builder.scoped_variables(self, environment: environment, dependencies: dependencies) + next variables if pipeline.use_variables_builder_definitions? + variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runnable? && runner @@ -60,49 +62,27 @@ module Ci end def user_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables if user.blank? - - variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) - variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) - variables.append(key: 'GITLAB_USER_LOGIN', value: user.username) - variables.append(key: 'GITLAB_USER_NAME', value: user.name) - end + pipeline.variables_builder.user_variables(user) end def kubernetes_variables - ::Gitlab::Ci::Variables::Collection.new.tap do |collection| - # Should get merged with the cluster kubeconfig in deployment_variables, see - # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 - template = ::Ci::GenerateKubeconfigService.new(self).execute - - if template.valid? - collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) - end - end + pipeline.variables_builder.kubernetes_variables(self) end def deployment_variables(environment:) - return [] unless environment - - project.deployment_variables( - environment: environment, - kubernetes_namespace: expanded_kubernetes_namespace - ) + pipeline.variables_builder.deployment_variables(job: self, environment: environment) end def secret_instance_variables - project.ci_instance_variables_for(ref: git_ref) + pipeline.variables_builder.secret_instance_variables(ref: git_ref) end def secret_group_variables(environment: expanded_environment_name) - return [] unless project.group - - project.group.ci_variables_for(git_ref, project, environment: environment) + pipeline.variables_builder.secret_group_variables(environment: environment, ref: git_ref) end def secret_project_variables(environment: expanded_environment_name) - project.ci_variables_for(ref: git_ref, environment: environment) + pipeline.variables_builder.secret_project_variables(environment: environment, ref: git_ref) end end end diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 611b27c722b..aa9669ee208 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -18,13 +18,19 @@ module Ci delegate :timeout, to: :metadata, prefix: true, allow_nil: true delegate :interruptible, to: :metadata, prefix: false, allow_nil: true - delegate :has_exposed_artifacts?, to: :metadata, prefix: false, allow_nil: true delegate :environment_auto_stop_in, to: :metadata, prefix: false, allow_nil: true delegate :set_cancel_gracefully, to: :metadata, prefix: false, allow_nil: false - delegate :cancel_gracefully?, to: :metadata, prefix: false, allow_nil: false before_create :ensure_metadata end + def has_exposed_artifacts? + !!metadata&.has_exposed_artifacts? + end + + def cancel_gracefully? + !!metadata&.cancel_gracefully? + end + def ensure_metadata metadata || build_metadata(project: project) end diff --git a/app/models/concerns/forced_email_confirmation.rb b/app/models/concerns/forced_email_confirmation.rb new file mode 100644 index 00000000000..649400184e5 --- /dev/null +++ b/app/models/concerns/forced_email_confirmation.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ForcedEmailConfirmation + extend ActiveSupport::Concern + + included do + attr_accessor :skip_confirmation_period_expiry_check + end + + def force_confirm(args = {}) + self.skip_confirmation_period_expiry_check = true + confirm(args) + ensure + self.skip_confirmation_period_expiry_check = nil + end + + protected + + # Override, from Devise::Models::Confirmable + # Link: https://github.com/heartcombo/devise/blob/main/lib/devise/models/confirmable.rb + def confirmation_period_expired? + return false if skip_confirmation_period_expiry_check + + super + end +end diff --git a/app/models/concerns/has_wiki.rb b/app/models/concerns/has_wiki.rb index df7bbe4dc08..89bcabafb84 100644 --- a/app/models/concerns/has_wiki.rb +++ b/app/models/concerns/has_wiki.rb @@ -17,7 +17,7 @@ module HasWiki def wiki strong_memoize(:wiki) do - Wiki.for_container(self, self.default_owner) + Wiki.for_container(self, self.first_owner) end end diff --git a/app/models/concerns/import_state/sidekiq_job_tracker.rb b/app/models/concerns/import_state/sidekiq_job_tracker.rb index 340bf4279bc..b7d0ed0f51b 100644 --- a/app/models/concerns/import_state/sidekiq_job_tracker.rb +++ b/app/models/concerns/import_state/sidekiq_job_tracker.rb @@ -15,7 +15,7 @@ module ImportState def refresh_jid_expiration return unless jid - Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) end def self.jid_by(project_id:, status:) diff --git a/app/models/concerns/incident_management/escalatable.rb b/app/models/concerns/incident_management/escalatable.rb index 81eef50603a..a9e4a066e0e 100644 --- a/app/models/concerns/incident_management/escalatable.rb +++ b/app/models/concerns/incident_management/escalatable.rb @@ -27,6 +27,8 @@ module IncidentManagement ignored: 'No action will be taken' }.freeze + OPEN_STATUSES = [:triggered, :acknowledged].freeze + included do validates :status, presence: true @@ -34,6 +36,7 @@ module IncidentManagement # Descending sort order sorts statuses: Triggered > Acknowledged > Resolved > Ignored # https://gitlab.com/gitlab-org/gitlab/-/issues/221242#what-is-the-expected-correct-behavior scope :order_status, -> (sort_order) { order(status: sort_order == :asc ? :desc : :asc) } + scope :open, -> { with_status(OPEN_STATUSES) } state_machine :status, initial: :triggered do state :triggered, value: STATUSES[:triggered] @@ -89,6 +92,10 @@ module IncidentManagement @status_names ||= state_machine_statuses.keys end + def open_status?(status) + OPEN_STATUSES.include?(status) + end + private def state_machine_statuses @@ -99,6 +106,10 @@ module IncidentManagement def status_event_for(status) self.class.state_machines[:status].events.transitions_for(self, to: status.to_s.to_sym).first&.event end + + def open? + self.class.open_status?(status_name) + end end end end diff --git a/app/models/concerns/packages/debian/distribution.rb b/app/models/concerns/packages/debian/distribution.rb index ff52769fce8..2d46889ce6a 100644 --- a/app/models/concerns/packages/debian/distribution.rb +++ b/app/models/concerns/packages/debian/distribution.rb @@ -96,6 +96,15 @@ module Packages architectures.pluck(:name).sort end + def package_files + if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + ::Packages::PackageFile.installable + .for_package_ids(packages.select(:id)) + else + ::Packages::PackageFile.for_package_ids(packages.select(:id)) + end + end + private def unique_codename_and_suite diff --git a/app/models/concerns/packages/destructible.rb b/app/models/concerns/packages/destructible.rb new file mode 100644 index 00000000000..a3b7d8580c1 --- /dev/null +++ b/app/models/concerns/packages/destructible.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Packages + module Destructible + extend ActiveSupport::Concern + + class_methods do + def next_pending_destruction(order_by: nil) + set = pending_destruction.limit(1).lock('FOR UPDATE SKIP LOCKED') + set = set.order(order_by) if order_by + set.take + end + end + end +end diff --git a/app/models/concerns/packages/installable.rb b/app/models/concerns/packages/installable.rb new file mode 100644 index 00000000000..e9303e55412 --- /dev/null +++ b/app/models/concerns/packages/installable.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Packages + # This module requires a status column. + # It also requires a constant INSTALLABLE_STATUSES. This should be + # an array that defines which values of the status column are + # considered as installable. + module Installable + extend ActiveSupport::Concern + + included do + scope :with_status, ->(status) { where(status: status) } + scope :installable, -> { with_status(const_get(:INSTALLABLE_STATUSES, false)) } + end + end +end diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index 1663aa6c886..20743ebcb52 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -64,8 +64,6 @@ module Participable # # Returns an Array of User instances. def visible_participants(user) - return participants(user) unless Feature.enabled?(:verify_participants_access, project, default_enabled: :yaml) - filter_by_ability(raw_participants(user, verify_access: true)) end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index f382b3624ed..2cf95ac0dae 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -190,5 +190,10 @@ module Routable route || build_route(source: self) route.path = build_full_path route.name = build_full_name + route.namespace = if is_a?(Namespace) + self + elsif is_a?(Project) + self.project_namespace + end end end diff --git a/app/models/concerns/runner_token_expiration_interval.rb b/app/models/concerns/runner_token_expiration_interval.rb new file mode 100644 index 00000000000..f84e69e7b7d --- /dev/null +++ b/app/models/concerns/runner_token_expiration_interval.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module RunnerTokenExpirationInterval + extend ActiveSupport::Concern + + def enforced_runner_token_expiration_interval_human_readable + interval = enforced_runner_token_expiration_interval + ChronicDuration.output(interval, format: :short) if interval + end + + def effective_runner_token_expiration_interval + [ + enforced_runner_token_expiration_interval, + runner_token_expiration_interval&.seconds + ].compact.min + end + + def effective_runner_token_expiration_interval_human_readable + interval = effective_runner_token_expiration_interval + ChronicDuration.output(interval, format: :short) if interval + end +end diff --git a/app/models/concerns/ttl_expirable.rb b/app/models/concerns/ttl_expirable.rb index 6d89521255c..1c2147beedd 100644 --- a/app/models/concerns/ttl_expirable.rb +++ b/app/models/concerns/ttl_expirable.rb @@ -7,16 +7,10 @@ module TtlExpirable validates :status, presence: true default_value_for :read_at, Time.zone.now - enum status: { default: 0, expired: 1, processing: 2, error: 3 } + enum status: { default: 0, pending_destruction: 1, processing: 2, error: 3 } scope :read_before, ->(number_of_days) { where("read_at <= ?", Time.zone.now - number_of_days.days) } scope :active, -> { where(status: :default) } - - scope :lock_next_by, ->(sort) do - order(sort) - .limit(1) - .lock('FOR UPDATE SKIP LOCKED') - end end def read! diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index c914819f79d..b03d946fc47 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -13,9 +13,15 @@ class ContainerRepository < ApplicationRecord validates :name, length: { minimum: 0, allow_nil: false } validates :name, uniqueness: { scope: :project_id } + validates :migration_state, presence: true + + validates :migration_retries_count, presence: true, + numericality: { greater_than_or_equal_to: 0 }, + allow_nil: false enum status: { delete_scheduled: 0, delete_failed: 1 } enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 } + enum migration_skipped_reason: { not_in_plan: 0, too_many_retries: 1, too_many_tags: 2, root_namespace_in_deny_list: 3 } delegate :client, to: :registry diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index d8669f1f4c2..168f1c48a6c 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -24,11 +24,12 @@ class CustomerRelations::Contact < ApplicationRecord validates :email, length: { maximum: 255 } validates :description, length: { maximum: 1024 } validate :validate_email_format + validate :unique_email_for_group_hierarchy - def self.find_ids_by_emails(group_id, emails) + def self.find_ids_by_emails(group, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK - where(group_id: group_id, email: emails) + where(group_id: group.self_and_ancestor_ids, email: emails) .pluck(:id) end @@ -39,4 +40,14 @@ class CustomerRelations::Contact < ApplicationRecord self.errors.add(:email, I18n.t(:invalid, scope: 'valid_email.validations.email')) unless ValidateEmail.valid?(self.email) end + + def unique_email_for_group_hierarchy + return unless group + return unless email + + duplicate_email_exists = CustomerRelations::Contact + .where(group_id: group.self_and_hierarchy.pluck(:id), email: email) + .where.not(id: id).exists? + self.errors.add(:email, _('contact with same email already exists in group hierarchy')) if duplicate_email_exists + end end diff --git a/app/models/customer_relations/issue_contact.rb b/app/models/customer_relations/issue_contact.rb index 78f662b6a58..89dac6bad22 100644 --- a/app/models/customer_relations/issue_contact.rb +++ b/app/models/customer_relations/issue_contact.rb @@ -6,7 +6,7 @@ class CustomerRelations::IssueContact < ApplicationRecord belongs_to :issue, optional: false, inverse_of: :customer_relations_contacts belongs_to :contact, optional: false, inverse_of: :issue_contacts - validate :contact_belongs_to_issue_group + validate :contact_belongs_to_issue_group_or_ancestor def self.find_contact_ids_by_emails(issue_id, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK @@ -18,11 +18,11 @@ class CustomerRelations::IssueContact < ApplicationRecord private - def contact_belongs_to_issue_group + def contact_belongs_to_issue_group_or_ancestor return unless contact&.group_id return unless issue&.project&.namespace_id - return if contact.group_id == issue.project.namespace_id + return if issue.project.group&.self_and_ancestor_ids&.include?(contact.group_id) - errors.add(:base, _('The contact does not belong to the same group as the issue')) + errors.add(:base, _('The contact does not belong to the issue group or an ancestor')) end end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb index bd5c022e692..e4018ab4770 100644 --- a/app/models/dependency_proxy/blob.rb +++ b/app/models/dependency_proxy/blob.rb @@ -3,6 +3,7 @@ class DependencyProxy::Blob < ApplicationRecord include FileStoreMounter include TtlExpirable + include Packages::Destructible include EachBatch belongs_to :group diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index 64f484942ef..fe887c99e81 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -3,6 +3,7 @@ class DependencyProxy::Manifest < ApplicationRecord include FileStoreMounter include TtlExpirable + include Packages::Destructible include EachBatch belongs_to :group diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 4c60ce57f49..2f04d99f9f6 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -8,6 +8,7 @@ class Deployment < ApplicationRecord include Importable include Gitlab::Utils::StrongMemoize include FastDestroyAll + include FromUnion StatusUpdateError = Class.new(StandardError) StatusSyncError = Class.new(StandardError) @@ -69,6 +70,10 @@ class Deployment < ApplicationRecord transition created: :blocked end + event :unblock do + transition blocked: :created + end + event :succeed do transition any - [:success] => :success end @@ -107,10 +112,7 @@ class Deployment < ApplicationRecord deployment.run_after_commit do Deployments::UpdateEnvironmentWorker.perform_async(id) Deployments::LinkMergeRequestWorker.perform_async(id) - - if ::Feature.enabled?(:deployments_archive, deployment.project, default_enabled: :yaml) - Deployments::ArchiveInProjectWorker.perform_async(deployment.project_id) - end + Deployments::ArchiveInProjectWorker.perform_async(deployment.project_id) end end diff --git a/app/models/email.rb b/app/models/email.rb index 676e79406e9..3896dfd5d22 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -6,8 +6,8 @@ class Email < ApplicationRecord belongs_to :user, optional: false - validates :email, presence: true, uniqueness: true - validate :validate_email_format + validates :email, presence: true, uniqueness: true, devise_email: true + validate :unique_email, if: ->(email) { email.email_changed? } scope :confirmed, -> { where.not(confirmed_at: nil) } @@ -19,6 +19,7 @@ class Email < ApplicationRecord # This module adds async behaviour to Devise emails # and should be added after Devise modules are initialized. include AsyncDeviseEmail + include ForcedEmailConfirmation self.reconfirmable = false # currently email can't be changed, no need to reconfirm @@ -32,10 +33,6 @@ class Email < ApplicationRecord self.errors.add(:email, 'has already been taken') if primary_email_of_another_user? end - def validate_email_format - self.errors.add(:email, I18n.t(:invalid, scope: 'valid_email.validations.email')) unless ValidateEmail.valid?(self.email) - end - # once email is confirmed, update the gpg signatures def update_invalid_gpg_signatures user.update_invalid_gpg_signatures if confirmed? diff --git a/app/models/experiment.rb b/app/models/experiment.rb index cd0814c476a..2300ec2996d 100644 --- a/app/models/experiment.rb +++ b/app/models/experiment.rb @@ -7,7 +7,7 @@ class Experiment < ApplicationRecord validates :name, presence: true, uniqueness: true, length: { maximum: 255 } def self.add_user(name, group_type, user, context = {}) - find_or_create_by!(name: name).record_user_and_group(user, group_type, context) + by_name(name).record_user_and_group(user, group_type, context) end def self.add_group(name, variant:, group:) @@ -15,11 +15,15 @@ class Experiment < ApplicationRecord end def self.add_subject(name, variant:, subject:) - find_or_create_by!(name: name).record_subject_and_variant!(subject, variant) + by_name(name).record_subject_and_variant!(subject, variant) end def self.record_conversion_event(name, user, context = {}) - find_or_create_by!(name: name).record_conversion_event_for_user(user, context) + by_name(name).record_conversion_event_for_user(user, context) + end + + def self.by_name(name) + find_or_create_by!(name: name) end # Create or update the recorded experiment_user row for the user in this experiment. @@ -41,6 +45,16 @@ class Experiment < ApplicationRecord experiment_user.update!(converted_at: Time.current, context: merged_context(experiment_user, context)) end + def record_conversion_event_for_subject(subject, context = {}) + raise 'Incompatible subject provided!' unless ExperimentSubject.valid_subject?(subject) + + attr_name = subject.class.table_name.singularize.to_sym + experiment_subject = experiment_subjects.find_by(attr_name => subject) + return unless experiment_subject + + experiment_subject.update!(converted_at: Time.current, context: merged_context(experiment_subject, context)) + end + def record_subject_and_variant!(subject, variant) raise 'Incompatible subject provided!' unless ExperimentSubject.valid_subject?(subject) @@ -57,7 +71,7 @@ class Experiment < ApplicationRecord private - def merged_context(experiment_user, new_context) - experiment_user.context.deep_merge(new_context.deep_stringify_keys) + def merged_context(experiment_subject, new_context) + experiment_subject.context.deep_merge(new_context.deep_stringify_keys) end end diff --git a/app/models/external_pull_request.rb b/app/models/external_pull_request.rb index 3fc166203e7..4654f7e2341 100644 --- a/app/models/external_pull_request.rb +++ b/app/models/external_pull_request.rb @@ -11,7 +11,7 @@ # When the mirror is updated and changes are pushed to branches we check # if there are open pull requests for the source and target branch. # If so, we create pipelines for external pull requests. -class ExternalPullRequest < ApplicationRecord +class ExternalPullRequest < Ci::ApplicationRecord include Gitlab::Utils::StrongMemoize include ShaAttribute @@ -40,6 +40,9 @@ class ExternalPullRequest < ApplicationRecord scope :by_source_branch, ->(branch) { where(source_branch: branch) } scope :by_source_repository, -> (repository) { where(source_repository: repository) } + # Needed to override Ci::ApplicationRecord as this does not have ci_ table prefix + self.table_name = 'external_pull_requests' + def self.create_or_update_from_params(params) find_params = params.slice(:project_id, :source_branch, :target_branch) diff --git a/app/models/group.rb b/app/models/group.rb index f51782785f9..53da70f47e5 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -17,6 +17,8 @@ class Group < Namespace include GroupAPICompatibility include EachBatch include BulkMemberAccessLoad + include ChronicDurationAttribute + include RunnerTokenExpirationInterval def self.sti_name 'Group' @@ -91,6 +93,11 @@ class Group < Namespace has_many :group_callouts, class_name: 'Users::GroupCallout', foreign_key: :group_id delegate :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, :setup_for_company, :jobs_to_be_done, to: :namespace_settings + delegate :runner_token_expiration_interval, :runner_token_expiration_interval=, :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval_human_readable=, to: :namespace_settings, allow_nil: true + delegate :subgroup_runner_token_expiration_interval, :subgroup_runner_token_expiration_interval=, :subgroup_runner_token_expiration_interval_human_readable, :subgroup_runner_token_expiration_interval_human_readable=, to: :namespace_settings, allow_nil: true + delegate :project_runner_token_expiration_interval, :project_runner_token_expiration_interval=, :project_runner_token_expiration_interval_human_readable, :project_runner_token_expiration_interval_human_readable=, to: :namespace_settings, allow_nil: true + + has_one :crm_settings, class_name: 'Group::CrmSettings', inverse_of: :group accepts_nested_attributes_for :variables, allow_destroy: true @@ -121,6 +128,8 @@ class Group < Namespace scope :by_id, ->(groups) { where(id: groups) } + scope :by_ids_or_paths, -> (ids, paths) { by_id(ids).or(where(path: paths)) } + scope :for_authorized_group_members, -> (user_ids) do joins(:group_members) .where(members: { user_id: user_ids }) @@ -212,6 +221,10 @@ class Group < Namespace Set.new(group_ids) end + def get_ids_by_ids_or_paths(ids, paths) + by_ids_or_paths(ids, paths).pluck(:id) + end + private def public_to_user_arel(user) @@ -619,7 +632,7 @@ class Group < Namespace end end - def group_member(user) + def member(user) if group_members.loaded? group_members.find { |gm| gm.user_id == user.id } else @@ -631,6 +644,10 @@ class Group < Namespace GroupMember.where(source_id: self_and_ancestors_ids, user_id: user.id).order(:access_level).last end + def bots + users.project_bot + end + def related_group_ids [id, *ancestors.pluck(:id), @@ -713,8 +730,8 @@ class Group < Namespace end end - def default_owner - owners.first || parent&.default_owner || owner + def first_owner + owners.first || parent&.first_owner || owner end def default_branch_name @@ -764,6 +781,29 @@ class Group < Namespace super || build_dependency_proxy_image_ttl_policy end + def dependency_proxy_setting + super || build_dependency_proxy_setting + end + + def crm_enabled? + crm_settings&.enabled? + end + + def shared_with_group_links_visible_to_user(user) + shared_with_group_links.preload_shared_with_groups.filter { |link| Ability.allowed?(user, :read_group, link.shared_with_group) } + end + + def enforced_runner_token_expiration_interval + all_parent_groups = Gitlab::ObjectHierarchy.new(Group.where(id: id)).ancestors + all_group_settings = NamespaceSetting.where(namespace_id: all_parent_groups) + group_interval = all_group_settings.where.not(subgroup_runner_token_expiration_interval: nil).minimum(:subgroup_runner_token_expiration_interval)&.seconds + + [ + Gitlab::CurrentSettings.group_runner_token_expiration_interval&.seconds, + group_interval + ].compact.min + end + private def max_member_access(user_ids) diff --git a/app/models/group/crm_settings.rb b/app/models/group/crm_settings.rb new file mode 100644 index 00000000000..30fbe6ae07f --- /dev/null +++ b/app/models/group/crm_settings.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class Group::CrmSettings < ApplicationRecord + self.primary_key = :group_id + self.table_name = 'group_crm_settings' + + belongs_to :group, -> { where(type: Group.sti_name) }, foreign_key: 'group_id' + + validates :group, presence: true +end diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index fdc54ba33ab..c4c3fc390e1 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -14,7 +14,7 @@ class GroupGroupLink < ApplicationRecord presence: true scope :non_guests, -> { where('group_access > ?', Gitlab::Access::GUEST) } - scope :public_or_visible_to_user, ->(group, user) { where(shared_group: group, shared_with_group: Group.public_or_visible_to_user(user)) } # rubocop:disable Cop/GroupPublicOrVisibleToUser + scope :preload_shared_with_groups, -> { preload(:shared_with_group) } def self.access_options Gitlab::Access.options_with_owner diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 16b95d2a2b9..9f45160d3a8 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -41,6 +41,11 @@ class ProjectHook < WebHook super.merge(project: project) end + override :parent + def parent + project + end + private override :web_hooks_disable_failed? diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb index 1a466b333a5..80e167b350b 100644 --- a/app/models/hooks/service_hook.rb +++ b/app/models/hooks/service_hook.rb @@ -2,6 +2,7 @@ class ServiceHook < WebHook include Presentable + extend ::Gitlab::Utils::Override belongs_to :integration, foreign_key: :service_id validates :integration, presence: true @@ -9,4 +10,7 @@ class ServiceHook < WebHook def execute(data, hook_name = 'service_hook') super(data, hook_name) end + + override :parent + delegate :parent, to: :integration end diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index e8a55abfc8f..3320c13e87b 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -122,6 +122,11 @@ class WebHook < ApplicationRecord nil end + # Returns the associated Project or Group for the WebHook if one exists. + # Overridden by inheriting classes. + def parent + end + # Custom attributes to be included in the worker context. def application_context { related_class: type } diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index bbddc18103a..dc025e576ed 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -117,7 +117,8 @@ class InstanceConfiguration group_export: application_setting_limit_per_minute(:group_export_limit), group_export_download: application_setting_limit_per_minute(:group_download_export_limit), group_import: application_setting_limit_per_minute(:group_import_limit), - raw_blob: application_setting_limit_per_minute(:raw_blob_request_limit) + raw_blob: application_setting_limit_per_minute(:raw_blob_request_limit), + user_email_lookup: application_setting_limit_per_minute(:user_email_lookup_limit) } end diff --git a/app/models/integration.rb b/app/models/integration.rb index 29d96650a81..89b34932e20 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -92,6 +92,7 @@ class Integration < ApplicationRecord scope :note_hooks, -> { where(note_events: true, active: true) } scope :confidential_note_hooks, -> { where(confidential_note_events: true, active: true) } scope :job_hooks, -> { where(job_events: true, active: true) } + scope :archive_trace_hooks, -> { where(archive_trace_events: true, active: true) } scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) } scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) } scope :deployment_hooks, -> { where(deployment_events: true, active: true) } diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index ca72de47d30..d0d54a92021 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -204,7 +204,7 @@ module Integrations when "wiki_page" Integrations::ChatMessage::WikiPageMessage.new(data) when "deployment" - Integrations::ChatMessage::DeploymentMessage.new(data) + Integrations::ChatMessage::DeploymentMessage.new(data) if notify_for_ref?(data) end end @@ -241,7 +241,11 @@ module Integrations def notify_for_ref?(data) return true if data[:object_kind] == 'tag_push' - return true if data.dig(:object_attributes, :tag) + return true if data[:object_kind] == 'deployment' && !Feature.enabled?(:chat_notification_deployment_protected_branch_filter, project) + + ref = data[:ref] || data.dig(:object_attributes, :ref) + return true if ref.blank? # No need to check protected branches when there is no ref + return true if Gitlab::Git.tag_ref?(ref) # Skip protected branch check because it doesn't support tags notify_for_branch?(data) end diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index 72e0ca22ac2..b86f0aaa7ef 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -2,7 +2,6 @@ module Integrations class Datadog < Integration - include ActionView::Helpers::UrlHelper include HasWebHook extend Gitlab::Utils::Override @@ -34,12 +33,21 @@ module Integrations SUPPORTED_EVENTS end + def supported_events + events = super + + return events + ['archive_trace'] if Feature.enabled?(:datadog_integration_logs_collection, parent) + + events + end + def self.default_test_event 'pipeline' end def configurable_events [] # do not allow to opt out of required hooks + # archive_trace is opt-in but we handle it with a more detailed field below end def title @@ -51,7 +59,11 @@ module Integrations end def help - docs_link = link_to s_('DatadogIntegration|How do I set up this integration?'), Rails.application.routes.url_helpers.help_page_url('integration/datadog'), target: '_blank', rel: 'noopener noreferrer' + docs_link = ActionController::Base.helpers.link_to( + s_('DatadogIntegration|How do I set up this integration?'), + Rails.application.routes.url_helpers.help_page_url('integration/datadog'), + target: '_blank', rel: 'noopener noreferrer' + ) s_('DatadogIntegration|Send CI/CD pipeline information to Datadog to monitor for job failures and troubleshoot performance issues. %{docs_link}').html_safe % { docs_link: docs_link.html_safe } end @@ -60,7 +72,7 @@ module Integrations end def fields - [ + f = [ { type: 'text', name: 'datadog_site', @@ -93,7 +105,21 @@ module Integrations linkClose: '</a>'.html_safe }, required: true - }, + } + ] + + if Feature.enabled?(:datadog_integration_logs_collection, parent) + f.append({ + type: 'checkbox', + name: 'archive_trace_events', + title: s_('Logs'), + checkbox_label: s_('Enable logs collection'), + help: s_('When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces.'), + required: false + }) + end + + f += [ { type: 'text', name: 'datadog_service', @@ -116,6 +142,8 @@ module Integrations } } ] + + f end override :hook_url @@ -136,8 +164,7 @@ module Integrations object_kind = 'job' if object_kind == 'build' return unless supported_events.include?(object_kind) - data = data.with_retried_builds if data.respond_to?(:with_retried_builds) - + data = hook_data(data, object_kind) execute_web_hook!(data, "#{object_kind} hook") end @@ -158,5 +185,13 @@ module Integrations # US3 needs to keep a prefix but other datacenters cannot have the listed "app" prefix datadog_site.delete_prefix("app.") end + + def hook_data(data, object_kind) + if object_kind == 'pipeline' && data.respond_to?(:with_retried_builds) + return data.with_retried_builds + end + + data + end end end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index d46299de1be..816f5cbe177 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -303,11 +303,7 @@ module Integrations private def branch_name(commit) - if Feature.enabled?(:jira_use_first_ref_by_oid, project, default_enabled: :yaml) - commit.first_ref_by_oid(project.repository) - else - commit.ref_names(project.repository).first - end + commit.first_ref_by_oid(project.repository) end def server_info diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 10d24ab50b2..b502d5e354d 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -57,7 +57,7 @@ class InternalId < ApplicationRecord self.internal_id_transactions_total.increment( operation: operation, usage: usage.to_s, - in_transaction: ActiveRecord::Base.connection.transaction_open?.to_s # rubocop: disable Database/MultipleDatabases + in_transaction: InternalId.connection.transaction_open?.to_s ) end diff --git a/app/models/issue.rb b/app/models/issue.rb index 537e16e5cc3..4f2773f4147 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -48,7 +48,7 @@ class Issue < ApplicationRecord belongs_to :duplicated_to, class_name: 'Issue' belongs_to :closed_by, class_name: 'User' belongs_to :iteration, foreign_key: 'sprint_id' - belongs_to :work_item_type, class_name: 'WorkItem::Type', inverse_of: :work_items + belongs_to :work_item_type, class_name: 'WorkItems::Type', inverse_of: :work_items belongs_to :moved_to, class_name: 'Issue' has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id @@ -85,13 +85,16 @@ class Issue < ApplicationRecord has_many :issue_customer_relations_contacts, class_name: 'CustomerRelations::IssueContact', inverse_of: :issue has_many :customer_relations_contacts, through: :issue_customer_relations_contacts, source: :contact, class_name: 'CustomerRelations::Contact', inverse_of: :issues + alias_attribute :escalation_status, :incident_management_issuable_escalation_status + accepts_nested_attributes_for :issuable_severity, update_only: true accepts_nested_attributes_for :sentry_issue + accepts_nested_attributes_for :incident_management_issuable_escalation_status, update_only: true validates :project, presence: true validates :issue_type, presence: true - enum issue_type: WorkItem::Type.base_types + enum issue_type: WorkItems::Type.base_types alias_method :issuing_parent, :project @@ -230,8 +233,6 @@ class Issue < ApplicationRecord end def next_object_by_relative_position(ignoring: nil, order: :asc) - return super unless Feature.enabled?(:optimized_issue_neighbor_queries, project, default_enabled: :yaml) - array_mapping_scope = -> (id_expression) do relation = Issue.where(Issue.arel_table[:project_id].eq(id_expression)) @@ -600,6 +601,11 @@ class Issue < ApplicationRecord author&.banned? end + # Necessary until all issues are backfilled and we add a NOT NULL constraint on the DB + def work_item_type + super || WorkItems::Type.default_by_type(issue_type) + end + private def spammable_attribute_changed? diff --git a/app/models/key.rb b/app/models/key.rb index a478434538c..933c939fdf5 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -22,7 +22,7 @@ class Key < ApplicationRecord validates :key, presence: true, length: { maximum: 5000 }, - format: { with: /\A(ssh|ecdsa)-.*\Z/ } + format: { with: /\A(#{Gitlab::SSHPublicKey.supported_algorithms.join('|')})/ } validates :fingerprint, uniqueness: true, diff --git a/app/models/label.rb b/app/models/label.rb index a46d6bc5c0f..0ebbb5b9bd3 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -260,7 +260,7 @@ class Label < ApplicationRecord attributes end - def present(attributes) + def present(attributes = {}) super(**attributes.merge(presenter_class: ::LabelPresenter)) end diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index 0fbdd2d8a5b..db82d5bbf29 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class LooseForeignKeys::DeletedRecord < ApplicationRecord +class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel PARTITION_DURATION = 1.day include PartitionedTable diff --git a/app/models/loose_foreign_keys/modification_tracker.rb b/app/models/loose_foreign_keys/modification_tracker.rb index 6eb04608cd9..72a596d2114 100644 --- a/app/models/loose_foreign_keys/modification_tracker.rb +++ b/app/models/loose_foreign_keys/modification_tracker.rb @@ -4,7 +4,7 @@ module LooseForeignKeys class ModificationTracker MAX_DELETES = 100_000 MAX_UPDATES = 50_000 - MAX_RUNTIME = 3.minutes + MAX_RUNTIME = 30.seconds # must be less than the scheduling frequency of the LooseForeignKeys::CleanupWorker cron worker delegate :monotonic_time, to: :'Gitlab::Metrics::System' diff --git a/app/models/member.rb b/app/models/member.rb index 90fb281abf4..6c0503dca3f 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -18,11 +18,15 @@ class Member < ApplicationRecord AVATAR_SIZE = 40 ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT = 10 + STATE_ACTIVE = 0 + STATE_AWAITING = 1 + attr_accessor :raw_invite_token belongs_to :created_by, class_name: "User" belongs_to :user belongs_to :source, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :member_namespace, inverse_of: :namespace_members, foreign_key: 'member_namespace_id', class_name: 'Namespace' has_one :member_task delegate :name, :username, :email, :last_activity_on, to: :user, prefix: true @@ -231,14 +235,7 @@ class Member < ApplicationRecord end def left_join_users - users = User.arel_table - members = Member.arel_table - - member_users = members.join(users, Arel::Nodes::OuterJoin) - .on(members[:user_id].eq(users[:id])) - .join_sources - - joins(member_users) + left_outer_joins(:user) end def access_for_user_ids(user_ids) diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 1ad4cb6d368..a8a4fbedc41 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -18,7 +18,7 @@ class GroupMember < Member default_scope { where(source_type: SOURCE_TYPE) } # rubocop:disable Cop/DefaultScope - scope :of_groups, ->(groups) { where(source_id: groups.select(:id)) } + scope :of_groups, ->(groups) { where(source_id: groups&.select(:id)) } scope :of_ldap_type, -> { where(ldap: true) } scope :count_users_by_group_id, -> { group(:source_id).count } scope :with_user, -> (user) { where(user: user) } diff --git a/app/models/members/project_namespace_member.rb b/app/models/members/project_namespace_member.rb new file mode 100644 index 00000000000..0e0c52ee3ca --- /dev/null +++ b/app/models/members/project_namespace_member.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# TODO: https://gitlab.com/groups/gitlab-org/-/epics/7054 +# This file is a part of the Consolidate Group and Project member management epic, +# and will be developed further as we progress through that epic. +class ProjectNamespaceMember < ProjectMember # rubocop:disable Gitlab/NamespacedClass +end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index f88aee38d67..cf36e72a565 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1315,9 +1315,9 @@ class MergeRequest < ApplicationRecord self.target_project.repository.branch_exists?(self.target_branch) end - def default_merge_commit_message(include_description: false) + def default_merge_commit_message(include_description: false, user: nil) if self.target_project.merge_commit_template.present? && !include_description - return ::Gitlab::MergeRequests::CommitMessageGenerator.new(merge_request: self).merge_message + return ::Gitlab::MergeRequests::CommitMessageGenerator.new(merge_request: self, current_user: user).merge_message end closes_issues_references = visible_closing_issues_for.map do |issue| @@ -1339,9 +1339,9 @@ class MergeRequest < ApplicationRecord message.join("\n\n") end - def default_squash_commit_message + def default_squash_commit_message(user: nil) if self.target_project.squash_commit_template.present? - return ::Gitlab::MergeRequests::CommitMessageGenerator.new(merge_request: self).squash_message + return ::Gitlab::MergeRequests::CommitMessageGenerator.new(merge_request: self, current_user: user).squash_message end title @@ -1395,20 +1395,6 @@ class MergeRequest < ApplicationRecord actual_head_pipeline.success? end - def environments_for(current_user, latest: false) - return [] unless diff_head_commit - - envs = Environments::EnvironmentsByDeploymentsFinder.new(target_project, current_user, - ref: target_branch, commit: diff_head_commit, with_tags: true, find_latest: latest).execute - - if source_project - envs.concat Environments::EnvironmentsByDeploymentsFinder.new(source_project, current_user, - ref: source_branch, commit: diff_head_commit, find_latest: latest).execute - end - - envs.uniq - end - ## # This method is for looking for active environments which created via pipelines for merge requests. # Since deployments run on a merge request ref (e.g. `refs/merge-requests/:iid/head`), diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 4b1cf2fa217..0dc20e0016c 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -43,6 +43,8 @@ class Namespace < ApplicationRecord has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true + has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' + has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' @@ -299,6 +301,10 @@ class Namespace < ApplicationRecord user_namespace? end + def first_owner + owner + end + def find_fork_of(project) return unless project.fork_network diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index 170b29e9e21..ef917c8a22e 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -3,6 +3,7 @@ class NamespaceSetting < ApplicationRecord include CascadingNamespaceSettingAttribute include Sanitizable + include ChronicDurationAttribute cascading_attr :delayed_project_removal @@ -12,17 +13,19 @@ class NamespaceSetting < ApplicationRecord validate :allow_mfa_for_group validate :allow_resource_access_token_creation_for_group - before_save :set_prevent_sharing_groups_outside_hierarchy, if: -> { user_cap_enabled? } - after_save :disable_project_sharing!, if: -> { user_cap_enabled? } - before_validation :normalize_default_branch_name enum jobs_to_be_done: { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5 }, _suffix: true + chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval + chronic_duration_attr :subgroup_runner_token_expiration_interval_human_readable, :subgroup_runner_token_expiration_interval + chronic_duration_attr :project_runner_token_expiration_interval_human_readable, :project_runner_token_expiration_interval + NAMESPACE_SETTINGS_PARAMS = [:default_branch_name, :delayed_project_removal, :lock_delayed_project_removal, :resource_access_token_creation_allowed, :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, - :setup_for_company, :jobs_to_be_done].freeze + :setup_for_company, :jobs_to_be_done, :runner_token_expiration_interval, + :subgroup_runner_token_expiration_interval, :project_runner_token_expiration_interval].freeze self.primary_key = :namespace_id @@ -59,18 +62,6 @@ class NamespaceSetting < ApplicationRecord errors.add(:resource_access_token_creation_allowed, _('is not allowed since the group is not top-level group.')) end end - - def set_prevent_sharing_groups_outside_hierarchy - self.prevent_sharing_groups_outside_hierarchy = true - end - - def disable_project_sharing! - namespace.update_attribute(:share_with_group_lock, true) - end - - def user_cap_enabled? - new_user_signups_cap.present? && namespace.root? - end end NamespaceSetting.prepend_mod_with('NamespaceSetting') diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 5a5f2a5d063..757a0e40eb3 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -57,6 +57,13 @@ module Namespaces traversal_ids.present? end + def use_traversal_ids_for_self_and_hierarchy? + return false unless use_traversal_ids? + return false unless Feature.enabled?(:use_traversal_ids_for_self_and_hierarchy, root_ancestor, default_enabled: :yaml) + + traversal_ids.present? + end + def use_traversal_ids_for_ancestors? return false unless use_traversal_ids? return false unless Feature.enabled?(:use_traversal_ids_for_ancestors, root_ancestor, default_enabled: :yaml) @@ -107,6 +114,12 @@ module Namespaces self_and_descendants.where.not(id: id) end + def self_and_hierarchy + return super unless use_traversal_ids_for_self_and_hierarchy? + + self_and_descendants.or(ancestors) + end + def ancestors(hierarchy_order: nil) return super unless use_traversal_ids_for_ancestors? diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 0dfb7320461..9f0f49e729c 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -22,19 +22,28 @@ module Namespaces unscoped.where(id: root_ids) end - def self_and_ancestors(include_self: true, hierarchy_order: nil) + def self_and_ancestors(include_self: true, upto: nil, hierarchy_order: nil) return super unless use_traversal_ids_for_ancestor_scopes? + ancestors_cte, base_cte = ancestor_ctes + namespaces = Arel::Table.new(:namespaces) + records = unscoped - .where(id: select('unnest(traversal_ids)')) + .with(base_cte.to_arel, ancestors_cte.to_arel) + .distinct + .from([ancestors_cte.table, namespaces]) + .where(namespaces[:id].eq(ancestors_cte.table[:ancestor_id])) .order_by_depth(hierarchy_order) - .normal_select - if include_self - records - else - records.where.not(id: all.as_ids) + unless include_self + records = records.where(ancestors_cte.table[:base_id].not_eq(ancestors_cte.table[:ancestor_id])) + end + + if upto + records = records.where.not(id: unscoped.where(id: upto).select('unnest(traversal_ids)')) end + + records end def self_and_ancestor_ids(include_self: true) @@ -150,6 +159,20 @@ module Namespaces records.where('namespaces.id <> base.id') end end + + def ancestor_ctes + base_scope = all.select('namespaces.id', 'namespaces.traversal_ids') + base_cte = Gitlab::SQL::CTE.new(:base_ancestors_cte, base_scope) + + # We have to alias id with 'AS' to avoid ambiguous column references by calling methods. + ancestors_scope = unscoped + .unscope(where: [:type]) + .select('id as base_id', 'unnest(traversal_ids) as ancestor_id') + .from(base_cte.table) + ancestors_cte = Gitlab::SQL::CTE.new(:ancestors_cte, ancestors_scope) + + [ancestors_cte, base_cte] + end end end end diff --git a/app/models/namespaces/traversal/recursive_scopes.rb b/app/models/namespaces/traversal/recursive_scopes.rb index 925d9b8bb0c..583c53f8221 100644 --- a/app/models/namespaces/traversal/recursive_scopes.rb +++ b/app/models/namespaces/traversal/recursive_scopes.rb @@ -17,8 +17,8 @@ module Namespaces .where(namespaces: { parent_id: nil }) end - def self_and_ancestors(include_self: true, hierarchy_order: nil) - records = Gitlab::ObjectHierarchy.new(all).base_and_ancestors(hierarchy_order: hierarchy_order) + def self_and_ancestors(include_self: true, upto: nil, hierarchy_order: nil) + records = Gitlab::ObjectHierarchy.new(all).base_and_ancestors(upto: upto, hierarchy_order: hierarchy_order) if include_self records diff --git a/app/models/onboarding_progress.rb b/app/models/onboarding_progress.rb index c12309d1852..58b7848f7e2 100644 --- a/app/models/onboarding_progress.rb +++ b/app/models/onboarding_progress.rb @@ -20,7 +20,14 @@ class OnboardingProgress < ApplicationRecord :issue_created, :issue_auto_closed, :repository_imported, - :repository_mirrored + :repository_mirrored, + :secure_dependency_scanning_run, + :secure_container_scanning_run, + :secure_dast_run, + :secure_secret_detection_run, + :secure_coverage_fuzzing_run, + :secure_api_fuzzing_run, + :secure_cluster_image_scanning_run ].freeze scope :incomplete_actions, -> (actions) do @@ -52,12 +59,19 @@ class OnboardingProgress < ApplicationRecord where(namespace: namespace).any? end - def register(namespace, action) - return unless root_namespace?(namespace) && ACTIONS.include?(action) + def register(namespace, actions) + actions = Array(actions) + return unless root_namespace?(namespace) && actions.difference(ACTIONS).empty? - action_column = column_name(action) - onboarding_progress = find_by(namespace: namespace, action_column => nil) - onboarding_progress&.update!(action_column => Time.current) + onboarding_progress = find_by(namespace: namespace) + return unless onboarding_progress + + now = Time.current + nil_actions = actions.select { |action| onboarding_progress[column_name(action)].nil? } + return if nil_actions.empty? + + updates = nil_actions.inject({}) { |sum, action| sum.merge!({ column_name(action) => now }) } + onboarding_progress.update!(updates) end def completed?(namespace, action) diff --git a/app/models/packages/debian/group_distribution.rb b/app/models/packages/debian/group_distribution.rb index 50c1ec9f163..01938f4a2ec 100644 --- a/app/models/packages/debian/group_distribution.rb +++ b/app/models/packages/debian/group_distribution.rb @@ -12,8 +12,4 @@ class Packages::Debian::GroupDistribution < ApplicationRecord .for_projects(group.all_projects.public_only) .with_debian_codename(codename) end - - def package_files - ::Packages::PackageFile.for_package_ids(packages.select(:id)) - end end diff --git a/app/models/packages/debian/project_distribution.rb b/app/models/packages/debian/project_distribution.rb index 5ac60d789b3..73777e3b9d8 100644 --- a/app/models/packages/debian/project_distribution.rb +++ b/app/models/packages/debian/project_distribution.rb @@ -9,5 +9,4 @@ class Packages::Debian::ProjectDistribution < ApplicationRecord has_many :publications, class_name: 'Packages::Debian::Publication', inverse_of: :distribution, foreign_key: :distribution_id has_many :packages, class_name: 'Packages::Package', through: :publications - has_many :package_files, class_name: 'Packages::PackageFile', through: :packages end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 962a1057a22..52dd0aba43b 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -5,9 +5,10 @@ class Packages::Package < ApplicationRecord include Gitlab::SQL::Pattern include UsageStatistics include Gitlab::Utils::StrongMemoize + include Packages::Installable DISPLAYABLE_STATUSES = [:default, :error].freeze - INSTALLABLE_STATUSES = [:default].freeze + INSTALLABLE_STATUSES = [:default, :hidden].freeze enum package_type: { maven: 1, @@ -31,6 +32,9 @@ class Packages::Package < ApplicationRecord # package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + # TODO: put the installable default scope on the :package_files association once the dependent: :destroy is removed + # See https://gitlab.com/gitlab-org/gitlab/-/issues/349191 + has_many :installable_package_files, -> { installable }, class_name: 'Packages::PackageFile', inverse_of: :package has_many :dependency_links, inverse_of: :package, class_name: 'Packages::DependencyLink' has_many :tags, inverse_of: :package, class_name: 'Packages::Tag' has_one :conan_metadatum, inverse_of: :package, class_name: 'Packages::Conan::Metadatum' @@ -100,9 +104,7 @@ class Packages::Package < ApplicationRecord scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } scope :with_package_type, ->(package_type) { where(package_type: package_type) } scope :without_package_type, ->(package_type) { where.not(package_type: package_type) } - scope :with_status, ->(status) { where(status: status) } scope :displayable, -> { with_status(DISPLAYABLE_STATUSES) } - scope :installable, -> { with_status(INSTALLABLE_STATUSES) } scope :including_project_route, -> { includes(project: { namespace: :route }) } scope :including_tags, -> { includes(:tags) } scope :including_dependency_links, -> { includes(dependency_links: :dependency) } @@ -131,7 +133,7 @@ class Packages::Package < ApplicationRecord scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) } scope :has_version, -> { where.not(version: nil) } - scope :preload_files, -> { preload(:package_files) } + scope :preload_files, -> { Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) ? preload(:installable_package_files) : preload(:package_files) } scope :preload_pipelines, -> { preload(pipelines: :user) } scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) } scope :limit_recent, ->(limit) { order_created_desc.limit(limit) } diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 87c9f56cc41..072ff4a3a5f 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -2,12 +2,18 @@ class Packages::PackageFile < ApplicationRecord include UpdateProjectStatistics include FileStoreMounter + include Packages::Installable + include Packages::Destructible + + INSTALLABLE_STATUSES = [:default].freeze delegate :project, :project_id, to: :package delegate :conan_file_type, to: :conan_file_metadatum delegate :file_type, :dsc?, :component, :architecture, :fields, to: :debian_file_metadatum, prefix: :debian delegate :channel, :metadata, to: :helm_file_metadatum, prefix: :helm + enum status: { default: 0, pending_destruction: 1, processing: 2, error: 3 } + belongs_to :package # used to move the linked file within object storage @@ -48,9 +54,12 @@ class Packages::PackageFile < ApplicationRecord end scope :for_helm_with_channel, ->(project, channel) do - joins(:package).merge(project.packages.helm.installable) - .joins(:helm_file_metadatum) - .where(packages_helm_file_metadata: { channel: channel }) + result = joins(:package) + .merge(project.packages.helm.installable) + .joins(:helm_file_metadatum) + .where(packages_helm_file_metadata: { channel: channel }) + result = result.installable if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + result end scope :with_conan_file_type, ->(file_type) do @@ -94,14 +103,19 @@ class Packages::PackageFile < ApplicationRecord skip_callback :commit, :after, :remove_previously_stored_file, if: :execute_move_in_object_storage? after_commit :move_in_object_storage, if: :execute_move_in_object_storage? - # Returns the most recent package files for *each* of the given packages. + # Returns the most recent installable package file for *each* of the given packages. # The order is not guaranteed. def self.most_recent_for(packages, extra_join: nil, extra_where: nil) cte_name = :packages_cte cte = Gitlab::SQL::CTE.new(cte_name, packages.select(:id)) - package_files = ::Packages::PackageFile.limit_recent(1) - .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) + package_files = if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) + ::Packages::PackageFile.installable.limit_recent(1) + .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) + else + ::Packages::PackageFile.limit_recent(1) + .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) + end package_files = package_files.joins(extra_join) if extra_join package_files = package_files.where(extra_where) if extra_where diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 0c5a155d48a..c21027455b1 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -46,9 +46,6 @@ class PagesDomain < ApplicationRecord algorithm: 'aes-256-cbc' after_initialize :set_verification_code - after_create :update_daemon - after_update :update_daemon, if: :saved_change_to_pages_config? - after_destroy :update_daemon scope :for_project, ->(project) { where(project: project) } @@ -233,32 +230,6 @@ class PagesDomain < ApplicationRecord self.verification_code = SecureRandom.hex(16) end - # rubocop: disable CodeReuse/ServiceClass - def update_daemon - return if usage_serverless? - return unless pages_deployed? - - run_after_commit { PagesUpdateConfigurationWorker.perform_async(project_id) } - end - # rubocop: enable CodeReuse/ServiceClass - - def saved_change_to_pages_config? - saved_change_to_project_id? || - saved_change_to_domain? || - saved_change_to_certificate? || - saved_change_to_key? || - became_enabled? || - became_disabled? - end - - def became_enabled? - enabled_until.present? && !enabled_until_before_last_save.present? - end - - def became_disabled? - !enabled_until.present? && enabled_until_before_last_save.present? - end - def validate_matching_key unless has_matching_key? self.errors.add(:key, "doesn't match the certificate") diff --git a/app/models/preloaders/environments/deployment_preloader.rb b/app/models/preloaders/environments/deployment_preloader.rb new file mode 100644 index 00000000000..fcf892698bb --- /dev/null +++ b/app/models/preloaders/environments/deployment_preloader.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Preloaders + module Environments + # This class is to batch-load deployments of multiple environments. + # The deployments to batch-load are fetched using UNION of N selects in a single query instead of default scoping with `IN (environment_id1, environment_id2 ...)`. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/345672#note_761852224 for more details. + class DeploymentPreloader + attr_reader :environments + + def initialize(environments) + @environments = environments + end + + def execute_with_union(association_name, association_attributes) + load_deployment_association(association_name, association_attributes) + end + + private + + def load_deployment_association(association_name, association_attributes) + return unless environments.present? + + union_arg = environments.inject([]) do |result, environment| + result << environment.association(association_name).scope + end + + union_sql = Deployment.from_union(union_arg).to_sql + + deployments = Deployment + .from("(#{union_sql}) #{::Deployment.table_name}") + .preload(association_attributes) + + deployments_by_environment_id = deployments.index_by(&:environment_id) + + environments.each do |environment| + environment.association(association_name).target = deployments_by_environment_id[environment.id] + environment.association(association_name).loaded! + end + end + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index a751e8adeb0..f2b3db684ae 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -37,6 +37,7 @@ class Project < ApplicationRecord include EachBatch include GitlabRoutingHelper include BulkMemberAccessLoad + include RunnerTokenExpirationInterval extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -340,6 +341,7 @@ class Project < ApplicationRecord has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' + has_many :secure_files, class_name: 'Ci::SecureFile' has_many :environments has_many :environments_for_dashboard, -> { from(with_rank.unfoldered.available, :environments).where('rank <= 3') }, class_name: 'Environment' has_many :deployments @@ -436,6 +438,7 @@ class Project < ApplicationRecord prefix: :import, to: :import_state, allow_nil: true delegate :squash_always?, :squash_never?, :squash_enabled_by_default?, :squash_readonly?, to: :project_setting delegate :squash_option, :squash_option=, to: :project_setting + delegate :mr_default_target_self, :mr_default_target_self=, to: :project_setting delegate :previous_default_branch, :previous_default_branch=, to: :project_setting delegate :no_import?, to: :import_state, allow_nil: true delegate :name, to: :owner, allow_nil: true, prefix: true @@ -452,7 +455,8 @@ class Project < ApplicationRecord delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :keep_latest_artifact, :keep_latest_artifact=, to: :ci_cd_settings, allow_nil: true delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, to: :ci_cd_settings, allow_nil: true - delegate :actual_limits, :actual_plan_name, to: :namespace, allow_nil: true + delegate :runner_token_expiration_interval, :runner_token_expiration_interval=, :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval_human_readable=, to: :ci_cd_settings, allow_nil: true + delegate :actual_limits, :actual_plan_name, :actual_plan, to: :namespace, allow_nil: true delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, :allow_merge_on_skipped_pipeline=, :has_confluence?, :has_shimo?, to: :project_setting @@ -983,7 +987,7 @@ class Project < ApplicationRecord end def context_commits_enabled? - Feature.enabled?(:context_commits, default_enabled: true) + Feature.enabled?(:context_commits, self, default_enabled: :yaml) end # LFS and hashed repository storage are required for using Design Management. @@ -1512,11 +1516,11 @@ class Project < ApplicationRecord group || namespace.try(:owner) end - def default_owner + def first_owner obj = owner - if obj.respond_to?(:default_owner) - obj.default_owner + if obj.respond_to?(:first_owner) + obj.first_owner else obj end @@ -1660,7 +1664,7 @@ class Project < ApplicationRecord attrs end - def project_member(user) + def member(user) if project_members.loaded? project_members.find { |member| member.user_id == user.id } else @@ -1773,17 +1777,12 @@ class Project < ApplicationRecord def all_runners Ci::Runner.from_union([runners, group_runners, shared_runners]) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') end def all_available_runners Ci::Runner.from_union([runners, group_runners, available_shared_runners]) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') end - # Once issue 339937 is fixed, please search for all mentioned of - # https://gitlab.com/gitlab-org/gitlab/-/issues/339937, - # and remove the allow_cross_joins_across_databases. def active_runners strong_memoize(:active_runners) do all_available_runners.active @@ -1791,9 +1790,7 @@ class Project < ApplicationRecord end def any_online_runners?(&block) - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do - online_runners_with_tags.any?(&block) - end + online_runners_with_tags.any?(&block) end def valid_runners_token?(token) @@ -2702,6 +2699,10 @@ class Project < ApplicationRecord ci_cd_settings.keep_latest_artifact? end + def runner_token_expiration_interval + ci_cd_settings&.runner_token_expiration_interval + end + def group_runners_enabled? return false unless ci_cd_settings @@ -2733,6 +2734,17 @@ class Project < ApplicationRecord end end + def enforced_runner_token_expiration_interval + all_parent_groups = Gitlab::ObjectHierarchy.new(Group.where(id: group)).base_and_ancestors + all_group_settings = NamespaceSetting.where(namespace_id: all_parent_groups) + group_interval = all_group_settings.where.not(project_runner_token_expiration_interval: nil).minimum(:project_runner_token_expiration_interval)&.seconds + + [ + Gitlab::CurrentSettings.project_runner_token_expiration_interval&.seconds, + group_interval + ].compact.min + end + private # overridden in EE diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index c0c2ea42d46..28a493cae33 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true class ProjectCiCdSetting < ApplicationRecord + include ChronicDurationAttribute + belongs_to :project, inverse_of: :ci_cd_settings - DEFAULT_GIT_DEPTH = 50 + DEFAULT_GIT_DEPTH = 20 before_create :set_default_git_depth @@ -17,6 +19,8 @@ class ProjectCiCdSetting < ApplicationRecord default_value_for :forward_deployment_enabled, true + chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval + def forward_deployment_enabled? super && ::Feature.enabled?(:forward_deployment_enabled, project, default_enabled: true) end diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index fc834286876..4e37174e604 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -22,6 +22,16 @@ class ProjectSetting < ApplicationRecord def squash_readonly? %w[always never].include?(squash_option) end + + validate :validates_mr_default_target_self + + private + + def validates_mr_default_target_self + if mr_default_target_self_changed? && !project.forked? + errors.add :mr_default_target_self, _('This setting is allowed for forked projects only') + end + end end ProjectSetting.prepend_mod diff --git a/app/models/protectable_dropdown.rb b/app/models/protectable_dropdown.rb index e1336be9528..855876f2ec9 100644 --- a/app/models/protectable_dropdown.rb +++ b/app/models/protectable_dropdown.rb @@ -2,6 +2,10 @@ class ProtectableDropdown REF_TYPES = %i[branches tags].freeze + REF_NAME_METHODS = { + branches: :branch_names, + tags: :tag_names + }.freeze def initialize(project, ref_type) raise ArgumentError, "invalid ref type `#{ref_type}`" unless ref_type.in?(REF_TYPES) @@ -23,12 +27,12 @@ class ProtectableDropdown private - def refs - @project.repository.public_send(@ref_type) # rubocop:disable GitlabSecurity/PublicSend + def ref_names + @project.repository.public_send(ref_name_method) # rubocop:disable GitlabSecurity/PublicSend end - def ref_names - refs.map(&:name) + def ref_name_method + REF_NAME_METHODS[@ref_type] end def protections diff --git a/app/models/ref_matcher.rb b/app/models/ref_matcher.rb index fa7d2c0f06c..46f4ce0ecc7 100644 --- a/app/models/ref_matcher.rb +++ b/app/models/ref_matcher.rb @@ -5,10 +5,10 @@ class RefMatcher @ref_name_or_pattern = ref_name_or_pattern end - # Returns all branches/tags (among the given list of refs [`Gitlab::Git::Branch`]) + # Returns all branches/tags (among the given list of refs [`Gitlab::Git::Branch`] or their names [`String`]) # that match the current protected ref. def matching(refs) - refs.select { |ref| matches?(ref.name) } + refs.select { |ref| ref.is_a?(String) ? matches?(ref) : matches?(ref.name) } end # Checks if the protected ref matches the given ref name. diff --git a/app/models/repository.rb b/app/models/repository.rb index 645cc9773bd..be8e530c650 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -191,7 +191,7 @@ class Repository end def find_tag(name) - if @tags.blank? && Feature.enabled?(:find_tag_via_gitaly, project, default_enabled: :yaml) + if @tags.blank? raw_repository.find_tag(name) else tags.find { |tag| tag.name == name } diff --git a/app/models/route.rb b/app/models/route.rb index fcc8459d6e5..12b2d5c5bb2 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -5,6 +5,7 @@ class Route < ApplicationRecord include Gitlab::SQL::Pattern belongs_to :source, polymorphic: true, inverse_of: :route # rubocop:disable Cop/PolymorphicAssociations + belongs_to :namespace, inverse_of: :namespace_route validates :source, presence: true validates :path, diff --git a/app/models/user.rb b/app/models/user.rb index a39da30220a..a587723053f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -48,7 +48,7 @@ class User < ApplicationRecord add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } add_authentication_token_field :feed_token - add_authentication_token_field :static_object_token + add_authentication_token_field :static_object_token, encrypted: :optional default_value_for :admin, false default_value_for(:external) { Gitlab::CurrentSettings.user_default_external } @@ -81,6 +81,7 @@ class User < ApplicationRecord # This module adds async behaviour to Devise emails # and should be added after Devise modules are initialized. include AsyncDeviseEmail + include ForcedEmailConfirmation MINIMUM_INACTIVE_DAYS = 90 @@ -250,7 +251,7 @@ class User < ApplicationRecord validate :notification_email_verified, if: :notification_email_changed? validate :public_email_verified, if: :public_email_changed? validate :commit_email_verified, if: :commit_email_changed? - validate :signup_email_valid?, on: :create, if: ->(user) { !user.created_by_id } + validate :email_allowed_by_restrictions?, if: ->(user) { user.new_record? ? !user.created_by_id : user.email_changed? } validate :check_username_format, if: :username_changed? validates :theme_id, allow_nil: true, inclusion: { in: Gitlab::Themes.valid_ids, @@ -330,6 +331,7 @@ class User < ApplicationRecord delegate :pronouns, :pronouns=, to: :user_detail, allow_nil: true delegate :pronunciation, :pronunciation=, to: :user_detail, allow_nil: true delegate :registration_objective, :registration_objective=, to: :user_detail, allow_nil: true + delegate :requires_credit_card_verification, :requires_credit_card_verification=, to: :user_detail, allow_nil: true accepts_nested_attributes_for :user_preference, update_only: true accepts_nested_attributes_for :user_detail, update_only: true @@ -465,7 +467,7 @@ class User < ApplicationRecord scope :dormant, -> { with_state(:active).human_or_service_user.where('last_activity_on <= ?', MINIMUM_INACTIVE_DAYS.day.ago.to_date) } scope :with_no_activity, -> { with_state(:active).human_or_service_user.where(last_activity_on: nil) } scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } - scope :get_ids_by_username, -> (username) { where(username: username).pluck(:id) } + scope :by_ids_or_usernames, -> (ids, usernames) { where(username: usernames).or(where(id: ids)) } strip_attributes! :name @@ -536,27 +538,15 @@ class User < ApplicationRecord end def self.with_two_factor - with_u2f_registrations = <<-SQL - EXISTS ( - SELECT * - FROM u2f_registrations AS u2f - WHERE u2f.user_id = users.id - ) OR users.otp_required_for_login = ? - OR - EXISTS ( - SELECT * - FROM webauthn_registrations AS webauthn - WHERE webauthn.user_id = users.id - ) - SQL - - where(with_u2f_registrations, true) + where(otp_required_for_login: true) + .or(where_exists(U2fRegistration.where(U2fRegistration.arel_table[:user_id].eq(arel_table[:id])))) + .or(where_exists(WebauthnRegistration.where(WebauthnRegistration.arel_table[:user_id].eq(arel_table[:id])))) end def self.without_two_factor - joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id - LEFT OUTER JOIN webauthn_registrations AS webauthn ON webauthn.user_id = users.id") - .where("u2f.id IS NULL AND webauthn.id IS NULL AND users.otp_required_for_login = ?", false) + where + .missing(:u2f_registrations, :webauthn_registrations) + .where(otp_required_for_login: false) end # @@ -720,13 +710,19 @@ class User < ApplicationRecord .take(1) # at most 1 record as there is a unique constraint where( - fuzzy_arel_match(:name, query) - .or(fuzzy_arel_match(:username, query)) + fuzzy_arel_match(:name, query, use_minimum_char_limit: user_search_minimum_char_limit) + .or(fuzzy_arel_match(:username, query, use_minimum_char_limit: user_search_minimum_char_limit)) .or(arel_table[:email].eq(query)) .or(arel_table[:id].eq(matched_by_email_user_id)) ) end + # This method is overridden in JiHu. + # https://gitlab.com/gitlab-org/gitlab/-/issues/348509 + def user_search_minimum_char_limit + true + end + def by_login(login) return unless login @@ -841,6 +837,10 @@ class User < ApplicationRecord def single_user User.non_internal.first if single_user? end + + def get_ids_by_ids_or_usernames(ids, usernames) + by_ids_or_usernames(ids, usernames).pluck(:id) + end end # @@ -1337,7 +1337,7 @@ class User < ApplicationRecord def can_leave_project?(project) project.namespace != namespace && - project.project_member(self) + project.member(self) end def full_website_url @@ -1536,8 +1536,8 @@ class User < ApplicationRecord end end - def manageable_namespaces - @manageable_namespaces ||= [namespace] + manageable_groups + def forkable_namespaces + @forkable_namespaces ||= [namespace] + manageable_groups(include_groups_with_developer_maintainer_access: true) end def manageable_groups(include_groups_with_developer_maintainer_access: false) @@ -1606,23 +1606,32 @@ class User < ApplicationRecord def ci_owned_runners @ci_owned_runners ||= begin - project_runners = Ci::RunnerProject - .where(project: authorized_projects(Gitlab::Access::MAINTAINER)) - .joins(:runner) - .select('ci_runners.*') - - group_runners = Ci::RunnerNamespace - .where(namespace_id: owned_groups.self_and_descendant_ids) - .joins(:runner) - .select('ci_runners.*') - - Ci::Runner.from_union([project_runners, group_runners]).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') + if ci_owned_runners_cross_joins_fix_enabled? + Ci::Runner + .from_union([ci_owned_project_runners_from_project_members, + ci_owned_project_runners_from_group_members, + ci_owned_group_runners]) + else + Ci::Runner + .from_union([ci_legacy_owned_project_runners, ci_legacy_owned_group_runners]) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') + end end end def owns_runner?(runner) - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do + if ci_owned_runners_cross_joins_fix_enabled? ci_owned_runners.exists?(runner.id) + else + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do + ci_owned_runners.exists?(runner.id) + end + end + end + + def ci_owned_runners_cross_joins_fix_enabled? + strong_memoize(:ci_owned_runners_cross_joins_fix_enabled) do + Feature.enabled?(:ci_owned_runners_cross_joins_fix, self, default_enabled: :yaml) end end @@ -1902,6 +1911,10 @@ class User < ApplicationRecord true end + def can_log_in_with_non_expired_password? + can?(:log_in) && !password_expired_if_applicable? + end + def can_be_deactivated? active? && no_recent_activity? && !internal? end @@ -1980,18 +1993,22 @@ class User < ApplicationRecord ci_job_token_scope.present? end - # override from Devise::Confirmable + # override from Devise::Models::Confirmable # # Add the primary email to user.emails (or confirm it if it was already # present) when the primary email is confirmed. - def confirm(*args) - saved = super(*args) + def confirm(args = {}) + saved = super(args) return false unless saved email_to_confirm = self.emails.find_by(email: self.email) if email_to_confirm.present? - email_to_confirm.confirm(*args) + if skip_confirmation_period_expiry_check + email_to_confirm.force_confirm(args) + else + email_to_confirm.confirm(args) + end else add_primary_email_to_emails! end @@ -2142,14 +2159,14 @@ class User < ApplicationRecord end end - def signup_email_valid? + def email_allowed_by_restrictions? error = validate_admin_signup_restrictions(email) errors.add(:email, error) if error end def signup_email_invalid_message - _('is not allowed for sign-up.') + self.new_record? ? _('is not allowed for sign-up.') : _('is not allowed.') end def check_username_format @@ -2192,6 +2209,50 @@ class User < ApplicationRecord ::Gitlab::Auth::Ldap::Access.allowed?(self) end + + def ci_legacy_owned_project_runners + Ci::RunnerProject + .select('ci_runners.*') + .joins(:runner) + .where(project: authorized_projects(Gitlab::Access::MAINTAINER)) + end + + def ci_legacy_owned_group_runners + Ci::RunnerNamespace + .select('ci_runners.*') + .joins(:runner) + .where(namespace_id: owned_groups.self_and_descendant_ids) + end + + def ci_owned_project_runners_from_project_members + Ci::RunnerProject + .select('ci_runners.*') + .joins(:runner) + .where(project: project_members.where('access_level >= ?', Gitlab::Access::MAINTAINER).pluck(:source_id)) + end + + def ci_owned_project_runners_from_group_members + Ci::RunnerProject + .select('ci_runners.*') + .joins(:runner) + .joins('JOIN ci_project_mirrors ON ci_project_mirrors.project_id = ci_runner_projects.project_id') + .joins('JOIN ci_namespace_mirrors ON ci_namespace_mirrors.namespace_id = ci_project_mirrors.namespace_id') + .merge(ci_namespace_mirrors_for_group_members(Gitlab::Access::MAINTAINER)) + end + + def ci_owned_group_runners + Ci::RunnerNamespace + .select('ci_runners.*') + .joins(:runner) + .joins('JOIN ci_namespace_mirrors ON ci_namespace_mirrors.namespace_id = ci_runner_namespaces.namespace_id') + .merge(ci_namespace_mirrors_for_group_members(Gitlab::Access::OWNER)) + end + + def ci_namespace_mirrors_for_group_members(level) + Ci::NamespaceMirror.contains_any_of_namespaces( + group_members.where('access_level >= ?', level).pluck(:source_id) + ) + end end User.prepend_mod_with('User') diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 9ce0beed3b3..8394192c5ae 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -40,7 +40,8 @@ module Users profile_personal_access_token_expiry: 37, # EE-only terraform_notification_dismissed: 38, security_newsletter_callout: 39, - verification_reminder: 40 # EE-only + verification_reminder: 40, # EE-only + ci_deprecation_warning_for_types_keyword: 41 } validates :feature_name, diff --git a/app/models/work_item.rb b/app/models/work_item.rb new file mode 100644 index 00000000000..02f52f04c85 --- /dev/null +++ b/app/models/work_item.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +class WorkItem < Issue + self.table_name = 'issues' + self.inheritance_column = :_type_disabled +end diff --git a/app/models/work_item/type.rb b/app/models/work_item/type.rb deleted file mode 100644 index 3acb9c0011c..00000000000 --- a/app/models/work_item/type.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -# Note: initial thinking behind `icon_name` is for it to do triple duty: -# 1. one of our svg icon names, such as `external-link` or a new one `bug` -# 2. if it's an absolute url, then url to a user uploaded icon/image -# 3. an emoji, with the format of `:smile:` -class WorkItem::Type < ApplicationRecord - self.table_name = 'work_item_types' - - include CacheMarkdownField - - # Base types need to exist on the DB on app startup - # This constant is used by the DB seeder - BASE_TYPES = { - issue: { name: 'Issue', icon_name: 'issue-type-issue', enum_value: 0 }, - incident: { name: 'Incident', icon_name: 'issue-type-incident', enum_value: 1 }, - test_case: { name: 'Test Case', icon_name: 'issue-type-test-case', enum_value: 2 }, ## EE-only - requirement: { name: 'Requirement', icon_name: 'issue-type-requirements', enum_value: 3 }, ## EE-only - task: { name: 'Task', icon_name: 'issue-type-task', enum_value: 4 } - }.freeze - - cache_markdown_field :description, pipeline: :single_line - - enum base_type: BASE_TYPES.transform_values { |value| value[:enum_value] } - - belongs_to :namespace, optional: true - has_many :work_items, class_name: 'Issue', foreign_key: :work_item_type_id, inverse_of: :work_item_type - - before_validation :strip_whitespace - - # TODO: review validation rules - # https://gitlab.com/gitlab-org/gitlab/-/issues/336919 - validates :name, presence: true - validates :name, uniqueness: { case_sensitive: false, scope: [:namespace_id] } - validates :name, length: { maximum: 255 } - validates :icon_name, length: { maximum: 255 } - - def self.default_by_type(type) - find_by(namespace_id: nil, base_type: type) - end - - def self.default_issue_type - default_by_type(:issue) - end - - def self.allowed_types_for_issues - base_types.keys.excluding('task') - end - - private - - def strip_whitespace - name&.strip! - end -end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb new file mode 100644 index 00000000000..494c4f5abe4 --- /dev/null +++ b/app/models/work_items/type.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +# Note: initial thinking behind `icon_name` is for it to do triple duty: +# 1. one of our svg icon names, such as `external-link` or a new one `bug` +# 2. if it's an absolute url, then url to a user uploaded icon/image +# 3. an emoji, with the format of `:smile:` +module WorkItems + class Type < ApplicationRecord + self.table_name = 'work_item_types' + + include CacheMarkdownField + + # Base types need to exist on the DB on app startup + # This constant is used by the DB seeder + BASE_TYPES = { + issue: { name: 'Issue', icon_name: 'issue-type-issue', enum_value: 0 }, + incident: { name: 'Incident', icon_name: 'issue-type-incident', enum_value: 1 }, + test_case: { name: 'Test Case', icon_name: 'issue-type-test-case', enum_value: 2 }, ## EE-only + requirement: { name: 'Requirement', icon_name: 'issue-type-requirements', enum_value: 3 }, ## EE-only + task: { name: 'Task', icon_name: 'issue-type-task', enum_value: 4 } + }.freeze + + cache_markdown_field :description, pipeline: :single_line + + enum base_type: BASE_TYPES.transform_values { |value| value[:enum_value] } + + belongs_to :namespace, optional: true + has_many :work_items, class_name: 'Issue', foreign_key: :work_item_type_id, inverse_of: :work_item_type + + before_validation :strip_whitespace + + # TODO: review validation rules + # https://gitlab.com/gitlab-org/gitlab/-/issues/336919 + validates :name, presence: true + validates :name, uniqueness: { case_sensitive: false, scope: [:namespace_id] } + validates :name, length: { maximum: 255 } + validates :icon_name, length: { maximum: 255 } + + scope :default, -> { where(namespace: nil) } + scope :order_by_name_asc, -> { order('LOWER(name)') } + + def self.default_by_type(type) + find_by(namespace_id: nil, base_type: type) + end + + def self.default_issue_type + default_by_type(:issue) + end + + def self.allowed_types_for_issues + base_types.keys.excluding('task') + end + + def default? + namespace.blank? + end + + private + + def strip_whitespace + name&.strip! + end + end +end |