diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-18 09:45:46 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-18 09:45:46 +0000 |
commit | a7b3560714b4d9cc4ab32dffcd1f74a284b93580 (patch) | |
tree | 7452bd5c3545c2fa67a28aa013835fb4fa071baf /app/models | |
parent | ee9173579ae56a3dbfe5afe9f9410c65bb327ca7 (diff) | |
download | gitlab-ce-a7b3560714b4d9cc4ab32dffcd1f74a284b93580.tar.gz |
Add latest changes from gitlab-org/gitlab@14-8-stable-eev14.8.0-rc42
Diffstat (limited to 'app/models')
77 files changed, 1303 insertions, 310 deletions
diff --git a/app/models/application_record.rb b/app/models/application_record.rb index b64e6c59817..06ff18ca409 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -4,6 +4,7 @@ class ApplicationRecord < ActiveRecord::Base include DatabaseReflection include Transactions include LegacyBulkInsert + include CrossDatabaseModification self.abstract_class = true diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 3c9f7c4dd7f..02fbf0f855e 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -563,6 +563,12 @@ class ApplicationSetting < ApplicationRecord presence: true, length: { maximum: 255 }, if: :sentry_enabled? + validates :users_get_by_id_limit, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :users_get_by_id_limit_allowlist, + length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, + allow_nil: false + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 25198178f69..415f0b35f3a 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -69,7 +69,9 @@ module ApplicationSettingImplementation domain_allowlist: Settings.gitlab['domain_allowlist'], dsa_key_restriction: 0, ecdsa_key_restriction: 0, + ecdsa_sk_key_restriction: 0, ed25519_key_restriction: 0, + ed25519_sk_key_restriction: 0, eks_access_key_id: nil, eks_account_id: nil, eks_integration_enabled: false, @@ -229,7 +231,9 @@ module ApplicationSettingImplementation rate_limiting_response_text: nil, whats_new_variant: 0, user_deactivation_emails_enabled: true, - user_email_lookup_limit: 60 + user_email_lookup_limit: 60, + users_get_by_id_limit: 300, + users_get_by_id_limit_allowlist: [] } end @@ -332,6 +336,14 @@ module ApplicationSettingImplementation self.notes_create_limit_allowlist = strings_to_array(values).map(&:downcase) end + def users_get_by_id_limit_allowlist_raw + array_to_string(self.users_get_by_id_limit_allowlist) + end + + def users_get_by_id_limit_allowlist_raw=(values) + self.users_get_by_id_limit_allowlist = strings_to_array(values).map(&:downcase) + end + def asset_proxy_whitelist=(values) values = strings_to_array(values) if values.is_a?(String) diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 35c4e08730e..8e8e9389e2d 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -69,8 +69,7 @@ class AuditEvent < ApplicationRecord end def author - lazy_author&.itself.presence || - ::Gitlab::Audit::NullAuthor.for(author_id, (self[:author_name] || details[:author_name])) + lazy_author&.itself.presence || default_author_value end def lazy_author @@ -98,7 +97,7 @@ class AuditEvent < ApplicationRecord end def default_author_value - ::Gitlab::Audit::NullAuthor.for(author_id, (self[:author_name] || details[:author_name])) + ::Gitlab::Audit::NullAuthor.for(author_id, self) end def parallel_persist diff --git a/app/models/blob.rb b/app/models/blob.rb index 5731d38abe4..cc7758d9674 100644 --- a/app/models/blob.rb +++ b/app/models/blob.rb @@ -178,6 +178,10 @@ class Blob < SimpleDelegator end end + def symlink? + mode == MODE_SYMLINK + end + def extension @extension ||= extname.downcase.delete('.') end diff --git a/app/models/board.rb b/app/models/board.rb index 7938819b6e4..8a7330e7320 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Board < ApplicationRecord + RECENT_BOARDS_SIZE = 4 + belongs_to :group belongs_to :project diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index ce3faf3546b..d5cbbb96134 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -6,7 +6,7 @@ module Ci class NamespaceMirror < ApplicationRecord belongs_to :namespace - scope :contains_namespace, -> (id) do + scope :by_group_and_descendants, -> (id) do where('traversal_ids @> ARRAY[?]::int[]', id) end @@ -32,7 +32,7 @@ module Ci private def sync_children_namespaces!(namespace_id, traversal_ids) - contains_namespace(namespace_id) + by_group_and_descendants(namespace_id) .where.not(namespace_id: namespace_id) .update_all( "traversal_ids = ARRAY[#{sanitize_sql(traversal_ids.join(','))}]::int[] || traversal_ids[array_position(traversal_ids, #{sanitize_sql(namespace_id)}) + 1:]" diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 00d331df4c3..a1311b8555f 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -40,6 +40,10 @@ module Ci # https://gitlab.com/gitlab-org/gitlab/-/issues/259010 attr_accessor :merged_yaml + # This is used to retain access to the method defined by `Ci::HasRef` + # before being overridden in this class. + alias_method :jobs_git_ref, :git_ref + belongs_to :project, inverse_of: :all_pipelines belongs_to :user belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' @@ -72,8 +76,6 @@ module Ci has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' - has_many :deployments, through: :builds - has_many :environments, -> { distinct.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338658') }, through: :deployments has_many :latest_builds, -> { latest.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :downloadable_artifacts, -> do not_expired.or(where_exists(::Ci::Pipeline.artifacts_locked.where('ci_pipelines.id = ci_builds.commit_id'))).downloadable.with_job @@ -352,7 +354,7 @@ module Ci # # ref - The name (or names) of the branch(es)/tag(s) to limit the list of # pipelines to. - # sha - The commit SHA (or mutliple SHAs) to limit the list of pipelines to. + # sha - The commit SHA (or multiple SHAs) to limit the list of pipelines to. # limit - This limits a backlog search, default to 100. def self.newest_first(ref: nil, sha: nil, limit: 100) relation = order(id: :desc) @@ -1163,7 +1165,11 @@ module Ci end def merge_request? - merge_request_id.present? + if Feature.enabled?(:ci_pipeline_merge_request_presence_check, default_enabled: :yaml) + merge_request_id.present? && merge_request + else + merge_request_id.present? + end end def external_pull_request? @@ -1284,18 +1290,6 @@ module Ci end end - def create_deployment_in_separate_transaction? - strong_memoize(:create_deployment_in_separate_transaction) do - ::Feature.enabled?(:create_deployment_in_separate_transaction, project, default_enabled: :yaml) - end - end - - def use_variables_builder_definitions? - strong_memoize(:use_variables_builder_definitions) do - ::Feature.enabled?(:ci_use_variables_builder_definitions, project, default_enabled: :yaml) - end - end - private def add_message(severity, content) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 809c245d2b9..11150e839a3 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -13,7 +13,7 @@ module Ci include TaggableQueries include Presentable - add_authentication_token_field :token, encrypted: :optional + add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration, expiration_enforced?: :token_expiration_enforced? enum access_level: { not_protected: 0, @@ -67,7 +67,7 @@ module Ci has_many :builds has_many :runner_projects, inverse_of: :runner, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, through: :runner_projects + has_many :projects, through: :runner_projects, disable_joins: true has_many :runner_namespaces, inverse_of: :runner, autosave: true has_many :groups, through: :runner_namespaces, disable_joins: true @@ -101,7 +101,7 @@ module Ci } scope :belonging_to_group_or_project_descendants, -> (group_id) { - group_ids = Ci::NamespaceMirror.contains_namespace(group_id).select(:namespace_id) + group_ids = Ci::NamespaceMirror.by_group_and_descendants(group_id).select(:namespace_id) project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id) group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_ids }) @@ -119,30 +119,6 @@ module Ci .where(ci_runner_namespaces: { namespace_id: group_self_and_ancestors_ids }) } - # deprecated - # split this into: belonging_to_group & belonging_to_group_and_ancestors - scope :legacy_belonging_to_group, -> (group_id, include_ancestors: false) { - groups = ::Group.where(id: group_id) - groups = groups.self_and_ancestors if include_ancestors - - joins(:runner_namespaces) - .where(ci_runner_namespaces: { namespace_id: groups }) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') - } - - # deprecated - scope :legacy_belonging_to_group_or_project, -> (group_id, project_id) { - groups = ::Group.where(id: group_id) - - group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups }) - project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) - - union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql - - from("(#{union_sql}) #{table_name}") - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') - } - scope :belonging_to_parent_group_of_project, -> (project_id) { raise ArgumentError, "only 1 project_id allowed for performance reasons" unless project_id.is_a?(Integer) @@ -152,11 +128,23 @@ module Ci } scope :owned_or_instance_wide, -> (project_id) do + project = project_id.respond_to?(:shared_runners) ? project_id : Project.find(project_id) + from_union( [ belonging_to_project(project_id), - belonging_to_parent_group_of_project(project_id), - instance_type + project.group_runners_enabled? ? belonging_to_parent_group_of_project(project_id) : nil, + project.shared_runners + ].compact, + remove_duplicates: false + ) + end + + scope :group_or_instance_wide, -> (group) do + from_union( + [ + belonging_to_group_and_ancestors(group.id), + group.shared_runners ], remove_duplicates: false ) @@ -179,6 +167,8 @@ module Ci scope :order_contacted_at_desc, -> { order(contacted_at: :desc) } scope :order_created_at_asc, -> { order(created_at: :asc) } scope :order_created_at_desc, -> { order(created_at: :desc) } + scope :order_token_expires_at_asc, -> { order(token_expires_at: :asc) } + scope :order_token_expires_at_desc, -> { order(token_expires_at: :desc) } scope :with_tags, -> { preload(:tags) } validate :tag_constraints @@ -210,7 +200,9 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } - validates :maintainer_note, length: { maximum: 255 } + validates :maintenance_note, length: { maximum: 255 } + + alias_attribute :maintenance_note, :maintainer_note # Searches for runners matching the given query. # @@ -247,6 +239,10 @@ module Ci order_contacted_at_desc when 'created_at_asc' order_created_at_asc + when 'token_expires_at_asc' + order_token_expires_at_asc + when 'token_expires_at_desc' + order_token_expires_at_desc else order_created_at_desc end @@ -360,27 +356,12 @@ module Ci runner_projects.any? end - # TODO: remove this method in favor of `matches_build?` once feature flag is removed - # https://gitlab.com/gitlab-org/gitlab/-/issues/323317 - def can_pick?(build) - if Feature.enabled?(:ci_runners_short_circuit_assignable_for, self, default_enabled: :yaml) - matches_build?(build) - else - # Run `matches_build?` checks before, since they are cheaper than - # `assignable_for?`. - # - matches_build?(build) && assignable_for?(build.project_id) - end - end - def match_build_if_online?(build) - active? && online? && can_pick?(build) + active? && online? && matches_build?(build) end def only_for?(project) - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do - projects == [project] - end + !runner_projects.where.not(project_id: project.id).exists? end def short_sha @@ -388,8 +369,6 @@ module Ci end def tag_list - return super unless Feature.enabled?(:ci_preload_runner_tags, default_enabled: :yaml) - if tags.loaded? tags.map(&:name) else @@ -455,6 +434,10 @@ module Ci tick_runner_queue if matches_build?(build) end + def matches_build?(build) + runner_matcher.matches?(build.build_matcher) + end + def uncached_contacted_at read_attribute(:contacted_at) end @@ -465,6 +448,21 @@ module Ci end end + def compute_token_expiration + case runner_type + when 'instance_type' + compute_token_expiration_instance + when 'group_type' + compute_token_expiration_group + when 'project_type' + compute_token_expiration_project + end + end + + def self.token_expiration_enforced? + Feature.enabled?(:enforce_runner_token_expires_at, default_enabled: :yaml) + end + private EXECUTOR_NAME_TO_TYPES = { @@ -484,6 +482,20 @@ module Ci EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze + def compute_token_expiration_instance + return unless expiration_interval = Gitlab::CurrentSettings.runner_token_expiration_interval + + expiration_interval.seconds.from_now + end + + def compute_token_expiration_group + ::Group.where(id: runner_namespaces.map(&:namespace_id)).map(&:effective_runner_token_expiration_interval).compact.min&.from_now + end + + def compute_token_expiration_project + Project.where(id: runner_projects.map(&:project_id)).map(&:effective_runner_token_expiration_interval).compact.min&.from_now + end + def cleanup_runner_queue Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) @@ -510,12 +522,6 @@ module Ci end end - # TODO: remove this method once feature flag ci_runners_short_circuit_assignable_for - # is removed. https://gitlab.com/gitlab-org/gitlab/-/issues/323317 - def assignable_for?(project_id) - self.class.owned_or_instance_wide(project_id).where(id: self.id).any? - end - def no_projects if runner_projects.any? errors.add(:runner, 'cannot have projects assigned') @@ -539,10 +545,6 @@ module Ci errors.add(:runner, 'needs to be assigned to exactly one group') end end - - def matches_build?(build) - runner_matcher.matches?(build.build_matcher) - end end end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 33cd5de3518..07eaca87fad 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.36.0' + VERSION = '0.37.1' self.table_name = 'clusters_applications_runners' diff --git a/app/models/commit.rb b/app/models/commit.rb index f0c5f3c2d12..5293bfcf1ab 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -513,9 +513,7 @@ class Commit # We don't want to do anything for `Commit` model, so this is empty. end - # WIP is deprecated in favor of Draft. Currently both options are supported - # https://gitlab.com/gitlab-org/gitlab/-/issues/227426 - DRAFT_REGEX = /\A\s*#{Regexp.union(Gitlab::Regex.merge_request_wip, Gitlab::Regex.merge_request_draft)}|(fixup!|squash!)\s/.freeze + DRAFT_REGEX = /\A\s*#{Gitlab::Regex.merge_request_draft}|(fixup!|squash!)\s/.freeze def work_in_progress? !!(title =~ DRAFT_REGEX) diff --git a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb index 324e0fb57cb..7cc4bc569d3 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true - module Analytics module CycleAnalytics module StageEventModel @@ -16,12 +15,39 @@ module Analytics scope :authored, ->(user) { where(author_id: user) } scope :with_milestone_id, ->(milestone_id) { where(milestone_id: milestone_id) } scope :end_event_is_not_happened_yet, -> { where(end_event_timestamp: nil) } + scope :order_by_end_event, -> (direction) do + # ORDER BY end_event_timestamp, merge_request_id/issue_id, start_event_timestamp + # start_event_timestamp must be included in the ORDER BY clause for the duration + # calculation to work: SELECT end_event_timestamp - start_event_timestamp + keyset_order( + :end_event_timestamp => { order_expression: arel_order(arel_table[:end_event_timestamp], direction), distinct: false }, + issuable_id_column => { order_expression: arel_order(arel_table[issuable_id_column], direction), distinct: true }, + :start_event_timestamp => { order_expression: arel_order(arel_table[:start_event_timestamp], direction), distinct: false } + ) + end + scope :order_by_duration, -> (direction) do + # ORDER BY EXTRACT('epoch', end_event_timestamp - start_event_timestamp) + duration = Arel::Nodes::Subtraction.new( + arel_table[:end_event_timestamp], + arel_table[:start_event_timestamp] + ) + duration_in_seconds = Arel::Nodes::Extract.new(duration, :epoch) + + keyset_order( + :total_time => { order_expression: arel_order(duration_in_seconds, direction), distinct: false, sql_type: 'double precision' }, + issuable_id_column => { order_expression: arel_order(arel_table[issuable_id_column], direction), distinct: true } + ) + end end def issuable_id attributes[self.class.issuable_id_column.to_s] end + def total_time + read_attribute(:total_time) || (end_event_timestamp - start_event_timestamp).to_f + end + class_methods do def upsert_data(data) upsert_values = data.map do |row| @@ -68,6 +94,18 @@ module Analytics result = connection.execute(query) result.cmd_tuples end + + def keyset_order(column_definition_options) + built_definitions = column_definition_options.map do |attribute_name, column_options| + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(attribute_name: attribute_name, **column_options) + end + + order(Gitlab::Pagination::Keyset::Order.build(built_definitions)) + end + + def arel_order(arel_node, direction) + direction.to_sym == :desc ? arel_node.desc : arel_node.asc + end end end end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index ed3b422251f..88b7bb89b89 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -11,26 +11,9 @@ module Ci # def scoped_variables(environment: expanded_environment_name, dependencies: true) track_duration do - variables = pipeline.variables_builder.scoped_variables(self, environment: environment, dependencies: dependencies) - - next variables if pipeline.use_variables_builder_definitions? - - variables.concat(project.predefined_variables) - variables.concat(pipeline.predefined_variables) - variables.concat(runner.predefined_variables) if runnable? && runner - variables.concat(kubernetes_variables) - variables.concat(deployment_variables(environment: environment)) - variables.concat(yaml_variables) - variables.concat(user_variables) - variables.concat(dependency_variables) if dependencies - variables.concat(secret_instance_variables) - variables.concat(secret_group_variables(environment: environment)) - variables.concat(secret_project_variables(environment: environment)) - variables.concat(trigger_request.user_variables) if trigger_request - variables.concat(pipeline.variables) - variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule - - variables + pipeline + .variables_builder + .scoped_variables(self, environment: environment, dependencies: dependencies) end end @@ -60,29 +43,5 @@ module Ci scoped_variables(environment: nil, dependencies: false) end end - - def user_variables - pipeline.variables_builder.user_variables(user) - end - - def kubernetes_variables - pipeline.variables_builder.kubernetes_variables(self) - end - - def deployment_variables(environment:) - pipeline.variables_builder.deployment_variables(job: self, environment: environment) - end - - def secret_instance_variables - pipeline.variables_builder.secret_instance_variables(ref: git_ref) - end - - def secret_group_variables(environment: expanded_environment_name) - pipeline.variables_builder.secret_group_variables(environment: environment, ref: git_ref) - end - - def secret_project_variables(environment: expanded_environment_name) - pipeline.variables_builder.secret_project_variables(environment: environment, ref: git_ref) - end end end diff --git a/app/models/concerns/ci/has_variable.rb b/app/models/concerns/ci/has_variable.rb index 7309469c77e..3b437fbba16 100644 --- a/app/models/concerns/ci/has_variable.rb +++ b/app/models/concerns/ci/has_variable.rb @@ -31,7 +31,24 @@ module Ci end def to_runner_variable + var_cache_key = to_runner_variable_cache_key + + return uncached_runner_variable unless var_cache_key + + ::Gitlab::SafeRequestStore.fetch(var_cache_key) { uncached_runner_variable } + end + + private + + def uncached_runner_variable { key: key, value: value, public: false, file: file? } end + + def to_runner_variable_cache_key + return unless persisted? + + variable_id = read_attribute(self.class.primary_key) + "#{self.class}#to_runner_variable:#{variable_id}:#{key}" + end end end diff --git a/app/models/concerns/cross_database_modification.rb b/app/models/concerns/cross_database_modification.rb new file mode 100644 index 00000000000..85645e482f6 --- /dev/null +++ b/app/models/concerns/cross_database_modification.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module CrossDatabaseModification + extend ActiveSupport::Concern + + class TransactionStackTrackRecord + DEBUG_STACK = Rails.env.test? && ENV['DEBUG_GITLAB_TRANSACTION_STACK'] + LOG_FILENAME = Rails.root.join("log", "gitlab_transaction_stack.log") + + EXCLUDE_DEBUG_TRACE = %w[ + lib/gitlab/database/query_analyzer + app/models/concerns/cross_database_modification.rb + ].freeze + + def self.logger + @logger ||= Logger.new(LOG_FILENAME, formatter: ->(_, _, _, msg) { Gitlab::Json.dump(msg) + "\n" }) + end + + def self.log_gitlab_transactions_stack(action: nil, example: nil) + return unless DEBUG_STACK + + message = "gitlab_transactions_stack performing #{action}" + message += " in example #{example}" if example + + cleaned_backtrace = Gitlab::BacktraceCleaner.clean_backtrace(caller) + .reject { |line| EXCLUDE_DEBUG_TRACE.any? { |exclusion| line.include?(exclusion) } } + .first(5) + + logger.warn({ + message: message, + action: action, + gitlab_transactions_stack: ::ApplicationRecord.gitlab_transactions_stack, + caller: cleaned_backtrace, + thread: Thread.current.object_id + }) + end + + def initialize(subject, gitlab_schema) + @subject = subject + @gitlab_schema = gitlab_schema + @subject.gitlab_transactions_stack.push(gitlab_schema) + + self.class.log_gitlab_transactions_stack(action: :after_push) + end + + def done! + unless @done + @done = true + + self.class.log_gitlab_transactions_stack(action: :before_pop) + @subject.gitlab_transactions_stack.pop + end + + true + end + + def trigger_transactional_callbacks? + false + end + + def before_committed! + end + + def rolledback!(force_restore_state: false, should_run_callbacks: true) + done! + end + + def committed!(should_run_callbacks: true) + done! + end + end + + included do + private_class_method :gitlab_schema + end + + class_methods do + def gitlab_transactions_stack + Thread.current[:gitlab_transactions_stack] ||= [] + end + + def transaction(**options, &block) + if track_gitlab_schema_in_current_transaction? + super(**options) do + # Hook into current transaction to ensure that once + # the `COMMIT` is executed the `gitlab_transactions_stack` + # will be allowing to execute `after_commit_queue` + record = TransactionStackTrackRecord.new(self, gitlab_schema) + + begin + connection.current_transaction.add_record(record) + + yield + ensure + record.done! + end + end + else + super(**options, &block) + end + end + + def track_gitlab_schema_in_current_transaction? + return false unless Feature::FlipperFeature.table_exists? + + Feature.enabled?(:track_gitlab_schema_in_current_transaction, default_enabled: :yaml) + rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad + false + end + + def gitlab_schema + case self.name + when 'ActiveRecord::Base', 'ApplicationRecord' + :gitlab_main + when 'Ci::ApplicationRecord' + :gitlab_ci + else + Gitlab::Database::GitlabSchema.table_schema(table_name) if table_name + end + end + end +end diff --git a/app/models/concerns/has_environment_scope.rb b/app/models/concerns/has_environment_scope.rb index 9553abe4dd3..c01996c0c4c 100644 --- a/app/models/concerns/has_environment_scope.rb +++ b/app/models/concerns/has_environment_scope.rb @@ -70,6 +70,14 @@ module HasEnvironmentScope relation end + + scope :for_environment, ->(environment) do + if environment + on_environment(environment) + else + where(environment_scope: '*') + end + end end def environment_scope=(new_environment_scope) diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index dcd80201d3f..0138c0ad20f 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -194,6 +194,8 @@ module Issuable end def supports_escalation? + return false unless ::Feature.enabled?(:incident_escalations, project) + incident? end @@ -363,9 +365,10 @@ module Issuable end # Includes table keys in group by clause when sorting - # preventing errors in postgres + # preventing errors in Postgres + # + # Returns an array of Arel columns # - # Returns an array of arel columns def grouping_columns(sort) sort = sort.to_s grouping_columns = [arel_table[:id]] @@ -384,9 +387,10 @@ module Issuable end # Includes all table keys in group by clause when sorting - # preventing errors in postgres when using CTE search optimisation + # preventing errors in Postgres when using CTE search optimization + # + # Returns an array of Arel columns # - # Returns an array of arel columns def issue_grouping_columns(use_cte: false) if use_cte attribute_names.map { |attr| arel_table[attr.to_sym] } @@ -576,7 +580,7 @@ module Issuable ## # Overridden in MergeRequest # - def wipless_title_changed(old_title) + def draftless_title_changed(old_title) old_title != title end end diff --git a/app/models/concerns/mirror_authentication.rb b/app/models/concerns/mirror_authentication.rb index 4dbf4dcec77..14c8be93ce0 100644 --- a/app/models/concerns/mirror_authentication.rb +++ b/app/models/concerns/mirror_authentication.rb @@ -4,11 +4,6 @@ # implements support for persisting the necessary data in a `credentials` # serialized attribute. It also needs an `url` method to be defined module MirrorAuthentication - SSH_PRIVATE_KEY_OPTS = { - type: 'RSA', - bits: 4096 - }.freeze - extend ActiveSupport::Concern included do @@ -84,10 +79,10 @@ module MirrorAuthentication return if ssh_private_key.blank? comment = "git@#{::Gitlab.config.gitlab.host}" - ::SSHKey.new(ssh_private_key, comment: comment).ssh_public_key + SSHData::PrivateKey.parse(ssh_private_key).first.public_key.openssh(comment: comment) end def generate_ssh_private_key! - self.ssh_private_key = ::SSHKey.generate(SSH_PRIVATE_KEY_OPTS).private_key + self.ssh_private_key = SSHData::PrivateKey::RSA.generate(4096).openssl.to_pem end end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index ea4fe5b27dc..c1aac235d33 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -176,7 +176,7 @@ module Noteable Gitlab::Routing.url_helpers.project_noteable_notes_path( project, - target_type: self.class.name.underscore, + target_type: noteable_target_type_name, target_id: id ) end @@ -201,6 +201,10 @@ module Noteable project_email.sub('@', "-#{iid}@") end + def noteable_target_type_name + model_name.singular + end + private # Synthetic system notes don't have discussion IDs because these are generated dynamically diff --git a/app/models/concerns/packages/debian/distribution.rb b/app/models/concerns/packages/debian/distribution.rb index 2d46889ce6a..1520ec0828e 100644 --- a/app/models/concerns/packages/debian/distribution.rb +++ b/app/models/concerns/packages/debian/distribution.rb @@ -97,12 +97,8 @@ module Packages end def package_files - if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) - ::Packages::PackageFile.installable - .for_package_ids(packages.select(:id)) - else - ::Packages::PackageFile.for_package_ids(packages.select(:id)) - end + ::Packages::PackageFile.installable + .for_package_ids(packages.select(:id)) end private diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index aae338e9759..92a88d2f7c8 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -99,6 +99,12 @@ module ResolvableDiscussion update { |notes| notes.unresolve! } end + def clear_memoized_values + self.class.memoized_values.each do |name| + clear_memoization(name) + end + end + private def update @@ -110,8 +116,6 @@ module ResolvableDiscussion # Set the notes array to the updated notes @notes = notes_relation.fresh.to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables - self.class.memoized_values.each do |name| - clear_memoization(name) - end + clear_memoized_values end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index 4d1c1d44af7..e41a0ca28f9 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -15,17 +15,16 @@ module Taskable INCOMPLETE_PATTERN = /(\[\s\])/.freeze ITEM_PATTERN = %r{ ^ - (?:(?:>\s{0,4})*) # optional blockquote characters - (?:\s*(?:[-+*]|(?:\d+\.)))+ # list prefix (one or more) required - task item has to be always in a list - \s+ # whitespace prefix has to be always presented for a list item - (\[\s\]|\[[xX]\]) # checkbox - (\s.+) # followed by whitespace and some text. + (?:(?:>\s{0,4})*) # optional blockquote characters + ((?:\s*(?:[-+*]|(?:\d+\.)))+) # list prefix (one or more) required - task item has to be always in a list + \s+ # whitespace prefix has to be always presented for a list item + (\[\s\]|\[[xX]\]) # checkbox + (\s.+) # followed by whitespace and some text. }x.freeze def self.get_tasks(content) - content.to_s.scan(ITEM_PATTERN).map do |checkbox, label| - # ITEM_PATTERN strips out the hyphen, but Item requires it. Rabble rabble. - TaskList::Item.new("- #{checkbox}", label.strip) + content.to_s.scan(ITEM_PATTERN).map do |prefix, checkbox, label| + TaskList::Item.new("#{prefix} #{checkbox}", label.strip) end end diff --git a/app/models/concerns/timebox.rb b/app/models/concerns/timebox.rb index 3fe9d7f4d71..943ef3fa59f 100644 --- a/app/models/concerns/timebox.rb +++ b/app/models/concerns/timebox.rb @@ -51,7 +51,7 @@ module Timebox validate :dates_within_4_digits cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description + cache_markdown_field :description, issuable_reference_expansion_enabled: true belongs_to :project belongs_to :group @@ -125,17 +125,6 @@ module Timebox fuzzy_search(query, [:title, :description]) end - # Searches for timeboxes with a matching title. - # - # This method uses ILIKE on PostgreSQL - # - # query - The search query as a String - # - # Returns an ActiveRecord::Relation. - def search_title(query) - fuzzy_search(query, [:title]) - end - def filter_by_state(timeboxes, state) case state when 'closed' then timeboxes.closed diff --git a/app/models/concerns/token_authenticatable.rb b/app/models/concerns/token_authenticatable.rb index 34c8630bb90..f44ad8ebe90 100644 --- a/app/models/concerns/token_authenticatable.rb +++ b/app/models/concerns/token_authenticatable.rb @@ -64,6 +64,18 @@ module TokenAuthenticatable mod.define_method("format_#{token_field}") do |token| token end + + mod.define_method("#{token_field}_expires_at") do + strategy.expires_at(self) + end + + mod.define_method("#{token_field}_expired?") do + strategy.expired?(self) + end + + mod.define_method("#{token_field}_with_expiration") do + strategy.token_with_expiration(self) + end end def token_authenticatable_module diff --git a/app/models/concerns/token_authenticatable_strategies/base.rb b/app/models/concerns/token_authenticatable_strategies/base.rb index f72a41f06b1..2cec4ab460e 100644 --- a/app/models/concerns/token_authenticatable_strategies/base.rb +++ b/app/models/concerns/token_authenticatable_strategies/base.rb @@ -7,6 +7,7 @@ module TokenAuthenticatableStrategies def initialize(klass, token_field, options) @klass = klass @token_field = token_field + @expires_at_field = "#{token_field}_expires_at" @options = options end @@ -44,6 +45,25 @@ module TokenAuthenticatableStrategies instance.save! if Gitlab::Database.read_write? end + def expires_at(instance) + instance.read_attribute(@expires_at_field) + end + + def expired?(instance) + return false unless expirable? && token_expiration_enforced? + + exp = expires_at(instance) + !!exp && Time.current > exp + end + + def expirable? + !!@options[:expires_at] + end + + def token_with_expiration(instance) + API::Support::TokenWithExpiration.new(self, instance) + end + def self.fabricate(model, field, options) if options[:digest] && options[:encrypted] raise ArgumentError, _('Incompatible options set!') @@ -64,6 +84,10 @@ module TokenAuthenticatableStrategies new_token = generate_available_token formatted_token = format_token(instance, new_token) set_token(instance, formatted_token) + + if expirable? + instance[@expires_at_field] = @options[:expires_at].to_proc.call(instance) + end end def unique @@ -82,11 +106,21 @@ module TokenAuthenticatableStrategies end def relation(unscoped) - unscoped ? @klass.unscoped : @klass + unscoped ? @klass.unscoped : @klass.where(not_expired) end def token_set?(instance) raise NotImplementedError end + + def token_expiration_enforced? + return true unless @options[:expiration_enforced?] + + @options[:expiration_enforced?].to_proc.call(@klass) + end + + def not_expired + Arel.sql("#{@expires_at_field} IS NULL OR #{@expires_at_field} >= NOW()") if expirable? && token_expiration_enforced? + end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index b03d946fc47..1f123cb0244 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -5,15 +5,24 @@ class ContainerRepository < ApplicationRecord include Gitlab::SQL::Pattern include EachBatch include Sortable + include AfterCommitQueue WAITING_CLEANUP_STATUSES = %i[cleanup_scheduled cleanup_unfinished].freeze REQUIRING_CLEANUP_STATUSES = %i[cleanup_unscheduled cleanup_scheduled].freeze + IDLE_MIGRATION_STATES = %w[default pre_import_done import_done import_aborted import_skipped].freeze + ACTIVE_MIGRATION_STATES = %w[pre_importing importing].freeze + ABORTABLE_MIGRATION_STATES = (ACTIVE_MIGRATION_STATES + %w[pre_import_done default]).freeze + MIGRATION_STATES = (IDLE_MIGRATION_STATES + ACTIVE_MIGRATION_STATES).freeze + + TooManyImportsError = Class.new(StandardError) + NativeImportError = Class.new(StandardError) belongs_to :project validates :name, length: { minimum: 0, allow_nil: false } validates :name, uniqueness: { scope: :project_id } - validates :migration_state, presence: true + validates :migration_state, presence: true, inclusion: { in: MIGRATION_STATES } + validates :migration_aborted_in_state, inclusion: { in: ABORTABLE_MIGRATION_STATES }, allow_nil: true validates :migration_retries_count, presence: true, numericality: { greater_than_or_equal_to: 0 }, @@ -23,7 +32,7 @@ class ContainerRepository < ApplicationRecord enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 } enum migration_skipped_reason: { not_in_plan: 0, too_many_retries: 1, too_many_tags: 2, root_namespace_in_deny_list: 3 } - delegate :client, to: :registry + delegate :client, :gitlab_api_client, to: :registry scope :ordered, -> { order(:name) } scope :with_api_entity_associations, -> { preload(project: [:route, { namespace: :route }]) } @@ -39,7 +48,152 @@ class ContainerRepository < ApplicationRecord scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :waiting_for_cleanup, -> { where(expiration_policy_cleanup_status: WAITING_CLEANUP_STATUSES) } scope :expiration_policy_started_at_nil_or_before, ->(timestamp) { where('expiration_policy_started_at < ? OR expiration_policy_started_at IS NULL', timestamp) } + scope :with_migration_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_import_started_at, '01-01-1970') < ?", timestamp) } + scope :with_migration_pre_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_started_at, '01-01-1970') < ?", timestamp) } + scope :with_migration_pre_import_done_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_done_at, '01-01-1970') < ?", timestamp) } scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.where('expiration_policy_started_at < ?', threshold) } + scope :import_in_process, -> { where(migration_state: %w[pre_importing pre_import_done importing]) } + + scope :recently_done_migration_step, -> do + where(migration_state: %w[import_done pre_import_done import_aborted]) + .order(Arel.sql('GREATEST(migration_pre_import_done_at, migration_import_done_at, migration_aborted_at) DESC')) + end + + scope :ready_for_import, -> do + # There is no yaml file for the container_registry_phase_2_deny_list + # feature flag since it is only accessed in this query. + # https://gitlab.com/gitlab-org/gitlab/-/issues/350543 tracks the rollout and + # removal of this feature flag. + joins(:project).where( + migration_state: [:default], + created_at: ...ContainerRegistry::Migration.created_before + ).with_target_import_tier + .where( + "NOT EXISTS ( + SELECT 1 + FROM feature_gates + WHERE feature_gates.feature_key = 'container_registry_phase_2_deny_list' + AND feature_gates.key = 'actors' + AND feature_gates.value = concat('Group:', projects.namespace_id) + )" + ) + end + + state_machine :migration_state, initial: :default, use_transactions: false do + state :pre_importing do + validates :migration_pre_import_started_at, presence: true + validates :migration_pre_import_done_at, presence: false + end + + state :pre_import_done do + validates :migration_pre_import_done_at, presence: true + end + + state :importing do + validates :migration_import_started_at, presence: true + validates :migration_import_done_at, presence: false + end + + state :import_done + + state :import_skipped do + validates :migration_skipped_reason, + :migration_skipped_at, + presence: true + end + + state :import_aborted do + validates :migration_aborted_at, presence: true + validates :migration_retries_count, presence: true, numericality: { greater_than_or_equal_to: 1 } + end + + event :start_pre_import do + transition default: :pre_importing + end + + event :finish_pre_import do + transition %i[pre_importing import_aborted] => :pre_import_done + end + + event :start_import do + transition pre_import_done: :importing + end + + event :finish_import do + transition %i[importing import_aborted] => :import_done + end + + event :already_migrated do + transition default: :import_done + end + + event :abort_import do + transition ABORTABLE_MIGRATION_STATES.map(&:to_sym) => :import_aborted + end + + event :skip_import do + transition ABORTABLE_MIGRATION_STATES.map(&:to_sym) => :import_skipped + end + + event :retry_pre_import do + transition import_aborted: :pre_importing + end + + event :retry_import do + transition import_aborted: :importing + end + + before_transition any => :pre_importing do |container_repository| + container_repository.migration_pre_import_started_at = Time.zone.now + container_repository.migration_pre_import_done_at = nil + end + + after_transition any => :pre_importing do |container_repository| + container_repository.try_import do + container_repository.migration_pre_import + end + end + + before_transition %i[pre_importing import_aborted] => :pre_import_done do |container_repository| + container_repository.migration_pre_import_done_at = Time.zone.now + end + + before_transition any => :importing do |container_repository| + container_repository.migration_import_started_at = Time.zone.now + container_repository.migration_import_done_at = nil + end + + after_transition any => :importing do |container_repository| + container_repository.try_import do + container_repository.migration_import + end + end + + before_transition %i[importing import_aborted] => :import_done do |container_repository| + container_repository.migration_import_done_at = Time.zone.now + end + + before_transition any => :import_aborted do |container_repository| + container_repository.migration_aborted_in_state = container_repository.migration_state + container_repository.migration_aborted_at = Time.zone.now + container_repository.migration_retries_count += 1 + end + + before_transition import_aborted: any do |container_repository| + container_repository.migration_aborted_at = nil + container_repository.migration_aborted_in_state = nil + end + + before_transition any => :import_skipped do |container_repository| + container_repository.migration_skipped_at = Time.zone.now + end + + before_transition any => %i[import_done import_aborted] do |container_repository| + container_repository.run_after_commit do + ::ContainerRegistry::Migration::EnqueuerWorker.perform_async + end + end + end def self.exists_by_path?(path) where( @@ -64,6 +218,114 @@ class ContainerRepository < ApplicationRecord with_enabled_policy.cleanup_unfinished end + def self.with_stale_migration(before_timestamp) + stale_pre_importing = with_migration_states(:pre_importing) + .with_migration_pre_import_started_at_nil_or_before(before_timestamp) + stale_pre_import_done = with_migration_states(:pre_import_done) + .with_migration_pre_import_done_at_nil_or_before(before_timestamp) + stale_importing = with_migration_states(:importing) + .with_migration_import_started_at_nil_or_before(before_timestamp) + + union = ::Gitlab::SQL::Union.new([ + stale_pre_importing, + stale_pre_import_done, + stale_importing + ]) + from("(#{union.to_sql}) #{ContainerRepository.table_name}") + end + + def self.with_target_import_tier + # overridden in ee + # + # Repositories are being migrated by tier on Saas, so we need to + # filter by plan/subscription which is not available in FOSS + all + end + + def skip_import(reason:) + self.migration_skipped_reason = reason + + super + end + + def start_pre_import + return false unless ContainerRegistry::Migration.enabled? + + super + end + + def retry_pre_import + return false unless ContainerRegistry::Migration.enabled? + + super + end + + def retry_import + return false unless ContainerRegistry::Migration.enabled? + + super + end + + def finish_pre_import_and_start_import + # nothing to do between those two transitions for now. + finish_pre_import && start_import + end + + def retry_aborted_migration + return unless migration_state == 'import_aborted' + + case external_import_status + when 'native' + raise NativeImportError + when 'import_in_progress' + nil + when 'import_complete' + finish_import + when 'import_failed' + retry_import + when 'pre_import_in_progress' + nil + when 'pre_import_complete' + finish_pre_import_and_start_import + when 'pre_import_failed' + retry_pre_import + else + # If the import_status request fails, use the timestamp to guess current state + migration_pre_import_done_at ? retry_import : retry_pre_import + end + end + + def try_import + raise ArgumentError, 'block not given' unless block_given? + + try_count = 0 + begin + try_count += 1 + return true if yield == :ok + + abort_import + false + rescue TooManyImportsError + if try_count <= ::ContainerRegistry::Migration.start_max_retries + sleep 0.1 * try_count + retry + else + abort_import + false + end + end + end + + def last_import_step_done_at + [migration_pre_import_done_at, migration_import_done_at, migration_aborted_at].compact.max + end + + def external_import_status + strong_memoize(:import_status) do + gitlab_api_client.import_status(self.path) + end + end + # rubocop: disable CodeReuse/ServiceClass def registry @registry ||= begin @@ -146,6 +408,36 @@ class ContainerRepository < ApplicationRecord update!(expiration_policy_started_at: Time.zone.now) end + def migration_in_active_state? + migration_state.in?(ACTIVE_MIGRATION_STATES) + end + + def migration_importing? + migration_state == 'importing' + end + + def migration_pre_importing? + migration_state == 'pre_importing' + end + + def migration_pre_import + return :error unless gitlab_api_client.supports_gitlab_api? + + response = gitlab_api_client.pre_import_repository(self.path) + raise TooManyImportsError if response == :too_many_imports + + response + end + + def migration_import + return :error unless gitlab_api_client.supports_gitlab_api? + + response = gitlab_api_client.import_repository(self.path) + raise TooManyImportsError if response == :too_many_imports + + response + end + def self.build_from_path(path) self.new(project: path.repository_project, name: path.repository_name) @@ -169,6 +461,11 @@ class ContainerRepository < ApplicationRecord self.find_by!(project: path.repository_project, name: path.repository_name) end + + def self.find_by_path(path) + self.find_by(project: path.repository_project, + name: path.repository_name) + end end ContainerRepository.prepend_mod_with('ContainerRepository') diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index 168f1c48a6c..a981351f4a0 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -26,6 +26,18 @@ class CustomerRelations::Contact < ApplicationRecord validate :validate_email_format validate :unique_email_for_group_hierarchy + def self.reference_prefix + '[contact:' + end + + def self.reference_prefix_quoted + '["contact:' + end + + def self.reference_postfix + ']' + end + def self.find_ids_by_emails(group, emails) raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK @@ -33,6 +45,12 @@ class CustomerRelations::Contact < ApplicationRecord .pluck(:id) end + def self.exists_for_group?(group) + return false unless group + + exists?(group_id: group.self_and_ancestor_ids) + end + private def validate_email_format diff --git a/app/models/customer_relations/issue_contact.rb b/app/models/customer_relations/issue_contact.rb index 89dac6bad22..3e9d1e97c8c 100644 --- a/app/models/customer_relations/issue_contact.rb +++ b/app/models/customer_relations/issue_contact.rb @@ -16,6 +16,12 @@ class CustomerRelations::IssueContact < ApplicationRecord .pluck(:contact_id) end + def self.delete_for_project(project_id) + joins(:issue) + .where(issues: { project_id: project_id }) + .delete_all + end + private def contact_belongs_to_issue_group_or_ancestor diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb index e4018ab4770..f7b08f1d077 100644 --- a/app/models/dependency_proxy/blob.rb +++ b/app/models/dependency_proxy/blob.rb @@ -14,6 +14,8 @@ class DependencyProxy::Blob < ApplicationRecord validates :file, presence: true validates :file_name, presence: true + scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) } + mount_file_store_uploader DependencyProxy::FileUploader def self.total_size @@ -24,3 +26,5 @@ class DependencyProxy::Blob < ApplicationRecord find_or_initialize_by(file_name: file_name) end end + +DependencyProxy::Blob.prepend_mod_with('DependencyProxy::Blob') diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index fe887c99e81..c2587ffac9d 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -17,6 +17,7 @@ class DependencyProxy::Manifest < ApplicationRecord validates :digest, presence: true scope :order_id_desc, -> { reorder(id: :desc) } + scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) } mount_file_store_uploader DependencyProxy::FileUploader @@ -24,3 +25,5 @@ class DependencyProxy::Manifest < ApplicationRecord find_by(file_name: file_name) || find_by(digest: digest) end end + +DependencyProxy::Manifest.prepend_mod_with('DependencyProxy::Manifest') diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 2f04d99f9f6..46409465209 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -255,10 +255,10 @@ class Deployment < ApplicationRecord end end - def includes_commit?(commit) - return false unless commit + def includes_commit?(ancestor_sha) + return false unless sha - project.repository.ancestor?(commit.id, sha) + project.repository.ancestor?(ancestor_sha, sha) end def update_merge_request_metrics! @@ -294,10 +294,6 @@ class Deployment < ApplicationRecord @stop_action ||= manual_actions.find { |action| action.name == self.on_stop } end - def finished_at - read_attribute(:finished_at) || legacy_finished_at - end - def deployed_at return unless success? @@ -405,10 +401,6 @@ class Deployment < ApplicationRecord raise ArgumentError, "The status #{status.inspect} is invalid" end end - - def legacy_finished_at - self.created_at if success? && !read_attribute(:finished_at) - end end Deployment.prepend_mod_with('Deployment') diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 203e14f1227..8a167034629 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -25,7 +25,7 @@ class Discussion :to_ability_name, :editable?, :resolved_by_id, - :system_note_with_references_visible_for?, + :system_note_visible_for?, :resource_parent, :save, to: :first_note diff --git a/app/models/draft_note.rb b/app/models/draft_note.rb index febede9beba..9f7977fce68 100644 --- a/app/models/draft_note.rb +++ b/app/models/draft_note.rb @@ -25,6 +25,7 @@ class DraftNote < ApplicationRecord validates :merge_request_id, presence: true validates :author_id, presence: true, uniqueness: { scope: [:merge_request_id, :discussion_id] }, if: :discussion_id? validates :discussion_id, allow_nil: true, format: { with: /\A\h{40}\z/ } + validates :line_code, length: { maximum: 255 }, allow_nil: true scope :authored_by, ->(u) { where(author_id: u.id) } @@ -89,7 +90,11 @@ class DraftNote < ApplicationRecord end def line_code - @line_code ||= diff_file&.line_code_for_position(original_position) + super.presence || find_line_code + end + + def find_line_code + write_attribute(:line_code, diff_file&.line_code_for_position(original_position)) end def publish_params diff --git a/app/models/environment.rb b/app/models/environment.rb index a830c04f291..51a9024721b 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -235,10 +235,10 @@ class Environment < ApplicationRecord self.environment_type = names.many? ? names.first : nil end - def includes_commit?(commit) + def includes_commit?(sha) return false unless last_deployment - last_deployment.includes_commit?(commit) + last_deployment.includes_commit?(sha) end def last_deployed_at diff --git a/app/models/event.rb b/app/models/event.rb index 409bc66c66c..a8cf2e2dfb0 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -354,7 +354,7 @@ class Event < ApplicationRecord # hence we add the extra WHERE clause for last_activity_at. Project.unscoped.where(id: project_id) .where('last_activity_at <= ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago) - .update_all(last_activity_at: created_at) + .touch_all(:last_activity_at, time: created_at) # rubocop: disable Rails/SkipsModelValidations end def authored_by?(user) @@ -430,7 +430,7 @@ class Event < ApplicationRecord def set_last_repository_updated_at Project.unscoped.where(id: project_id) .where("last_repository_updated_at < ? OR last_repository_updated_at IS NULL", REPOSITORY_UPDATED_AT_INTERVAL.ago) - .update_all(last_repository_updated_at: created_at) + .touch_all(:last_repository_updated_at, time: created_at) # rubocop: disable Rails/SkipsModelValidations end def design_action_names diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 3320c13e87b..7e538238cbd 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -50,8 +50,9 @@ class WebHook < ApplicationRecord end # rubocop: disable CodeReuse/ServiceClass - def execute(data, hook_name) - WebHookService.new(self, data, hook_name).execute if executable? + def execute(data, hook_name, force: false) + # hook.executable? is checked in WebHookService#execute + WebHookService.new(self, data, hook_name, force: force).execute end # rubocop: enable CodeReuse/ServiceClass diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index dc025e576ed..2016024b2f4 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -118,7 +118,12 @@ class InstanceConfiguration group_export_download: application_setting_limit_per_minute(:group_download_export_limit), group_import: application_setting_limit_per_minute(:group_import_limit), raw_blob: application_setting_limit_per_minute(:raw_blob_request_limit), - user_email_lookup: application_setting_limit_per_minute(:user_email_lookup_limit) + user_email_lookup: application_setting_limit_per_minute(:user_email_lookup_limit), + users_get_by_id: { + enabled: application_settings[:users_get_by_id_limit] > 0, + requests_per_period: application_settings[:users_get_by_id_limit], + period_in_seconds: 10.minutes + } } end @@ -147,7 +152,7 @@ class InstanceConfiguration end def ssh_algorithm_sha256(ssh_file_content) - Gitlab::SSHPublicKey.new(ssh_file_content).fingerprint('SHA256') + Gitlab::SSHPublicKey.new(ssh_file_content).fingerprint_sha256 end def application_settings diff --git a/app/models/integration.rb b/app/models/integration.rb index 89b34932e20..e9cd90649ba 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -392,8 +392,7 @@ class Integration < ApplicationRecord end def api_field_names - fields.map { |field| field[:name] } - .reject { |field_name| field_name =~ /(password|token|key|title|description)/ } + fields.pluck(:name).grep_v(/password|token|key|title|description/) end def global_fields diff --git a/app/models/integrations/chat_message/base_message.rb b/app/models/integrations/chat_message/base_message.rb index ab213f4b43f..554b422c0fa 100644 --- a/app/models/integrations/chat_message/base_message.rb +++ b/app/models/integrations/chat_message/base_message.rb @@ -47,16 +47,21 @@ module Integrations format(message) end + # NOTE: Make sure to call `#strip_markup` on any untrusted user input that's added to the + # `title`, `subtitle`, `text`, `fallback`, or `author_name` fields. def attachments raise NotImplementedError end + # NOTE: Make sure to call `#strip_markup` on any untrusted user input that's added to the + # `title`, `subtitle`, `text`, `fallback`, or `author_name` fields. def activity raise NotImplementedError end private + # NOTE: Make sure to call `#strip_markup` on any untrusted user input that's added to the string. def message raise NotImplementedError end diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index b86f0aaa7ef..bb0fb6b9079 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -13,7 +13,11 @@ module Integrations pipeline job ].freeze - prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env + TAG_KEY_VALUE_RE = %r{\A [\w-]+ : .*\S.* \z}x.freeze + + prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env, :datadog_tags + + before_validation :strip_properties with_options if: :activated? do validates :api_key, presence: true, format: { with: /\A\w+\z/ } @@ -21,6 +25,7 @@ module Integrations validates :api_url, public_url: { allow_blank: true } validates :datadog_site, presence: true, unless: -> (obj) { obj.api_url.present? } validates :api_url, presence: true, unless: -> (obj) { obj.datadog_site.present? } + validate :datadog_tags_are_valid end def initialize_properties @@ -140,6 +145,20 @@ module Integrations linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, linkClose: '</a>'.html_safe } + }, + { + type: 'textarea', + name: 'datadog_tags', + title: s_('DatadogIntegration|Tags'), + placeholder: "tag:value\nanother_tag:value", + help: ERB::Util.html_escape( + s_('DatadogIntegration|Custom tags in Datadog. Enter one tag per line in the %{codeOpen}key:value%{codeClose} format. %{linkOpen}How do I use tags?%{linkClose}') + ) % { + codeOpen: '<code>'.html_safe, + codeClose: '</code>'.html_safe, + linkOpen: '<a href="https://docs.datadoghq.com/getting_started/tagging/#using-tags" target="_blank" rel="noopener noreferrer">'.html_safe, + linkClose: '</a>'.html_safe + } } ] @@ -153,7 +172,8 @@ module Integrations query = { "dd-api-key" => api_key, service: datadog_service.presence, - env: datadog_env.presence + env: datadog_env.presence, + tags: datadog_tags_query_param.presence }.compact url.query = query.to_query url.to_s @@ -193,5 +213,35 @@ module Integrations data end + + def strip_properties + datadog_service.strip! if datadog_service && !datadog_service.frozen? + datadog_env.strip! if datadog_env && !datadog_env.frozen? + datadog_tags.strip! if datadog_tags && !datadog_tags.frozen? + end + + def datadog_tags_are_valid + return unless datadog_tags + + unless datadog_tags.split("\n").select(&:present?).all? { _1 =~ TAG_KEY_VALUE_RE } + errors.add(:datadog_tags, s_("DatadogIntegration|have an invalid format")) + end + end + + def datadog_tags_query_param + return unless datadog_tags + + datadog_tags.split("\n").filter_map do |tag| + tag.strip! + + next if tag.blank? + + if tag.include?(',') + "\"#{tag}\"" + else + tag + end + end.join(',') + end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 4f2773f4147..68ea6cb3abc 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -29,8 +29,10 @@ class Issue < ApplicationRecord DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze - AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze + AnyDueDate = DueDateStruct.new('Any Due Date', 'any').freeze Overdue = DueDateStruct.new('Overdue', 'overdue').freeze + DueToday = DueDateStruct.new('Due Today', 'today').freeze + DueTomorrow = DueDateStruct.new('Due Tomorrow', 'tomorrow').freeze DueThisWeek = DueDateStruct.new('Due This Week', 'week').freeze DueThisMonth = DueDateStruct.new('Due This Month', 'month').freeze DueNextMonthAndPreviousTwoWeeks = DueDateStruct.new('Due Next Month And Previous Two Weeks', 'next_month_and_previous_two_weeks').freeze @@ -107,7 +109,9 @@ class Issue < ApplicationRecord scope :without_due_date, -> { where(due_date: nil) } scope :due_before, ->(date) { where('issues.due_date < ?', date) } scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) } + scope :due_today, -> { where(due_date: Date.current) } scope :due_tomorrow, -> { where(due_date: Date.tomorrow) } + scope :not_authored_by, ->(user) { where.not(author_id: user) } scope :order_due_date_asc, -> { reorder(::Gitlab::Database.nulls_last_order('due_date', 'ASC')) } @@ -121,7 +125,6 @@ class Issue < ApplicationRecord scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } scope :preload_awardable, -> { preload(:award_emoji) } - scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) } scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } scope :with_self_managed_prometheus_alert_events, -> { joins(:issues_self_managed_prometheus_alert_events) } @@ -140,7 +143,7 @@ class Issue < ApplicationRecord scope :confidential_only, -> { where(confidential: true) } scope :without_hidden, -> { - if Feature.enabled?(:ban_user_feature_flag) + if Feature.enabled?(:ban_user_feature_flag, default_enabled: :yaml) where('NOT EXISTS (?)', Users::BannedUser.select(1).where('issues.author_id = banned_users.user_id')) else all @@ -584,7 +587,7 @@ class Issue < ApplicationRecord def readable_by?(user) if user.can_read_all_resources? true - elsif project.owner == user + elsif project.personal? && project.team.owner?(user) true elsif confidential? && !assignee_or_author?(user) project.team.member?(user, Gitlab::Access::REPORTER) diff --git a/app/models/key.rb b/app/models/key.rb index 933c939fdf5..4a4e792c074 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -130,7 +130,7 @@ class Key < ApplicationRecord return unless public_key.valid? self.fingerprint_md5 = public_key.fingerprint - self.fingerprint_sha256 = public_key.fingerprint("SHA256").gsub("SHA256:", "") + self.fingerprint_sha256 = public_key.fingerprint_sha256.gsub("SHA256:", "") end def key_meets_restrictions diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index db82d5bbf29..ebda5872f1c 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -46,17 +46,39 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel .to_a end - def self.mark_records_processed(all_records) - # Run a query for each partition to optimize the row lookup by primary key (partition, id) + def self.mark_records_processed(records) + update_by_partition(records) do |partitioned_scope| + partitioned_scope.update_all(status: :processed) + end + end + + def self.reschedule(records, consume_after) + update_by_partition(records) do |partitioned_scope| + partitioned_scope.update_all(consume_after: consume_after, cleanup_attempts: 0) + end + end + + def self.increment_attempts(records) + update_by_partition(records) do |partitioned_scope| + # Naive incrementing of the cleanup_attempts is good enough for us. + partitioned_scope.update_all('cleanup_attempts = cleanup_attempts + 1') + end + end + + def self.update_by_partition(records) update_count = 0 - all_records.group_by(&:partition_number).each do |partition, records_within_partition| - update_count += status_pending + # Run a query for each partition to optimize the row lookup by primary key (partition, id) + records.group_by(&:partition_number).each do |partition, records_within_partition| + partitioned_scope = status_pending .for_partition(partition) .where(id: records_within_partition.pluck(:id)) - .update_all(status: :processed) + + update_count += yield(partitioned_scope) end update_count end + + private_class_method :update_by_partition end diff --git a/app/models/member.rb b/app/models/member.rb index 6c0503dca3f..528c6855d9c 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -108,6 +108,8 @@ class Member < ApplicationRecord .reorder(nil) end + scope :active_state, -> { where(state: STATE_ACTIVE) } + scope :connected_to_user, -> { where.not(user_id: nil) } # This scope is exclusively used to get the members @@ -115,6 +117,7 @@ class Member < ApplicationRecord # to projects/groups. scope :authorizable, -> do connected_to_user + .active_state .non_request .non_minimal_access end @@ -128,7 +131,8 @@ class Member < ApplicationRecord end scope :without_invites_and_requests, -> do - non_request + active_state + .non_request .non_invite .non_minimal_access end @@ -180,6 +184,7 @@ class Member < ApplicationRecord scope :on_project_and_ancestors, ->(project) { where(source: [project] + project.ancestors) } + before_validation :set_member_namespace_id, on: :create before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? && !member.invite_accepted_at? } after_create :send_invite, if: :invite?, unless: :importing? @@ -203,7 +208,7 @@ class Member < ApplicationRecord class << self def search(query) - joins(:user).merge(User.search(query)) + joins(:user).merge(User.search(query, use_minimum_char_limit: false)) end def search_invite_email(query) @@ -380,6 +385,12 @@ class Member < ApplicationRecord private + # TODO: https://gitlab.com/groups/gitlab-org/-/epics/7054 + # temporary until we can we properly remove the source columns + def set_member_namespace_id + self.member_namespace_id = self.source_id + end + def access_level_inclusion return if access_level.in?(Gitlab::Access.all_values) diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 6fc665cb87a..3a449055bc1 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -118,6 +118,13 @@ class ProjectMember < Member # rubocop:enable CodeReuse/ServiceClass end + # TODO: https://gitlab.com/groups/gitlab-org/-/epics/7054 + # temporary until we can we properly remove the source columns + override :set_member_namespace_id + def set_member_namespace_id + self.member_namespace_id = project&.project_namespace_id + end + def send_invite run_after_commit_or_now { notification_service.invite_project_member(self, @raw_invite_token) } diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index cf36e72a565..29540cbde2f 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -406,6 +406,17 @@ class MergeRequest < ApplicationRecord ) end + scope :attention, ->(user) do + # rubocop: disable Gitlab/Union + union = Gitlab::SQL::Union.new([ + MergeRequestReviewer.select(:merge_request_id).where(user_id: user.id, state: MergeRequestReviewer.states[:attention_requested]), + MergeRequestAssignee.select(:merge_request_id).where(user_id: user.id, state: MergeRequestAssignee.states[:attention_requested]) + ]) + # rubocop: enable Gitlab/Union + + with(Gitlab::SQL::CTE.new(:reviewers_and_assignees, union).to_arel).where('merge_requests.id in (select merge_request_id from reviewers_and_assignees)') + end + def self.total_time_to_merge join_metrics .merge(MergeRequest::Metrics.with_valid_time_to_merge) @@ -471,6 +482,12 @@ class MergeRequest < ApplicationRecord rebase_jid.present? && Gitlab::SidekiqStatus.running?(rebase_jid) end + def permits_force_push? + return true unless ProtectedBranch.protected?(source_project, source_branch) + + ProtectedBranch.allow_force_push?(source_project, source_branch) + end + # Use this method whenever you need to make sure the head_pipeline is synced with the # branch head commit, for example checking if a merge request can be merged. # For more information check: https://gitlab.com/gitlab-org/gitlab-foss/issues/40004 @@ -561,20 +578,24 @@ class MergeRequest < ApplicationRecord end end - # WIP is deprecated in favor of Draft. Currently both options are supported - # https://gitlab.com/gitlab-org/gitlab/-/issues/227426 - DRAFT_REGEX = /\A*#{Regexp.union(Gitlab::Regex.merge_request_wip, Gitlab::Regex.merge_request_draft)}+\s*/i.freeze + DRAFT_REGEX = /\A*#{Gitlab::Regex.merge_request_draft}+\s*/i.freeze - def self.work_in_progress?(title) + def self.draft?(title) !!(title =~ DRAFT_REGEX) end - def self.wipless_title(title) + def self.draftless_title(title) title.sub(DRAFT_REGEX, "") end - def self.wip_title(title) - work_in_progress?(title) ? title : "Draft: #{title}" + def self.draft_title(title) + draft?(title) ? title : "Draft: #{title}" + end + + class << self + alias_method :work_in_progress?, :draft? + alias_method :wipless_title, :draftless_title + alias_method :wip_title, :draft_title end def self.participant_includes @@ -587,9 +608,10 @@ class MergeRequest < ApplicationRecord # Verifies if title has changed not taking into account Draft prefix # for merge requests. - def wipless_title_changed(old_title) - self.class.wipless_title(old_title) != self.wipless_title + def draftless_title_changed(old_title) + self.class.draftless_title(old_title) != self.draftless_title end + alias_method :wipless_title_changed, :draftless_title_changed def hook_attrs Gitlab::HookData::MergeRequestBuilder.new(self).build @@ -1088,18 +1110,20 @@ class MergeRequest < ApplicationRecord @closed_event ||= target_project.events.where(target_id: self.id, target_type: "MergeRequest", action: :closed).last end - def work_in_progress? - self.class.work_in_progress?(title) + def draft? + self.class.draft?(title) end - alias_method :draft?, :work_in_progress? + alias_method :work_in_progress?, :draft? - def wipless_title - self.class.wipless_title(self.title) + def draftless_title + self.class.draftless_title(self.title) end + alias_method :wipless_title, :draftless_title - def wip_title - self.class.wip_title(self.title) + def draft_title + self.class.draft_title(self.title) end + alias_method :wip_title, :draft_title def mergeable?(skip_ci_check: false, skip_discussions_check: false) return false unless mergeable_state?(skip_ci_check: skip_ci_check, @@ -1754,6 +1778,8 @@ class MergeRequest < ApplicationRecord paths = active_diff_discussions.flat_map { |n| n.diff_file.paths }.uniq + active_discussions_resolved = active_diff_discussions.all?(&:resolved?) + service = Discussions::UpdateDiffPositionService.new( self.project, current_user, @@ -1764,9 +1790,15 @@ class MergeRequest < ApplicationRecord active_diff_discussions.each do |discussion| service.execute(discussion) + discussion.clear_memoized_values end - if project.resolve_outdated_diff_discussions? + # If they were all already resolved, this method will have already been called. + # If they all don't get resolved, we don't need to call the method + # If they go from unresolved -> resolved, then we call the method + if !active_discussions_resolved && + active_diff_discussions.all?(&:resolved?) && + project.resolve_outdated_diff_discussions? MergeRequests::ResolvedDiscussionNotificationService .new(project: project, current_user: current_user) .execute(self) diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 868bee9961b..2c95cc2672c 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -52,6 +52,17 @@ class Milestone < ApplicationRecord state :active end + # Searches for timeboxes with a matching title. + # + # This method uses ILIKE on PostgreSQL + # + # query - The search query as a String + # + # Returns an ActiveRecord::Relation. + def self.search_title(query) + fuzzy_search(query, [:title]) + end + def self.min_chars_for_partial_matching 2 end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 0dc20e0016c..5c55f4d3def 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -43,6 +43,7 @@ class Namespace < ApplicationRecord has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true + has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' @@ -492,6 +493,10 @@ class Namespace < ApplicationRecord end end + def shared_runners + @shared_runners ||= shared_runners_enabled ? Ci::Runner.instance_type : Ci::Runner.none + end + def root? !has_parent? end @@ -508,6 +513,12 @@ class Namespace < ApplicationRecord Feature.enabled?(:create_project_namespace_on_project_create, self, default_enabled: :yaml) end + def storage_enforcement_date + # should return something like Date.new(2022, 02, 03) + # TBD: https://gitlab.com/gitlab-org/gitlab/-/issues/350632 + nil + end + private def expire_child_caches diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index 99e32537595..ee04ec39b1e 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -27,10 +27,17 @@ class Namespace::RootStorageStatistics < ApplicationRecord update!(merged_attributes) end + def self.namespace_statistics_attributes + %w(storage_size dependency_proxy_size) + end + private def merged_attributes - attributes_from_project_statistics.merge!(attributes_from_personal_snippets) { |key, v1, v2| v1 + v2 } + attributes_from_project_statistics.merge!( + attributes_from_personal_snippets, + attributes_from_namespace_statistics + ) { |key, v1, v2| v1 + v2 } end def attributes_from_project_statistics @@ -68,6 +75,27 @@ class Namespace::RootStorageStatistics < ApplicationRecord .where(author: namespace.owner_id) .select("COALESCE(SUM(s.repository_size), 0) AS #{SNIPPETS_SIZE_STAT_NAME}") end + + def from_namespace_statistics + namespace + .self_and_descendants + .joins("INNER JOIN namespace_statistics ns ON ns.namespace_id = namespaces.id") + .select( + 'COALESCE(SUM(ns.storage_size), 0) AS storage_size', + 'COALESCE(SUM(ns.dependency_proxy_size), 0) AS dependency_proxy_size' + ) + end + + def attributes_from_namespace_statistics + # At the moment, only groups can have some storage data because of dependency proxy assets. + # Therefore, if the namespace is not a group one, there is no need to perform + # the query. If this changes in the future and we add some sort of resource to + # users that it's store in NamespaceStatistics, we will need to remove this + # guard clause. + return {} unless namespace.group_namespace? + + from_namespace_statistics.take.slice(*self.class.namespace_statistics_attributes) + end end Namespace::RootStorageStatistics.prepend_mod_with('Namespace::RootStorageStatistics') diff --git a/app/models/namespace_statistics.rb b/app/models/namespace_statistics.rb new file mode 100644 index 00000000000..04ca05d85ff --- /dev/null +++ b/app/models/namespace_statistics.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +class NamespaceStatistics < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass + include AfterCommitQueue + + belongs_to :namespace + + validates :namespace, presence: true + + scope :for_namespaces, -> (namespaces) { where(namespace: namespaces) } + + before_save :update_storage_size + after_save :update_root_storage_statistics, if: :saved_change_to_storage_size? + after_destroy :update_root_storage_statistics + + delegate :group_namespace?, to: :namespace + + def refresh!(only: []) + return if Gitlab::Database.read_only? + return unless group_namespace? + + self.class.columns_to_refresh.each do |column| + if only.empty? || only.include?(column) + public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend + end + end + + save! + end + + def update_storage_size + # This prevents failures with older database schemas, such as those + # in migration specs. + return unless self.class.database.cached_column_exists?(:dependency_proxy_size) + + self.storage_size = dependency_proxy_size + end + + def update_dependency_proxy_size + return unless group_namespace? + + self.dependency_proxy_size = namespace.dependency_proxy_manifests.sum(:size) + namespace.dependency_proxy_blobs.sum(:size) + end + + def self.columns_to_refresh + [:dependency_proxy_size] + end + + private + + def update_root_storage_statistics + return unless group_namespace? + + run_after_commit do + Namespaces::ScheduleAggregationWorker.perform_async(namespace.id) + end + end +end + +NamespaceStatistics.prepend_mod_with('NamespaceStatistics') diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb index 8534d8afb8c..fbe047f2c5a 100644 --- a/app/models/namespaces/sync_event.rb +++ b/app/models/namespaces/sync_event.rb @@ -13,4 +13,8 @@ class Namespaces::SyncEvent < ApplicationRecord def self.enqueue_worker ::Namespaces::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker end + + def self.upper_bound_count + select('COALESCE(MAX(id) - MIN(id) + 1, 0) AS upper_bound_count').to_a.first.upper_bound_count + end end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 757a0e40eb3..99a5b8cb063 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -43,14 +43,23 @@ module Namespaces included do before_update :lock_both_roots, if: -> { sync_traversal_ids? && parent_id_changed? } - after_create :sync_traversal_ids, if: -> { sync_traversal_ids? } after_update :sync_traversal_ids, if: -> { sync_traversal_ids? && saved_change_to_parent_id? } + # sync traversal_ids on namespace create, which can happen quite early within a transaction, thus keeping the lock on root namespace record + # for a relatively long time, e.g. creating the project namespace when a project is being created. + after_create :sync_traversal_ids, if: -> { sync_traversal_ids? && !sync_traversal_ids_before_commit? } + # This uses rails internal before_commit API to sync traversal_ids on namespace create, right before transaction is committed. + # This helps reduce the time during which the root namespace record is locked to ensure updated traversal_ids are valid + before_commit :sync_traversal_ids, on: [:create], if: -> { sync_traversal_ids? && sync_traversal_ids_before_commit? } end def sync_traversal_ids? Feature.enabled?(:sync_traversal_ids, root_ancestor, default_enabled: :yaml) end + def sync_traversal_ids_before_commit? + Feature.enabled?(:sync_traversal_ids_before_commit, root_ancestor, default_enabled: :yaml) + end + def use_traversal_ids? return false unless Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 9f0f49e729c..09d69a5f77a 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -12,7 +12,7 @@ module Namespaces def as_ids return super unless use_traversal_ids? - select('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id') + select(Arel.sql('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)]').as('id')) end def roots @@ -53,7 +53,7 @@ module Namespaces end def self_and_descendants(include_self: true) - return super unless use_traversal_ids? + return super unless use_traversal_ids_for_descendants_scopes? if Feature.enabled?(:traversal_ids_btree, default_enabled: :yaml) self_and_descendants_with_comparison_operators(include_self: include_self) @@ -65,7 +65,7 @@ module Namespaces end def self_and_descendant_ids(include_self: true) - return super unless use_traversal_ids? + return super unless use_traversal_ids_for_descendants_scopes? if Feature.enabled?(:traversal_ids_btree, default_enabled: :yaml) self_and_descendants_with_comparison_operators(include_self: include_self).as_ids @@ -75,6 +75,12 @@ module Namespaces end end + def self_and_hierarchy + return super unless use_traversal_ids_for_self_and_hierarchy_scopes? + + unscoped.from_union([all.self_and_ancestors, all.self_and_descendants(include_self: false)]) + end + def order_by_depth(hierarchy_order) return all unless hierarchy_order @@ -109,6 +115,16 @@ module Namespaces use_traversal_ids? end + def use_traversal_ids_for_descendants_scopes? + Feature.enabled?(:use_traversal_ids_for_descendants_scopes, default_enabled: :yaml) && + use_traversal_ids? + end + + def use_traversal_ids_for_self_and_hierarchy_scopes? + Feature.enabled?(:use_traversal_ids_for_self_and_hierarchy_scopes, default_enabled: :yaml) && + use_traversal_ids? + end + def self_and_descendants_with_comparison_operators(include_self: true) base = all.select( :traversal_ids, diff --git a/app/models/namespaces/traversal/recursive_scopes.rb b/app/models/namespaces/traversal/recursive_scopes.rb index 583c53f8221..c6f09a4d134 100644 --- a/app/models/namespaces/traversal/recursive_scopes.rb +++ b/app/models/namespaces/traversal/recursive_scopes.rb @@ -53,6 +53,11 @@ module Namespaces self_and_descendants(include_self: include_self).as_ids end alias_method :recursive_self_and_descendant_ids, :self_and_descendant_ids + + def self_and_hierarchy + Gitlab::ObjectHierarchy.new(all).all_objects + end + alias_method :recursive_self_and_hierarchy, :self_and_hierarchy end end end diff --git a/app/models/namespaces/user_namespace.rb b/app/models/namespaces/user_namespace.rb index 14b867b2607..408acb6dcce 100644 --- a/app/models/namespaces/user_namespace.rb +++ b/app/models/namespaces/user_namespace.rb @@ -25,5 +25,9 @@ module Namespaces def self.sti_name 'User' end + + def owners + Array.wrap(owner) + end end end diff --git a/app/models/note.rb b/app/models/note.rb index a143c21c0f9..3f3fa968393 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -27,10 +27,14 @@ class Note < ApplicationRecord redact_field :note - TYPES_RESTRICTED_BY_ABILITY = { + TYPES_RESTRICTED_BY_PROJECT_ABILITY = { branch: :download_code }.freeze + TYPES_RESTRICTED_BY_GROUP_ABILITY = { + contact: :read_crm_contact + }.freeze + # Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with notes. # See https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/10392/diffs#note_28719102 alias_attribute :last_edited_by, :updated_by @@ -119,7 +123,7 @@ class Note < ApplicationRecord scope :inc_author, -> { includes(:author) } scope :with_api_entity_associations, -> { preload(:note_diff_file, :author) } scope :inc_relations_for_view, -> do - includes(:project, { author: :status }, :updated_by, :resolved_by, :award_emoji, + includes({ project: :group }, { author: :status }, :updated_by, :resolved_by, :award_emoji, { system_note_metadata: :description_version }, :note_diff_file, :diff_note_positions, :suggestions) end @@ -565,10 +569,10 @@ class Note < ApplicationRecord noteable.user_mentions.where(note: self) end - def system_note_with_references_visible_for?(user) + def system_note_visible_for?(user) return true unless system? - (!system_note_with_references? || all_referenced_mentionables_allowed?(user)) && system_note_viewable_by?(user) + system_note_viewable_by?(user) && all_referenced_mentionables_allowed?(user) end def parent_user @@ -617,10 +621,17 @@ class Note < ApplicationRecord def system_note_viewable_by?(user) return true unless system_note_metadata - restriction = TYPES_RESTRICTED_BY_ABILITY[system_note_metadata.action.to_sym] - return Ability.allowed?(user, restriction, project) if restriction + system_note_viewable_by_project_ability?(user) && system_note_viewable_by_group_ability?(user) + end - true + def system_note_viewable_by_project_ability?(user) + project_restriction = TYPES_RESTRICTED_BY_PROJECT_ABILITY[system_note_metadata.action.to_sym] + !project_restriction || Ability.allowed?(user, project_restriction, project) + end + + def system_note_viewable_by_group_ability?(user) + group_restriction = TYPES_RESTRICTED_BY_GROUP_ABILITY[system_note_metadata.action.to_sym] + !group_restriction || Ability.allowed?(user, group_restriction, project&.group) end def keep_around_commit @@ -646,6 +657,8 @@ class Note < ApplicationRecord end def all_referenced_mentionables_allowed?(user) + return true unless system_note_with_references? + if user_visible_reference_count.present? && total_reference_count.present? # if they are not equal, then there are private/confidential references as well user_visible_reference_count > 0 && user_visible_reference_count == total_reference_count diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 4a97ae97ea0..c76473c9438 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -138,7 +138,7 @@ class Packages::Package < ApplicationRecord scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) } scope :has_version, -> { where.not(version: nil) } - scope :preload_files, -> { Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) ? preload(:installable_package_files) : preload(:package_files) } + scope :preload_files, -> { preload(:installable_package_files) } scope :preload_pipelines, -> { preload(pipelines: :user) } scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) } scope :limit_recent, ->(limit) { order_created_desc.limit(limit) } diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 190081c4e8e..fc7c348dfdb 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -55,12 +55,11 @@ class Packages::PackageFile < ApplicationRecord end scope :for_helm_with_channel, ->(project, channel) do - result = joins(:package) - .merge(project.packages.helm.installable) - .joins(:helm_file_metadatum) - .where(packages_helm_file_metadata: { channel: channel }) - result = result.installable if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) - result + joins(:package) + .merge(project.packages.helm.installable) + .joins(:helm_file_metadatum) + .where(packages_helm_file_metadata: { channel: channel }) + .installable end scope :with_conan_file_type, ->(file_type) do @@ -110,13 +109,9 @@ class Packages::PackageFile < ApplicationRecord cte_name = :packages_cte cte = Gitlab::SQL::CTE.new(cte_name, packages.select(:id)) - package_files = if Feature.enabled?(:packages_installable_package_files, default_enabled: :yaml) - ::Packages::PackageFile.installable.limit_recent(1) - .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) - else - ::Packages::PackageFile.limit_recent(1) - .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) - end + package_files = ::Packages::PackageFile.installable + .limit_recent(1) + .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) package_files = package_files.joins(extra_join) if extra_join package_files = package_files.where(extra_where) if extra_where diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index c21027455b1..2804588be85 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -245,8 +245,8 @@ class PagesDomain < ApplicationRecord def validate_pages_domain return unless domain - if domain.downcase.ends_with?(Settings.pages.host.downcase) - self.errors.add(:domain, "*.#{Settings.pages.host} is restricted. Please compare our documentation at https://docs.gitlab.com/ee/administration/pages/#advanced-configuration against your configuration.") + if domain.downcase.ends_with?(".#{Settings.pages.host.downcase}") || domain.casecmp(Settings.pages.host) == 0 + self.errors.add(:domain, "#{Settings.pages.host} and its subdomains cannot be used as custom pages domains. Please compare our documentation at https://docs.gitlab.com/ee/administration/pages/#advanced-configuration against your configuration.") end end diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index 1778e927dd1..2f515f3443d 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -33,6 +33,7 @@ class PersonalAccessToken < ApplicationRecord scope :preload_users, -> { preload(:user) } scope :order_expires_at_asc, -> { reorder(expires_at: :asc) } scope :order_expires_at_desc, -> { reorder(expires_at: :desc) } + scope :project_access_token, -> { includes(:user).where(user: { user_type: :project_bot }) } validates :scopes, presence: true validate :validate_scopes @@ -93,6 +94,10 @@ class PersonalAccessToken < ApplicationRecord "#{self.class.token_prefix}#{token}" end + def project_access_token? + user&.project_bot? + end + protected def validate_scopes diff --git a/app/models/preloaders/single_hierarchy_project_group_plans_preloader.rb b/app/models/preloaders/single_hierarchy_project_group_plans_preloader.rb new file mode 100644 index 00000000000..179214666ed --- /dev/null +++ b/app/models/preloaders/single_hierarchy_project_group_plans_preloader.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Preloaders + class SingleHierarchyProjectGroupPlansPreloader + attr_reader :projects + + def initialize(projects_relation) + @projects = projects_relation + end + + def execute + # no-op in FOSS + end + end +end + +Preloaders::SingleHierarchyProjectGroupPlansPreloader.prepend_mod_with('Preloaders::SingleHierarchyProjectGroupPlansPreloader') diff --git a/app/models/preloaders/users_max_access_level_in_projects_preloader.rb b/app/models/preloaders/users_max_access_level_in_projects_preloader.rb new file mode 100644 index 00000000000..b4ce61a869c --- /dev/null +++ b/app/models/preloaders/users_max_access_level_in_projects_preloader.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Preloaders + # This class preloads the max access level (role) for the users within the given projects and + # stores the values in requests store via the ProjectTeam class. + class UsersMaxAccessLevelInProjectsPreloader + def initialize(projects:, users:) + @projects = projects + @users = users + end + + def execute + return unless @projects.present? && @users.present? + + access_levels.each do |(project_id, user_id), access_level| + project = projects_by_id[project_id] + + project.team.write_member_access_for_user_id(user_id, access_level) + end + end + + private + + def access_levels + ProjectAuthorization + .where(project_id: project_ids, user_id: user_ids) + .group(:project_id, :user_id) + .maximum(:access_level) + end + + # Use reselect to override the existing select to prevent + # the error `subquery has too many columns` + # NotificationsController passes in an Array so we need to check the type + def project_ids + @projects.is_a?(ActiveRecord::Relation) ? @projects.reselect(:id) : @projects + end + + def user_ids + @users.is_a?(ActiveRecord::Relation) ? @users.reselect(:id) : @users + end + + def projects_by_id + @projects_by_id ||= @projects.index_by(&:id) + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 7f823b5ed6b..512c6ac1acb 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -74,6 +74,21 @@ class Project < ApplicationRecord GL_REPOSITORY_TYPES = [Gitlab::GlRepository::PROJECT, Gitlab::GlRepository::WIKI, Gitlab::GlRepository::DESIGN].freeze + MAX_SUGGESTIONS_TEMPLATE_LENGTH = 255 + MAX_COMMIT_TEMPLATE_LENGTH = 500 + + DEFAULT_MERGE_COMMIT_TEMPLATE = <<~MSG.rstrip.freeze + Merge branch '%{source_branch}' into '%{target_branch}' + + %{title} + + %{issues} + + See merge request %{reference} + MSG + + DEFAULT_SQUASH_COMMIT_TEMPLATE = '%{title}' + cache_markdown_field :description, pipeline: :description default_value_for :packages_enabled, true @@ -506,11 +521,12 @@ class Project < ApplicationRecord validates :variables, nested_attributes_duplicates: { scope: :environment_scope } validates :bfg_object_map, file_size: { maximum: :max_attachment_size } validates :max_artifacts_size, numericality: { only_integer: true, greater_than: 0, allow_nil: true } - validates :suggestion_commit_message, length: { maximum: 255 } + validates :suggestion_commit_message, length: { maximum: MAX_SUGGESTIONS_TEMPLATE_LENGTH } # Scopes scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } + scope :not_aimed_for_deletion, -> { where(marked_for_deletion_at: nil).without_deleted } scope :with_storage_feature, ->(feature) do where(arel_table[:storage_version].gteq(HASHED_STORAGE_FEATURES[feature])) @@ -727,6 +743,7 @@ class Project < ApplicationRecord scope :joins_import_state, -> { joins("INNER JOIN project_mirror_data import_state ON import_state.project_id = projects.id") } scope :for_group, -> (group) { where(group: group) } scope :for_group_and_its_subgroups, ->(group) { where(namespace_id: group.self_and_descendants.select(:id)) } + scope :for_group_and_its_ancestor_groups, ->(group) { where(namespace_id: group.self_and_ancestors.select(:id)) } class << self # Searches for a list of projects based on the query given in `query`. @@ -987,7 +1004,7 @@ class Project < ApplicationRecord end def context_commits_enabled? - Feature.enabled?(:context_commits, self, default_enabled: :yaml) + Feature.enabled?(:context_commits, self.group, default_enabled: :yaml) end # LFS and hashed repository storage are required for using Design Management. @@ -1513,9 +1530,25 @@ class Project < ApplicationRecord # rubocop: enable CodeReuse/ServiceClass def owner + # This will be phased out and replaced with `owners` relationship + # backed by memberships with direct/inherited Owner access roles + # See https://gitlab.com/groups/gitlab-org/-/epics/7405 + group || namespace.try(:owner) + end + + def deprecated_owner + # Kept in order to maintain webhook structures until we remove owner_name and owner_email + # See https://gitlab.com/gitlab-org/gitlab/-/issues/350603 group || namespace.try(:owner) end + def owners + # This will be phased out and replaced with `owners` relationship + # backed by memberships with direct/inherited Owner access roles + # See https://gitlab.com/groups/gitlab-org/-/epics/7405 + team.owners + end + def first_owner obj = owner @@ -2168,14 +2201,6 @@ class Project < ApplicationRecord end end - def ci_instance_variables_for(ref:) - if protected_for?(ref) - Ci::InstanceVariable.all_cached - else - Ci::InstanceVariable.unprotected_cached - end - end - def protected_for?(ref) raise Repository::AmbiguousRefError if repository.ambiguous_ref?(ref) @@ -2610,6 +2635,14 @@ class Project < ApplicationRecord [project&.id, root_group&.id] end + def related_group_ids + ids = invited_group_ids + + ids += group.self_and_ancestors_ids if group + + ids + end + def package_already_taken?(package_name, package_version, package_type:) Packages::Package.with_name(package_name) .with_version(package_version) @@ -2746,6 +2779,32 @@ class Project < ApplicationRecord ].compact.min end + def merge_commit_template_or_default + merge_commit_template.presence || DEFAULT_MERGE_COMMIT_TEMPLATE + end + + def merge_commit_template_or_default=(value) + project_setting.merge_commit_template = + if value.blank? || value.delete("\r") == DEFAULT_MERGE_COMMIT_TEMPLATE + nil + else + value + end + end + + def squash_commit_template_or_default + squash_commit_template.presence || DEFAULT_SQUASH_COMMIT_TEMPLATE + end + + def squash_commit_template_or_default=(value) + project_setting.squash_commit_template = + if value.blank? || value.delete("\r") == DEFAULT_SQUASH_COMMIT_TEMPLATE + nil + else + value + end + end + private # overridden in EE @@ -2754,6 +2813,12 @@ class Project < ApplicationRecord end def save_topics + topic_ids_before = self.topic_ids + update_topics + Projects::Topic.update_non_private_projects_counter(topic_ids_before, self.topic_ids, visibility_level_previously_was, visibility_level) + end + + def update_topics return if @topic_list.nil? @topic_list = @topic_list.split(',') if @topic_list.instance_of?(String) diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb index 633e669b5fc..0f04eb7d4af 100644 --- a/app/models/project_import_state.rb +++ b/app/models/project_import_state.rb @@ -57,6 +57,12 @@ class ProjectImportState < ApplicationRecord end end + after_transition any => :failed do |state, _| + if Feature.enabled?(:remove_import_data_on_failure, state.project, default_enabled: :yaml) + state.project.remove_import_data + end + end + after_transition started: :finished do |state, _| project = state.project diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index 4e37174e604..ae3d7038a88 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class ProjectSetting < ApplicationRecord + include IgnorableColumns + + ignore_column :show_diff_preview_in_email, remove_with: '14.10', remove_after: '2022-03-22' + belongs_to :project, inverse_of: :project_setting enum squash_option: { @@ -12,8 +16,12 @@ class ProjectSetting < ApplicationRecord self.primary_key = :project_id - validates :merge_commit_template, length: { maximum: 500 } - validates :squash_commit_template, length: { maximum: 500 } + validates :merge_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH } + validates :squash_commit_template, length: { maximum: Project::MAX_COMMIT_TEMPLATE_LENGTH } + + default_value_for(:legacy_open_source_license_available) do + Feature.enabled?(:legacy_open_source_license_available, default_enabled: :yaml, type: :ops) + end def squash_enabled_by_default? %w[always default_on].include?(squash_option) diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 8061554006d..c3c7508df9f 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -107,6 +107,10 @@ class ProjectTeam end end + def owner?(user) + owners.include?(user) + end + def import(source_project, current_user = nil) target_project = project diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb index 5221b00c55f..7af863c0cf0 100644 --- a/app/models/projects/sync_event.rb +++ b/app/models/projects/sync_event.rb @@ -13,4 +13,8 @@ class Projects::SyncEvent < ApplicationRecord def self.enqueue_worker ::Projects::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker end + + def self.upper_bound_count + select('COALESCE(MAX(id) - MIN(id) + 1, 0) AS upper_bound_count').to_a.first.upper_bound_count + end end diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb index 8d6f8c3a9ca..78bc2df2e1e 100644 --- a/app/models/projects/topic.rb +++ b/app/models/projects/topic.rb @@ -25,6 +25,29 @@ module Projects def search(query) fuzzy_search(query, [:name]) end + + def update_non_private_projects_counter(ids_before, ids_after, project_visibility_level_before, project_visibility_level_after) + project_visibility_level_before ||= project_visibility_level_after + + topics_to_decrement = [] + topics_to_increment = [] + topic_ids_removed = ids_before - ids_after + topic_ids_retained = ids_before & ids_after + topic_ids_added = ids_after - ids_before + + if project_visibility_level_before > Gitlab::VisibilityLevel::PRIVATE + topics_to_decrement += topic_ids_removed + topics_to_decrement += topic_ids_retained if project_visibility_level_after == Gitlab::VisibilityLevel::PRIVATE + end + + if project_visibility_level_after > Gitlab::VisibilityLevel::PRIVATE + topics_to_increment += topic_ids_added + topics_to_increment += topic_ids_retained if project_visibility_level_before == Gitlab::VisibilityLevel::PRIVATE + end + + where(id: topics_to_increment).update_counters(non_private_projects_count: 1) unless topics_to_increment.empty? + where(id: topics_to_decrement).where('non_private_projects_count > 0').update_counters(non_private_projects_count: -1) unless topics_to_decrement.empty? + end end end end diff --git a/app/models/resource_label_event.rb b/app/models/resource_label_event.rb index 68f0ab06bea..0a59d9cef9b 100644 --- a/app/models/resource_label_event.rb +++ b/app/models/resource_label_event.rb @@ -54,7 +54,7 @@ class ResourceLabelEvent < ResourceEvent end def banzai_render_context(field) - super.merge(pipeline: :label, only_path: true) + super.merge(pipeline: :label, only_path: true, label_url_method: label_url_method) end def refresh_invalid_reference @@ -91,6 +91,10 @@ class ResourceLabelEvent < ResourceEvent end end + def label_url_method + issuable.is_a?(MergeRequest) ? :project_merge_requests_url : :project_issues_url + end + def expire_etag_cache issuable.expire_note_etag_cache end diff --git a/app/models/state_note.rb b/app/models/state_note.rb index 5e35f15aac4..93c025a9bf0 100644 --- a/app/models/state_note.rb +++ b/app/models/state_note.rb @@ -18,11 +18,11 @@ class StateNote < SyntheticNote def note_text(html: false) if event.state == 'closed' if event.close_after_error_tracking_resolve - return 'resolved the corresponding error and closed the issue.' + return 'resolved the corresponding error and closed the issue' end if event.close_auto_resolve_prometheus_alert - return 'automatically closed this issue because the alert resolved.' + return 'automatically closed this incident because the alert resolved' end end diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index a3c9db90b5d..0be56d8b4a4 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -24,7 +24,7 @@ class SystemNoteMetadata < ApplicationRecord opened closed merged duplicate locked unlocked outdated reviewer tag due_date pinned_embed cherry_pick health_status approved unapproved status alert_issue_added relate unrelate new_alert_added severity - attention_requested attention_request_removed + attention_requested attention_request_removed contact ].freeze validates :note, presence: true, unless: :importing? diff --git a/app/models/user.rb b/app/models/user.rb index 1d452fc2e50..74832bff9ac 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -149,6 +149,7 @@ class User < ApplicationRecord has_many :members has_many :group_members, -> { where(requested_at: nil).where("access_level >= ?", Gitlab::Access::GUEST) }, class_name: 'GroupMember' has_many :groups, through: :group_members + has_many :groups_with_active_memberships, -> { where(members: { state: ::Member::STATE_ACTIVE }) }, through: :group_members, source: :group has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :group has_many :maintainers_groups, -> { where(members: { access_level: Gitlab::Access::MAINTAINER }) }, through: :group_members, source: :group has_many :developer_groups, -> { where(members: { access_level: ::Gitlab::Access::DEVELOPER }) }, through: :group_members, source: :group @@ -170,6 +171,7 @@ class User < ApplicationRecord has_many :project_members, -> { where(requested_at: nil) } has_many :projects, through: :project_members has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' + has_many :projects_with_active_memberships, -> { where(members: { state: ::Member::STATE_ACTIVE }) }, through: :project_members, source: :project has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :starred_projects, through: :users_star_projects, source: :project has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -668,7 +670,8 @@ class User < ApplicationRecord sanitized_order_sql = Arel.sql(sanitize_sql_array([order, query: query])) - scope = options[:with_private_emails] ? search_with_secondary_emails(query) : search_with_public_emails(query) + scope = options[:with_private_emails] ? with_primary_or_secondary_email(query) : with_public_email(query) + scope = scope.or(search_by_name_or_username(query, use_minimum_char_limit: options[:use_minimum_char_limit])) scope.reorder(sanitized_order_sql, :name) end @@ -685,50 +688,32 @@ class User < ApplicationRecord reorder(:name) end - def search_with_public_emails(query) - return none if query.blank? - - query = query.downcase + # searches user by given pattern + # it compares name and username fields with given pattern + # This method uses ILIKE on PostgreSQL. + def search_by_name_or_username(query, use_minimum_char_limit: nil) + use_minimum_char_limit = user_search_minimum_char_limit if use_minimum_char_limit.nil? where( - fuzzy_arel_match(:name, query, use_minimum_char_limit: user_search_minimum_char_limit) - .or(fuzzy_arel_match(:username, query, use_minimum_char_limit: user_search_minimum_char_limit)) - .or(arel_table[:public_email].eq(query)) + fuzzy_arel_match(:name, query, use_minimum_char_limit: use_minimum_char_limit) + .or(fuzzy_arel_match(:username, query, use_minimum_char_limit: use_minimum_char_limit)) ) end - def search_without_secondary_emails(query) - return none if query.blank? - - query = query.downcase - - where( - fuzzy_arel_match(:name, query, lower_exact_match: true) - .or(fuzzy_arel_match(:username, query, lower_exact_match: true)) - .or(arel_table[:email].eq(query)) - ) + def with_public_email(email_address) + where(public_email: email_address) end - # searches user by given pattern - # it compares name, email, username fields and user's secondary emails with given pattern - # This method uses ILIKE on PostgreSQL. - - def search_with_secondary_emails(query) - return none if query.blank? - - query = query.downcase - + def with_primary_or_secondary_email(email_address) email_table = Email.arel_table matched_by_email_user_id = email_table .project(email_table[:user_id]) - .where(email_table[:email].eq(query)) + .where(email_table[:email].eq(email_address)) .take(1) # at most 1 record as there is a unique constraint where( - fuzzy_arel_match(:name, query, use_minimum_char_limit: user_search_minimum_char_limit) - .or(fuzzy_arel_match(:username, query, use_minimum_char_limit: user_search_minimum_char_limit)) - .or(arel_table[:email].eq(query)) - .or(arel_table[:id].eq(matched_by_email_user_id)) + arel_table[:email].eq(email_address) + .or(arel_table[:id].eq(matched_by_email_user_id)) ) end @@ -1608,7 +1593,7 @@ class User < ApplicationRecord .distinct .reorder(nil) - Project.where(id: events) + Project.where(id: events).not_aimed_for_deletion end def can_be_removed? diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 8394192c5ae..5c39e29a128 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -40,8 +40,9 @@ module Users profile_personal_access_token_expiry: 37, # EE-only terraform_notification_dismissed: 38, security_newsletter_callout: 39, - verification_reminder: 40, # EE-only - ci_deprecation_warning_for_types_keyword: 41 + verification_reminder: 40, # EE-only + ci_deprecation_warning_for_types_keyword: 41, + security_training_feature_promotion: 42 # EE-only } validates :feature_name, diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index da9b95fd718..0dc449719ab 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -9,7 +9,12 @@ module Users belongs_to :group enum feature_name: { - invite_members_banner: 1 + invite_members_banner: 1, + approaching_seat_count_threshold: 2, # EE-only + storage_enforcement_banner_first_enforcement_threshold: 43, + storage_enforcement_banner_second_enforcement_threshold: 44, + storage_enforcement_banner_third_enforcement_threshold: 45, + storage_enforcement_banner_fourth_enforcement_threshold: 46 } validates :group, presence: true diff --git a/app/models/users_star_project.rb b/app/models/users_star_project.rb index c633e2d8b3d..1549c099a64 100644 --- a/app/models/users_star_project.rb +++ b/app/models/users_star_project.rb @@ -32,7 +32,7 @@ class UsersStarProject < ApplicationRecord end def search(query) - joins(:user).merge(User.search(query)) + joins(:user).merge(User.search(query, use_minimum_char_limit: false)) end end end diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb index 4e1f48227d9..a5881e80e88 100644 --- a/app/models/vulnerability.rb +++ b/app/models/vulnerability.rb @@ -2,6 +2,7 @@ # Placeholder class for model that is implemented in EE class Vulnerability < ApplicationRecord + include EachBatch include IgnorableColumns def self.link_reference_pattern diff --git a/app/models/work_item.rb b/app/models/work_item.rb index 02f52f04c85..99f05e4a181 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -3,4 +3,8 @@ class WorkItem < Issue self.table_name = 'issues' self.inheritance_column = :_type_disabled + + def noteable_target_type_name + 'issue' + end end |