diff options
Diffstat (limited to 'app/models/concerns')
-rw-r--r-- | app/models/concerns/avatarable.rb | 1 | ||||
-rw-r--r-- | app/models/concerns/cache_markdown_field.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/cached_commit.rb | 4 | ||||
-rw-r--r-- | app/models/concerns/ci/partitionable.rb | 44 | ||||
-rw-r--r-- | app/models/concerns/ci/partitionable/partitioned_filter.rb | 41 | ||||
-rw-r--r-- | app/models/concerns/commit_signature.rb | 4 | ||||
-rw-r--r-- | app/models/concerns/counter_attribute.rb | 201 | ||||
-rw-r--r-- | app/models/concerns/has_user_type.rb | 6 | ||||
-rw-r--r-- | app/models/concerns/issuable.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/milestoneable.rb | 23 | ||||
-rw-r--r-- | app/models/concerns/sensitive_serializable_hash.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/signature_type.rb | 13 | ||||
-rw-r--r-- | app/models/concerns/sortable.rb | 2 | ||||
-rw-r--r-- | app/models/concerns/taskable.rb | 15 | ||||
-rw-r--r-- | app/models/concerns/time_trackable.rb | 10 |
15 files changed, 189 insertions, 181 deletions
diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index b32502c3ee2..f419fa8518e 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -16,7 +16,6 @@ module Avatarable included do prepend ShadowMethods - include ObjectStorage::BackgroundMove include Gitlab::Utils::StrongMemoize include ApplicationHelper diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index ec0cf36d875..6a855198697 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -40,7 +40,7 @@ module CacheMarkdownField # Banzai is less strict about authors, so don't always have an author key context[:author] = self.author if self.respond_to?(:author) - context[:markdown_engine] = :common_mark + context[:markdown_engine] = Banzai::Filter::MarkdownFilter::DEFAULT_ENGINE if Feature.enabled?(:personal_snippet_reference_filters, context[:author]) context[:user] = self.parent_user diff --git a/app/models/concerns/cached_commit.rb b/app/models/concerns/cached_commit.rb index 183d5728743..0fb72552dd5 100644 --- a/app/models/concerns/cached_commit.rb +++ b/app/models/concerns/cached_commit.rb @@ -4,8 +4,8 @@ module CachedCommit extend ActiveSupport::Concern def to_hash - Gitlab::Git::Commit::SERIALIZE_KEYS.each_with_object({}) do |key, hash| - hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + Gitlab::Git::Commit::SERIALIZE_KEYS.index_with do |key| + public_send(key) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/concerns/ci/partitionable.rb b/app/models/concerns/ci/partitionable.rb index 68a6714c892..d6ba0f4488f 100644 --- a/app/models/concerns/ci/partitionable.rb +++ b/app/models/concerns/ci/partitionable.rb @@ -25,10 +25,21 @@ module Ci PARTITIONABLE_MODELS = %w[ CommitStatus Ci::BuildMetadata - Ci::Stage + Ci::BuildNeed + Ci::BuildReportResult + Ci::BuildRunnerSession + Ci::BuildTraceChunk + Ci::BuildTraceMetadata + Ci::BuildPendingState Ci::JobArtifact - Ci::PipelineVariable + Ci::JobVariable Ci::Pipeline + Ci::PendingBuild + Ci::RunningBuild + Ci::PipelineVariable + Ci::Sources::Pipeline + Ci::Stage + Ci::UnitTestFailure ].freeze def self.check_inclusion(klass) @@ -57,14 +68,31 @@ module Ci end class_methods do - def partitionable(scope:, through: nil) - if through - define_singleton_method(:routing_table_name) { through[:table] } - define_singleton_method(:routing_table_name_flag) { through[:flag] } + def partitionable(scope:, through: nil, partitioned: false) + handle_partitionable_through(through) + handle_partitionable_dml(partitioned) + handle_partitionable_scope(scope) + end - include Partitionable::Switch - end + private + + def handle_partitionable_through(options) + return unless options + + define_singleton_method(:routing_table_name) { options[:table] } + define_singleton_method(:routing_table_name_flag) { options[:flag] } + + include Partitionable::Switch + end + + def handle_partitionable_dml(partitioned) + define_singleton_method(:partitioned?) { partitioned } + return unless partitioned + + include Partitionable::PartitionedFilter + end + def handle_partitionable_scope(scope) define_method(:partition_scope_value) do strong_memoize(:partition_scope_value) do next Ci::Pipeline.current_partition_value if respond_to?(:importing?) && importing? diff --git a/app/models/concerns/ci/partitionable/partitioned_filter.rb b/app/models/concerns/ci/partitionable/partitioned_filter.rb new file mode 100644 index 00000000000..4adae3be26a --- /dev/null +++ b/app/models/concerns/ci/partitionable/partitioned_filter.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Ci + module Partitionable + # Used to patch the save, update, delete, destroy methods to use the + # partition_id attributes for their SQL queries. + module PartitionedFilter + extend ActiveSupport::Concern + + if Rails::VERSION::MAJOR >= 7 + # These methods are updated in Rails 7 to use `_primary_key_constraints_hash` + # by default, so this patch will no longer be required. + # + # rubocop:disable Gitlab/NoCodeCoverageComment + # :nocov: + raise "`#{__FILE__}` should be double checked" if Rails.env.test? + + warn "Update `#{__FILE__}`. Patches Rails internals for partitioning" + # :nocov: + # rubocop:enable Gitlab/NoCodeCoverageComment + else + def _update_row(attribute_names, attempted_action = "update") + self.class._update_record( + attributes_with_values(attribute_names), + _primary_key_constraints_hash + ) + end + + def _delete_row + self.class._delete_record(_primary_key_constraints_hash) + end + end + + # Introduced in Rails 7, but updated to include `partition_id` filter. + # https://github.com/rails/rails/blob/a4dbb153fd390ac31bb9808809e7ac4d3a2c5116/activerecord/lib/active_record/persistence.rb#L1031-L1033 + def _primary_key_constraints_hash + { @primary_key => id_in_database, partition_id: partition_id } # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/app/models/concerns/commit_signature.rb b/app/models/concerns/commit_signature.rb index 5bdfa9a2966..7f1fbbefd94 100644 --- a/app/models/concerns/commit_signature.rb +++ b/app/models/concerns/commit_signature.rb @@ -44,7 +44,7 @@ module CommitSignature project.commit(commit_sha) end - def user - commit.committer + def signed_by_user + raise NoMethodError, 'must implement `signed_by_user` method' end end diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 03e062a9855..f1efbba67e1 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -17,14 +17,29 @@ # counter_attribute :storage_size # end # +# It's possible to define a conditional counter attribute. You need to pass a proc +# that must accept a single argument, the object instance on which this concern is +# included. +# +# @example: +# +# class ProjectStatistics +# include CounterAttribute +# +# counter_attribute :conditional_one, if: -> { |object| object.use_counter_attribute? } +# end +# # To increment the counter we can use the method: -# delayed_increment_counter(:commit_count, 3) +# increment_counter(:commit_count, 3) +# +# This method would determine whether it would increment the counter using Redis, +# or fallback to legacy increment on ActiveRecord counters. # # It is possible to register callbacks to be executed after increments have # been flushed to the database. Callbacks are not executed if there are no increments # to flush. # -# counter_attribute_after_flush do |statistic| +# counter_attribute_after_commit do |statistic| # Namespaces::ScheduleAggregationWorker.perform_async(statistic.namespace_id) # end # @@ -32,99 +47,51 @@ module CounterAttribute extend ActiveSupport::Concern extend AfterCommitQueue include Gitlab::ExclusiveLeaseHelpers - - LUA_STEAL_INCREMENT_SCRIPT = <<~EOS - local increment_key, flushed_key = KEYS[1], KEYS[2] - local increment_value = redis.call("get", increment_key) or 0 - local flushed_value = redis.call("incrby", flushed_key, increment_value) - if flushed_value == 0 then - redis.call("del", increment_key, flushed_key) - else - redis.call("del", increment_key) - end - return flushed_value - EOS - - WORKER_DELAY = 10.minutes - WORKER_LOCK_TTL = 10.minutes + include Gitlab::Utils::StrongMemoize class_methods do - def counter_attribute(attribute) - counter_attributes << attribute + def counter_attribute(attribute, if: nil) + counter_attributes << { + attribute: attribute, + if_proc: binding.local_variable_get(:if) # can't read `if` directly + } end def counter_attributes - @counter_attributes ||= Set.new + @counter_attributes ||= [] end - def after_flush_callbacks - @after_flush_callbacks ||= [] + def after_commit_callbacks + @after_commit_callbacks ||= [] end - # perform registered callbacks after increments have been flushed to the database - def counter_attribute_after_flush(&callback) - after_flush_callbacks << callback - end - - def counter_attribute_enabled?(attribute) - counter_attributes.include?(attribute) + # perform registered callbacks after increments have been committed to the database + def counter_attribute_after_commit(&callback) + after_commit_callbacks << callback end end - # This method must only be called by FlushCounterIncrementsWorker - # because it should run asynchronously and with exclusive lease. - # This will - # 1. temporarily move the pending increment for a given attribute - # to a relative "flushed" Redis key, delete the increment key and return - # the value. If new increments are performed at this point, the increment - # key is recreated as part of `delayed_increment_counter`. - # The "flushed" key is used to ensure that we can keep incrementing - # counters in Redis while flushing existing values. - # 2. then the value is used to update the counter in the database. - # 3. finally the "flushed" key is deleted. - def flush_increments_to_database!(attribute) - lock_key = counter_lock_key(attribute) - - with_exclusive_lease(lock_key) do - previous_db_value = read_attribute(attribute) - increment_key = counter_key(attribute) - flushed_key = counter_flushed_key(attribute) - increment_value = steal_increments(increment_key, flushed_key) - new_db_value = nil - - next if increment_value == 0 - - transaction do - update_counters_with_lease({ attribute => increment_value }) - redis_state { |redis| redis.del(flushed_key) } - new_db_value = reset.read_attribute(attribute) - end + def counter_attribute_enabled?(attribute) + counter_attribute = self.class.counter_attributes.find { |registered| registered[:attribute] == attribute } + return false unless counter_attribute + return true unless counter_attribute[:if_proc] - execute_after_flush_callbacks + counter_attribute[:if_proc].call(self) + end - log_flush_counter(attribute, increment_value, previous_db_value, new_db_value) + def counter(attribute) + strong_memoize_with(:counter, attribute) do + # This needs #to_sym because attribute could come from a Sidekiq param, + # which would be a string. + build_counter_for(attribute.to_sym) end end - def delayed_increment_counter(attribute, increment) - raise ArgumentError, "#{attribute} is not a counter attribute" unless counter_attribute_enabled?(attribute) - + def increment_counter(attribute, increment) return if increment == 0 run_after_commit_or_now do - increment_counter(attribute, increment) - - FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, self.class.name, self.id, attribute) - end - - true - end - - def increment_counter(attribute, increment) - if counter_attribute_enabled?(attribute) - new_value = redis_state do |redis| - redis.incrby(counter_key(attribute), increment) - end + new_value = counter(attribute).increment(increment) log_increment_counter(attribute, increment, new_value) end @@ -137,74 +104,33 @@ module CounterAttribute end def reset_counter!(attribute) - if counter_attribute_enabled?(attribute) - detect_race_on_record(log_fields: { caller: __method__, attributes: attribute }) do - update!(attribute => 0) - clear_counter!(attribute) - end - - log_clear_counter(attribute) + detect_race_on_record(log_fields: { caller: __method__, attributes: attribute }) do + counter(attribute).reset! end - end - def get_counter_value(attribute) - if counter_attribute_enabled?(attribute) - redis_state do |redis| - redis.get(counter_key(attribute)).to_i - end - end + log_clear_counter(attribute) end - def counter_key(attribute) - "project:{#{project_id}}:counters:#{self.class}:#{id}:#{attribute}" - end - - def counter_flushed_key(attribute) - counter_key(attribute) + ':flushed' - end - - def counter_lock_key(attribute) - counter_key(attribute) + ':lock' - end - - def counter_attribute_enabled?(attribute) - self.class.counter_attribute_enabled?(attribute) + def execute_after_commit_callbacks + self.class.after_commit_callbacks.each do |callback| + callback.call(self.reset) + end end private - def database_lock_key - "project:{#{project_id}}:#{self.class}:#{id}" - end - - def steal_increments(increment_key, flushed_key) - redis_state do |redis| - redis.eval(LUA_STEAL_INCREMENT_SCRIPT, keys: [increment_key, flushed_key]) - end - end + def build_counter_for(attribute) + raise ArgumentError, %(attribute "#{attribute}" does not exist) unless has_attribute?(attribute) - def clear_counter!(attribute) - redis_state do |redis| - redis.del(counter_key(attribute)) - end - end - - def execute_after_flush_callbacks - self.class.after_flush_callbacks.each do |callback| - callback.call(self) + if counter_attribute_enabled?(attribute) + Gitlab::Counters::BufferedCounter.new(self, attribute) + else + Gitlab::Counters::LegacyCounter.new(self, attribute) end end - def redis_state(&block) - Gitlab::Redis::SharedState.with(&block) - end - - def with_exclusive_lease(lock_key) - in_lock(lock_key, ttl: WORKER_LOCK_TTL) do - yield - end - rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError - # a worker is already updating the counters + def database_lock_key + "project:{#{project_id}}:#{self.class}:#{id}" end # detect_race_on_record uses a lease to monitor access @@ -258,19 +184,6 @@ module CounterAttribute Gitlab::AppLogger.info(payload) end - def log_flush_counter(attribute, increment, previous_db_value, new_db_value) - payload = Gitlab::ApplicationContext.current.merge( - message: 'Flush counter attribute to database', - attribute: attribute, - project_id: project_id, - increment: increment, - previous_db_value: previous_db_value, - new_db_value: new_db_value - ) - - Gitlab::AppLogger.info(payload) - end - def log_clear_counter(attribute) payload = Gitlab::ApplicationContext.current.merge( message: 'Clear counter attribute', diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb index ad070090dd5..1af655277b8 100644 --- a/app/models/concerns/has_user_type.rb +++ b/app/models/concerns/has_user_type.rb @@ -13,10 +13,11 @@ module HasUserType project_bot: 6, migration_bot: 7, security_bot: 8, - automation_bot: 9 + automation_bot: 9, + admin_bot: 11 }.with_indifferent_access.freeze - BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot automation_bot].freeze + BOT_USER_TYPES = %w[alert_bot project_bot support_bot visual_review_bot migration_bot security_bot automation_bot admin_bot].freeze NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze @@ -24,7 +25,6 @@ module HasUserType scope :humans, -> { where(user_type: :human) } scope :bots, -> { where(user_type: BOT_USER_TYPES) } scope :without_bots, -> { humans.or(where.not(user_type: BOT_USER_TYPES)) } - scope :bots_without_project_bot, -> { where(user_type: BOT_USER_TYPES - ['project_bot']) } scope :non_internal, -> { humans.or(where(user_type: NON_INTERNAL_USER_TYPES)) } scope :without_ghosts, -> { humans.or(where.not(user_type: :ghost)) } scope :without_project_bot, -> { humans.or(where.not(user_type: :project_bot)) } diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 31b2a8d7cc1..9f0cd96a8f8 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -366,7 +366,7 @@ module Issuable select(issuable_columns) .select(extra_select_columns) - .from("#{table_name}") + .from(table_name.to_s) .joins("JOIN LATERAL(#{highest_priority}) as highest_priorities ON TRUE") .group(group_columns) .reorder(highest_priority_arel_with_direction.nulls_last) diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index a95bed7ad42..e95a8a42aa6 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -9,6 +9,12 @@ module Milestoneable extend ActiveSupport::Concern + class_methods do + def milestone_releases_subquery + Milestone.joins(:releases).where("#{table_name}.milestone_id = milestones.id") + end + end + included do belongs_to :milestone @@ -17,9 +23,15 @@ module Milestoneable scope :any_milestone, -> { where.not(milestone_id: nil) } scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) } scope :without_particular_milestones, ->(titles) { left_outer_joins(:milestone).where("milestones.title NOT IN (?) OR milestone_id IS NULL", titles) } - scope :any_release, -> { joins_milestone_releases } - scope :with_release, -> (tag, project_id) { joins_milestone_releases.where(milestones: { releases: { tag: tag, project_id: project_id } }) } - scope :without_particular_release, -> (tag, project_id) { joins_milestone_releases.where.not(milestones: { releases: { tag: tag, project_id: project_id } }) } + scope :any_release, -> do + where("EXISTS (?)", milestone_releases_subquery) + end + scope :with_release, -> (tag, project_id) do + where("EXISTS (?)", milestone_releases_subquery.where(releases: { tag: tag, project_id: project_id })) + end + scope :without_particular_release, -> (tag, project_id) do + where("EXISTS (?)", milestone_releases_subquery.where.not(releases: { tag: tag, project_id: project_id })) + end scope :left_joins_milestones, -> { joins("LEFT OUTER JOIN milestones ON #{table_name}.milestone_id = milestones.id") } scope :order_milestone_due_desc, -> { left_joins_milestones.reorder(Arel.sql('milestones.due_date IS NULL, milestones.id IS NULL, milestones.due_date DESC')) } @@ -30,11 +42,6 @@ module Milestoneable .where(milestone_releases: { release_id: nil }) end - scope :joins_milestone_releases, -> do - joins("JOIN milestone_releases ON #{table_name}.milestone_id = milestone_releases.milestone_id - JOIN releases ON milestone_releases.release_id = releases.id").distinct - end - private def milestone_is_valid diff --git a/app/models/concerns/sensitive_serializable_hash.rb b/app/models/concerns/sensitive_serializable_hash.rb index 4ad8d16fcb9..794748483e4 100644 --- a/app/models/concerns/sensitive_serializable_hash.rb +++ b/app/models/concerns/sensitive_serializable_hash.rb @@ -19,8 +19,6 @@ module SensitiveSerializableHash # In general, prefer NOT to use serializable_hash / to_json / as_json in favor # of serializers / entities instead which has an allowlist of attributes def serializable_hash(options = nil) - return super if options && options[:unsafe_serialization_hash] - options = options.try(:dup) || {} options[:except] = Array(options[:except]).dup diff --git a/app/models/concerns/signature_type.rb b/app/models/concerns/signature_type.rb new file mode 100644 index 00000000000..804f42b6f72 --- /dev/null +++ b/app/models/concerns/signature_type.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module SignatureType + TYPES = %i[gpg ssh x509].freeze + + def type + raise NoMethodError, 'must implement `type` method' + end + + TYPES.each do |type| + define_method("#{type}?") { self.type == type } + end +end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index eccb004b503..6532a18d1b8 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -72,7 +72,7 @@ module Sortable private - def highest_label_priority(target_type_column: nil, target_type: nil, target_column:, project_column:, excluded_labels: []) + def highest_label_priority(target_column:, project_column:, target_type_column: nil, target_type: nil, excluded_labels: []) query = Label.select(LabelPriority.arel_table[:priority].minimum.as('label_priority')) .left_join_priorities .joins(:label_links) diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index ee5774d4868..05addcf83d2 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -63,14 +63,15 @@ module Taskable def task_status(short: false) return '' if description.blank? - prep, completed = if short - ['/', ''] - else - [' of ', ' completed'] - end - sum = tasks.summary - "#{sum.complete_count}#{prep}#{sum.item_count} #{'checklist item'.pluralize(sum.item_count)}#{completed}" + checklist_item_noun = n_('checklist item', 'checklist items', sum.item_count) + if short + format(s_('Tasks|%{complete_count}/%{total_count} %{checklist_item_noun}'), +checklist_item_noun: checklist_item_noun, complete_count: sum.complete_count, total_count: sum.item_count) + else + format(s_('Tasks|%{complete_count} of %{total_count} %{checklist_item_noun} completed'), +checklist_item_noun: checklist_item_noun, complete_count: sum.complete_count, total_count: sum.item_count) + end end # Return a short string that describes the current state of this Taskable's diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 54fe9eac2bc..2b7447dc700 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -15,12 +15,13 @@ module TimeTrackable alias_method :time_spent?, :time_spent - default_value_for :time_estimate, value: 0, allows_nil: false + attribute :time_estimate, default: 0 validates :time_estimate, numericality: { message: 'has an invalid format' }, allow_nil: false validate :check_negative_time_spent has_many :timelogs, dependent: :destroy, autosave: true # rubocop:disable Cop/ActiveRecordDependent + after_initialize :set_time_estimate_default_value end # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -67,6 +68,13 @@ module TimeTrackable val.is_a?(Integer) ? super([val, Gitlab::Database::MAX_INT_VALUE].min) : super(val) end + def set_time_estimate_default_value + return if new_record? + return unless has_attribute?(:time_estimate) + + self.time_estimate ||= self.class.column_defaults['time_estimate'] + end + private def reset_spent_time |