diff options
Diffstat (limited to 'app/models/internal_id.rb')
-rw-r--r-- | app/models/internal_id.rb | 171 |
1 files changed, 147 insertions, 24 deletions
diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index b56bac58705..f114094d69c 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -16,7 +16,7 @@ # * Add `usage` value to enum # * (Optionally) add columns to `internal_ids` if needed for scope. class InternalId < ApplicationRecord - include Gitlab::Utils::StrongMemoize + extend Gitlab::Utils::StrongMemoize belongs_to :project belongs_to :namespace @@ -25,6 +25,10 @@ class InternalId < ApplicationRecord validates :usage, presence: true + scope :filter_by, -> (scope, usage) do + where(**scope, usage: usage) + end + # Increments #last_value and saves the record # # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). @@ -53,18 +57,15 @@ class InternalId < ApplicationRecord class << self def track_greatest(subject, scope, usage, new_value, init) - InternalIdGenerator.new(subject, scope, usage, init) - .track_greatest(new_value) + build_generator(subject, scope, usage, init).track_greatest(new_value) end def generate_next(subject, scope, usage, init) - InternalIdGenerator.new(subject, scope, usage, init) - .generate + build_generator(subject, scope, usage, init).generate end def reset(subject, scope, usage, value) - InternalIdGenerator.new(subject, scope, usage) - .reset(value) + build_generator(subject, scope, usage).reset(value) end # Flushing records is generally safe in a sense that those @@ -77,11 +78,36 @@ class InternalId < ApplicationRecord where(filter).delete_all end + + def internal_id_transactions_increment(operation:, usage:) + self.internal_id_transactions_total.increment( + operation: operation, + usage: usage.to_s, + in_transaction: ActiveRecord::Base.connection.transaction_open?.to_s + ) + end + + def internal_id_transactions_total + strong_memoize(:internal_id_transactions_total) do + name = :gitlab_internal_id_transactions_total + comment = 'Counts all the internal ids happening within transaction' + + Gitlab::Metrics.counter(name, comment) + end + end + + private + + def build_generator(subject, scope, usage, init = nil) + if Feature.enabled?(:generate_iids_without_explicit_locking) + ImplicitlyLockingInternalIdGenerator.new(subject, scope, usage, init) + else + InternalIdGenerator.new(subject, scope, usage, init) + end + end end class InternalIdGenerator - extend Gitlab::Utils::StrongMemoize - # Generate next internal id for a given scope and usage. # # For currently supported usages, see #usage enum. @@ -117,7 +143,7 @@ class InternalId < ApplicationRecord # init: Block that gets called to initialize InternalId record if not present # Make sure to not throw exceptions in the absence of records (if this is expected). def generate - self.class.internal_id_transactions_increment(operation: :generate, usage: usage) + InternalId.internal_id_transactions_increment(operation: :generate, usage: usage) subject.transaction do # Create a record in internal_ids if one does not yet exist @@ -134,7 +160,7 @@ class InternalId < ApplicationRecord def reset(value) return false unless value - self.class.internal_id_transactions_increment(operation: :reset, usage: usage) + InternalId.internal_id_transactions_increment(operation: :reset, usage: usage) updated = InternalId @@ -149,8 +175,9 @@ class InternalId < ApplicationRecord # and set its new_value if it is higher than the current last_value # # Note this will acquire a ROW SHARE lock on the InternalId record + def track_greatest(new_value) - self.class.internal_id_transactions_increment(operation: :track_greatest, usage: usage) + InternalId.internal_id_transactions_increment(operation: :track_greatest, usage: usage) subject.transaction do record.track_greatest_and_save!(new_value) @@ -162,7 +189,7 @@ class InternalId < ApplicationRecord end def with_lock(&block) - self.class.internal_id_transactions_increment(operation: :with_lock, usage: usage) + InternalId.internal_id_transactions_increment(operation: :with_lock, usage: usage) record.with_lock(&block) end @@ -199,22 +226,118 @@ class InternalId < ApplicationRecord rescue ActiveRecord::RecordNotUnique lookup end + end - def self.internal_id_transactions_increment(operation:, usage:) - self.internal_id_transactions_total.increment( - operation: operation, - usage: usage.to_s, - in_transaction: ActiveRecord::Base.connection.transaction_open?.to_s - ) + class ImplicitlyLockingInternalIdGenerator + # Generate next internal id for a given scope and usage. + # + # For currently supported usages, see #usage enum. + # + # The method implements a locking scheme that has the following properties: + # 1) Generated sequence of internal ids is unique per (scope and usage) + # 2) The method is thread-safe and may be used in concurrent threads/processes. + # 3) The generated sequence is gapless. + # 4) In the absence of a record in the internal_ids table, one will be created + # and last_value will be calculated on the fly. + # + # subject: The instance or class we're generating an internal id for. + # scope: Attributes that define the scope for id generation. + # Valid keys are `project/project_id` and `namespace/namespace_id`. + # usage: Symbol to define the usage of the internal id, see InternalId.usages + # init: Proc that accepts the subject and the scope and returns Integer|NilClass + attr_reader :subject, :scope, :scope_attrs, :usage, :init + + def initialize(subject, scope, usage, init = nil) + @subject = subject + @scope = scope + @usage = usage + @init = init + + raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? + + unless InternalId.usages.has_key?(usage.to_s) + raise ArgumentError, "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages" + end end - def self.internal_id_transactions_total - strong_memoize(:internal_id_transactions_total) do - name = :gitlab_internal_id_transactions_total - comment = 'Counts all the internal ids happening within transaction' + # Generates next internal id and returns it + # init: Block that gets called to initialize InternalId record if not present + # Make sure to not throw exceptions in the absence of records (if this is expected). + def generate + InternalId.internal_id_transactions_increment(operation: :generate, usage: usage) - Gitlab::Metrics.counter(name, comment) + next_iid = update_record!(subject, scope, usage, arel_table[:last_value] + 1) + + return next_iid if next_iid + + create_record!(subject, scope, usage, init) do |iid| + iid.last_value += 1 end + rescue ActiveRecord::RecordNotUnique + retry + end + + # Reset tries to rewind to `value-1`. This will only succeed, + # if `value` stored in database is equal to `last_value`. + # value: The expected last_value to decrement + def reset(value) + return false unless value + + InternalId.internal_id_transactions_increment(operation: :reset, usage: usage) + + iid = update_record!(subject, scope.merge(last_value: value), usage, arel_table[:last_value] - 1) + iid == value - 1 + end + + # Create a record in internal_ids if one does not yet exist + # and set its new_value if it is higher than the current last_value + def track_greatest(new_value) + InternalId.internal_id_transactions_increment(operation: :track_greatest, usage: usage) + + function = Arel::Nodes::NamedFunction.new('GREATEST', [ + arel_table[:last_value], + new_value.to_i + ]) + + next_iid = update_record!(subject, scope, usage, function) + return next_iid if next_iid + + create_record!(subject, scope, usage, init) do |object| + object.last_value = [object.last_value, new_value].max + end + rescue ActiveRecord::RecordNotUnique + retry + end + + private + + def update_record!(subject, scope, usage, new_value) + stmt = Arel::UpdateManager.new + stmt.table(arel_table) + stmt.set(arel_table[:last_value] => new_value) + stmt.wheres = InternalId.filter_by(scope, usage).arel.constraints + + ActiveRecord::Base.connection.insert(stmt, 'Update InternalId', 'last_value') + end + + def create_record!(subject, scope, usage, init) + raise ArgumentError, 'Cannot initialize without init!' unless init + + instance = subject.is_a?(::Class) ? nil : subject + + subject.transaction(requires_new: true) do + last_value = init.call(instance, scope) || 0 + + internal_id = InternalId.create!(**scope, usage: usage, last_value: last_value) do |subject| + yield subject if block_given? + end + + internal_id.last_value + end + end + + def arel_table + InternalId.arel_table end end end |