From a0abb904782970de456dae5539ad5de2afef0e05 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Tue, 6 Mar 2018 20:06:27 +0100
Subject: Deprecate InternalId concern and rename.

---
 app/models/concerns/internal_id.rb           | 22 ----------------------
 app/models/concerns/nonatomic_internal_id.rb | 22 ++++++++++++++++++++++
 app/models/deployment.rb                     |  2 +-
 app/models/issue.rb                          |  2 +-
 app/models/merge_request.rb                  |  2 +-
 app/models/milestone.rb                      |  2 +-
 spec/models/merge_request_spec.rb            |  2 +-
 7 files changed, 27 insertions(+), 27 deletions(-)
 delete mode 100644 app/models/concerns/internal_id.rb
 create mode 100644 app/models/concerns/nonatomic_internal_id.rb

diff --git a/app/models/concerns/internal_id.rb b/app/models/concerns/internal_id.rb
deleted file mode 100644
index 01079fb8bd6..00000000000
--- a/app/models/concerns/internal_id.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-module InternalId
-  extend ActiveSupport::Concern
-
-  included do
-    validate :set_iid, on: :create
-    validates :iid, presence: true, numericality: true
-  end
-
-  def set_iid
-    if iid.blank?
-      parent = project || group
-      records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend
-      max_iid = records.maximum(:iid)
-
-      self.iid = max_iid.to_i + 1
-    end
-  end
-
-  def to_param
-    iid.to_s
-  end
-end
diff --git a/app/models/concerns/nonatomic_internal_id.rb b/app/models/concerns/nonatomic_internal_id.rb
new file mode 100644
index 00000000000..9d0c9b8512f
--- /dev/null
+++ b/app/models/concerns/nonatomic_internal_id.rb
@@ -0,0 +1,22 @@
+module NonatomicInternalId
+  extend ActiveSupport::Concern
+
+  included do
+    validate :set_iid, on: :create
+    validates :iid, presence: true, numericality: true
+  end
+
+  def set_iid
+    if iid.blank?
+      parent = project || group
+      records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend
+      max_iid = records.maximum(:iid)
+
+      self.iid = max_iid.to_i + 1
+    end
+  end
+
+  def to_param
+    iid.to_s
+  end
+end
diff --git a/app/models/deployment.rb b/app/models/deployment.rb
index 66e61c06765..e18ea8bfea4 100644
--- a/app/models/deployment.rb
+++ b/app/models/deployment.rb
@@ -1,5 +1,5 @@
 class Deployment < ActiveRecord::Base
-  include InternalId
+  include NonatomicInternalId
 
   belongs_to :project, required: true
   belongs_to :environment, required: true
diff --git a/app/models/issue.rb b/app/models/issue.rb
index c81f7e52bb1..cca6224d65c 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -1,7 +1,7 @@
 require 'carrierwave/orm/activerecord'
 
 class Issue < ActiveRecord::Base
-  include InternalId
+  include NonatomicInternalId
   include Issuable
   include Noteable
   include Referable
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 149ef7ec429..7e6d89ec9c7 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -1,5 +1,5 @@
 class MergeRequest < ActiveRecord::Base
-  include InternalId
+  include NonatomicInternalId
   include Issuable
   include Noteable
   include Referable
diff --git a/app/models/milestone.rb b/app/models/milestone.rb
index 77c19380e66..e7d397f40f5 100644
--- a/app/models/milestone.rb
+++ b/app/models/milestone.rb
@@ -8,7 +8,7 @@ class Milestone < ActiveRecord::Base
   Started = MilestoneStruct.new('Started', '#started', -3)
 
   include CacheMarkdownField
-  include InternalId
+  include NonatomicInternalId
   include Sortable
   include Referable
   include StripAttribute
diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb
index 4e783acbd8b..ff5a6f63010 100644
--- a/spec/models/merge_request_spec.rb
+++ b/spec/models/merge_request_spec.rb
@@ -17,7 +17,7 @@ describe MergeRequest do
   describe 'modules' do
     subject { described_class }
 
-    it { is_expected.to include_module(InternalId) }
+    it { is_expected.to include_module(NonatomicInternalId) }
     it { is_expected.to include_module(Issuable) }
     it { is_expected.to include_module(Referable) }
     it { is_expected.to include_module(Sortable) }
-- 
cgit v1.2.1


From 754272e392c0da088200a1b56156600973f63267 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Tue, 6 Mar 2018 20:09:01 +0100
Subject: Atomic generation of internal ids for issues.

---
 app/models/concerns/atomic_internal_id.rb          | 19 +++++
 app/models/internal_id.rb                          | 84 +++++++++++++++++++++
 app/models/issue.rb                                |  2 +-
 app/models/project.rb                              |  2 +
 .../20180305095250_create_internal_ids_table.rb    | 35 +++++++++
 db/schema.rb                                       |  9 +++
 spec/factories/internal_ids.rb                     |  6 ++
 spec/models/internal_id_spec.rb                    | 87 ++++++++++++++++++++++
 8 files changed, 243 insertions(+), 1 deletion(-)
 create mode 100644 app/models/concerns/atomic_internal_id.rb
 create mode 100644 app/models/internal_id.rb
 create mode 100644 db/migrate/20180305095250_create_internal_ids_table.rb
 create mode 100644 spec/factories/internal_ids.rb
 create mode 100644 spec/models/internal_id_spec.rb

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
new file mode 100644
index 00000000000..3cc9ce7f03f
--- /dev/null
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -0,0 +1,19 @@
+module AtomicInternalId
+  extend ActiveSupport::Concern
+
+  included do
+    before_validation(on: :create) do
+      set_iid
+    end
+
+    validates :iid, presence: true, numericality: true
+  end
+
+  def set_iid
+    self.iid = InternalId.generate_next(self.project, :issues) if iid.blank?
+  end
+
+  def to_param
+    iid.to_s
+  end
+end
diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb
new file mode 100644
index 00000000000..24c7cbf988f
--- /dev/null
+++ b/app/models/internal_id.rb
@@ -0,0 +1,84 @@
+# An InternalId is a strictly monotone sequence of integers
+# for a given project and usage (e.g. issues).
+#
+# For possible usages, see InternalId#usage enum.
+class InternalId < ActiveRecord::Base
+  belongs_to :project
+
+  enum usage: { issues: 0 }
+
+  validates :usage, presence: true
+  validates :project_id, presence: true
+
+  # Increments #last_value and saves the record
+  #
+  # The operation locks the record and gathers
+  # a `ROW SHARE` lock (in PostgreSQL). As such,
+  # the increment is atomic and safe to be called
+  # concurrently.
+  def increment_and_save!
+    lock!
+    self.last_value = (last_value || 0) + 1
+    save!
+    last_value
+  end
+
+  before_create :calculate_last_value!
+
+  # Calculate #last_value by counting the number of
+  # existing records for this usage.
+  def calculate_last_value!
+    return if last_value
+
+    parent = project # ??|| group
+    self.last_value = parent.send(usage.to_sym).maximum(:iid) || 0 # rubocop:disable GitlabSecurity/PublicSend
+  end
+
+  class << self
+    # Generate next internal id for a given project and usage.
+    #
+    # For currently supported usages, see #usage enum.
+    #
+    # The method implements a locking scheme that has the following properties:
+    # 1) Generated sequence of internal ids is unique per (project, usage)
+    # 2) The method is thread-safe and may be used in concurrent threads/processes.
+    # 3) The generated sequence is gapless.
+    # 4) In the absence of a record in the internal_ids table, one will be created
+    #    and last_value will be calculated on the fly.
+    def generate_next(project, usage)
+      raise 'project not set - this is required' unless project
+
+      project.transaction do
+        # Create a record in internal_ids if one does not yet exist
+        id = (lookup(project, usage) || create_record(project, usage))
+
+        # This will lock the InternalId record with ROW SHARE
+        # and increment #last_value
+        id.increment_and_save!
+      end
+    end
+
+    private
+
+    # Retrieve InternalId record for (project, usage) combination, if it exists
+    def lookup(project, usage)
+      project.internal_ids.find_by(usage: usages[usage.to_s])
+    end
+
+    # Create InternalId record for (project, usage) combination, if it doesn't exist
+    #
+    # We blindly insert without any synchronization. If another process
+    # was faster in doing this, we'll realize once we hit the unique key constraint
+    # violation. We can safely roll-back the nested transaction and perform
+    # a lookup instead to retrieve the record.
+    def create_record(project, usage)
+      begin
+        project.transaction(requires_new: true) do
+          create!(project: project, usage: usages[usage.to_s])
+        end
+      rescue ActiveRecord::RecordNotUnique
+        lookup(project, usage)
+      end
+    end
+  end
+end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index cca6224d65c..cf547106122 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -1,7 +1,7 @@
 require 'carrierwave/orm/activerecord'
 
 class Issue < ActiveRecord::Base
-  include NonatomicInternalId
+  include AtomicInternalId
   include Issuable
   include Noteable
   include Referable
diff --git a/app/models/project.rb b/app/models/project.rb
index a291ad7eed5..93b0da5cce1 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -188,6 +188,8 @@ class Project < ActiveRecord::Base
   has_many :todos
   has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :internal_ids
+
   has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true
   has_one :project_feature, inverse_of: :project
   has_one :statistics, class_name: 'ProjectStatistics'
diff --git a/db/migrate/20180305095250_create_internal_ids_table.rb b/db/migrate/20180305095250_create_internal_ids_table.rb
new file mode 100644
index 00000000000..19c1904bb43
--- /dev/null
+++ b/db/migrate/20180305095250_create_internal_ids_table.rb
@@ -0,0 +1,35 @@
+class CreateInternalIdsTable < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :internal_ids do |t|
+      t.references :project
+      t.integer :usage, null: false
+      t.integer :last_value, null: false
+    end
+
+    unless index_exists?(:internal_ids, [:usage, :project_id])
+      add_index :internal_ids, [:usage, :project_id], unique: true
+    end
+
+    unless foreign_key_exists?(:internal_ids, :project_id)
+      add_concurrent_foreign_key :internal_ids, :projects, column: :project_id, on_delete: :cascade
+    end
+  end
+
+  def down
+    drop_table :internal_ids
+  end
+
+  private
+
+  def foreign_key_exists?(table, column)
+    foreign_keys(table).any? do |key|
+      key.options[:column] == column.to_s
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ab4370e2754..3785bf14d5c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -866,6 +866,14 @@ ActiveRecord::Schema.define(version: 20180309160427) do
 
   add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree
 
+  create_table "internal_ids", force: :cascade do |t|
+    t.integer "project_id"
+    t.integer "usage", null: false
+    t.integer "last_value", null: false
+  end
+
+  add_index "internal_ids", ["usage", "project_id"], name: "index_internal_ids_on_usage_and_project_id", unique: true, using: :btree
+
   create_table "issue_assignees", id: false, force: :cascade do |t|
     t.integer "user_id", null: false
     t.integer "issue_id", null: false
@@ -2058,6 +2066,7 @@ ActiveRecord::Schema.define(version: 20180309160427) do
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
   add_foreign_key "gpg_signatures", "projects", on_delete: :cascade
   add_foreign_key "group_custom_attributes", "namespaces", column: "group_id", on_delete: :cascade
+  add_foreign_key "internal_ids", "projects", name: "fk_f7d46b66c6", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade
   add_foreign_key "issue_assignees", "users", name: "fk_5e0c8d9154", on_delete: :cascade
   add_foreign_key "issue_metrics", "issues", on_delete: :cascade
diff --git a/spec/factories/internal_ids.rb b/spec/factories/internal_ids.rb
new file mode 100644
index 00000000000..b4c14d22a29
--- /dev/null
+++ b/spec/factories/internal_ids.rb
@@ -0,0 +1,6 @@
+FactoryBot.define do
+  factory :internal_id do
+    project
+    usage { InternalId.usages.keys.first }
+  end
+end
diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb
new file mode 100644
index 00000000000..b953b6a2df8
--- /dev/null
+++ b/spec/models/internal_id_spec.rb
@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+describe InternalId do
+  let(:project) { create(:project) }
+  let(:usage) { :issues }
+
+  context 'validations' do
+    it { is_expected.to validate_presence_of(:usage) }
+    it { is_expected.to validate_presence_of(:project_id) }
+  end
+
+  describe '.generate_next' do
+    context 'in the absence of a record' do
+      subject { described_class.generate_next(project, usage) }
+
+      it 'creates a record if not yet present' do
+        expect { subject }.to change { described_class.count }.from(0).to(1)
+      end
+
+      it 'stores record attributes' do
+        subject
+
+        described_class.first.tap do |record|
+          expect(record.project).to eq(project)
+          expect(record.usage).to eq(usage.to_s) # TODO
+        end
+      end
+
+      context 'with existing issues' do
+        before do
+          rand(10).times { create(:issue, project: project) }
+        end
+
+        it 'calculates last_value values automatically' do
+          expect(subject).to eq(project.issues.size + 1)
+        end
+      end
+    end
+
+    it 'generates a strictly monotone, gapless sequence' do
+      seq = (0..rand(1000)).map do
+        described_class.generate_next(project, usage)
+      end
+      normalized = seq.map { |i| i - seq.min }
+      expect(normalized).to eq((0..seq.size - 1).to_a)
+    end
+  end
+
+  describe '#increment_and_save!' do
+    let(:id) { create(:internal_id) }
+    subject { id.increment_and_save! }
+
+    it 'returns incremented iid' do
+      value = id.last_value
+      expect(subject).to eq(value + 1)
+    end
+
+    it 'saves the record' do
+      subject
+      expect(id.changed?).to be_falsey
+    end
+
+    context 'with last_value=nil' do
+      let(:id) { build(:internal_id, last_value: nil) }
+
+      it 'returns 1' do
+        expect(subject).to eq(1)
+      end
+    end
+  end
+
+  describe '#calculate_last_value! (for issues)' do
+    subject do
+      build(:internal_id, project: project, usage: :issues)
+    end
+
+    context 'with existing issues' do
+      before do
+        rand(10).times { create(:issue, project: project) }
+      end
+
+      it 'counts related issues and saves' do
+        expect { subject.calculate_last_value! }.to change { subject.last_value }.from(nil).to(project.issues.size)
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From 0360b0928aada1db7635e6c6bc40f0f571b72c30 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Mon, 12 Mar 2018 15:38:56 +0100
Subject: More flexible way of internal id generation.

---
 app/models/concerns/atomic_internal_id.rb | 20 +++++++++-------
 app/models/internal_id.rb                 | 40 +++++++++++++++----------------
 app/models/issue.rb                       |  2 ++
 spec/factories/internal_ids.rb            |  3 ++-
 spec/models/internal_id_spec.rb           | 26 ++++++--------------
 5 files changed, 42 insertions(+), 49 deletions(-)

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index 3cc9ce7f03f..eef5c0bfcd1 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -2,15 +2,19 @@ module AtomicInternalId
   extend ActiveSupport::Concern
 
   included do
-    before_validation(on: :create) do
-      set_iid
-    end
-
-    validates :iid, presence: true, numericality: true
-  end
+    class << self
+      def has_internal_id(on, scope:, usage: nil, init: nil)
+        before_validation(on: :create) do
+          if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
+            usage = (usage || self.class.name.tableize).to_sym
+            new_iid = InternalId.generate_next(self, scope, usage, init)
+            self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
+          end
+        end
 
-  def set_iid
-    self.iid = InternalId.generate_next(self.project, :issues) if iid.blank?
+        validates on, presence: true, numericality: true
+      end
+    end
   end
 
   def to_param
diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb
index 24c7cbf988f..58e71b623d0 100644
--- a/app/models/internal_id.rb
+++ b/app/models/internal_id.rb
@@ -23,17 +23,6 @@ class InternalId < ActiveRecord::Base
     last_value
   end
 
-  before_create :calculate_last_value!
-
-  # Calculate #last_value by counting the number of
-  # existing records for this usage.
-  def calculate_last_value!
-    return if last_value
-
-    parent = project # ??|| group
-    self.last_value = parent.send(usage.to_sym).maximum(:iid) || 0 # rubocop:disable GitlabSecurity/PublicSend
-  end
-
   class << self
     # Generate next internal id for a given project and usage.
     #
@@ -45,12 +34,21 @@ class InternalId < ActiveRecord::Base
     # 3) The generated sequence is gapless.
     # 4) In the absence of a record in the internal_ids table, one will be created
     #    and last_value will be calculated on the fly.
-    def generate_next(project, usage)
-      raise 'project not set - this is required' unless project
+    def generate_next(subject, scope, usage, init)
+      scope = [scope].flatten.compact
+      raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
+      raise "usage #{usage} is unknown. Supported values are InternalId.usages = #{InternalId.usages.keys.to_s}" unless InternalId.usages.include?(usage.to_sym)
+
+      init ||= ->(s) { 0 }
+
+      scope_attrs = scope.inject({}) do |h, e|
+        h[e] = subject.public_send(e)
+        h
+      end
 
-      project.transaction do
+      transaction do
         # Create a record in internal_ids if one does not yet exist
-        id = (lookup(project, usage) || create_record(project, usage))
+        id = (lookup(scope_attrs, usage) || create_record(scope_attrs, usage, init, subject))
 
         # This will lock the InternalId record with ROW SHARE
         # and increment #last_value
@@ -61,8 +59,8 @@ class InternalId < ActiveRecord::Base
     private
 
     # Retrieve InternalId record for (project, usage) combination, if it exists
-    def lookup(project, usage)
-      project.internal_ids.find_by(usage: usages[usage.to_s])
+    def lookup(scope_attrs, usage)
+      InternalId.find_by(usage: usages[usage.to_s], **scope_attrs)
     end
 
     # Create InternalId record for (project, usage) combination, if it doesn't exist
@@ -71,13 +69,13 @@ class InternalId < ActiveRecord::Base
     # was faster in doing this, we'll realize once we hit the unique key constraint
     # violation. We can safely roll-back the nested transaction and perform
     # a lookup instead to retrieve the record.
-    def create_record(project, usage)
+    def create_record(scope_attrs, usage, init, subject)
       begin
-        project.transaction(requires_new: true) do
-          create!(project: project, usage: usages[usage.to_s])
+        transaction(requires_new: true) do
+          create!(usage: usages[usage.to_s], **scope_attrs, last_value: init.call(subject) || 0)
         end
       rescue ActiveRecord::RecordNotUnique
-        lookup(project, usage)
+        lookup(scope_attrs, usage)
       end
     end
   end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index cf547106122..509e214c601 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -24,6 +24,8 @@ class Issue < ActiveRecord::Base
   belongs_to :project
   belongs_to :moved_to, class_name: 'Issue'
 
+  has_internal_id :iid, scope: :project, init: ->(s) { s.project.issues.maximum(:iid) }
+
   has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   has_many :merge_requests_closing_issues,
diff --git a/spec/factories/internal_ids.rb b/spec/factories/internal_ids.rb
index b4c14d22a29..fbde07a391a 100644
--- a/spec/factories/internal_ids.rb
+++ b/spec/factories/internal_ids.rb
@@ -1,6 +1,7 @@
 FactoryBot.define do
   factory :internal_id do
     project
-    usage { InternalId.usages.keys.first }
+    usage :issues
+    last_value { project.issues.maximum(:iid) || 0 }
   end
 end
diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb
index b953b6a2df8..5971d8f47a6 100644
--- a/spec/models/internal_id_spec.rb
+++ b/spec/models/internal_id_spec.rb
@@ -3,6 +3,9 @@ require 'spec_helper'
 describe InternalId do
   let(:project) { create(:project) }
   let(:usage) { :issues }
+  let(:issue) { build(:issue, project: project) }
+  let(:scope) { :project }
+  let(:init) { ->(s) { project.issues.size } }
 
   context 'validations' do
     it { is_expected.to validate_presence_of(:usage) }
@@ -11,7 +14,7 @@ describe InternalId do
 
   describe '.generate_next' do
     context 'in the absence of a record' do
-      subject { described_class.generate_next(project, usage) }
+      subject { described_class.generate_next(issue, scope, usage, init) }
 
       it 'creates a record if not yet present' do
         expect { subject }.to change { described_class.count }.from(0).to(1)
@@ -22,13 +25,14 @@ describe InternalId do
 
         described_class.first.tap do |record|
           expect(record.project).to eq(project)
-          expect(record.usage).to eq(usage.to_s) # TODO
+          expect(record.usage).to eq(usage.to_s)
         end
       end
 
       context 'with existing issues' do
         before do
           rand(10).times { create(:issue, project: project) }
+          InternalId.delete_all
         end
 
         it 'calculates last_value values automatically' do
@@ -39,7 +43,7 @@ describe InternalId do
 
     it 'generates a strictly monotone, gapless sequence' do
       seq = (0..rand(1000)).map do
-        described_class.generate_next(project, usage)
+        described_class.generate_next(issue, scope, usage, init)
       end
       normalized = seq.map { |i| i - seq.min }
       expect(normalized).to eq((0..seq.size - 1).to_a)
@@ -68,20 +72,4 @@ describe InternalId do
       end
     end
   end
-
-  describe '#calculate_last_value! (for issues)' do
-    subject do
-      build(:internal_id, project: project, usage: :issues)
-    end
-
-    context 'with existing issues' do
-      before do
-        rand(10).times { create(:issue, project: project) }
-      end
-
-      it 'counts related issues and saves' do
-        expect { subject.calculate_last_value! }.to change { subject.last_value }.from(nil).to(project.issues.size)
-      end
-    end
-  end
 end
-- 
cgit v1.2.1


From 3fa2eb4e10885c9b3a01f0f9f244c9fbb26cf6a1 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Mon, 12 Mar 2018 16:51:38 +0100
Subject: Refactor, extract class and improve comments.

---
 app/models/concerns/atomic_internal_id.rb | 32 ++++++++++-
 app/models/internal_id.rb                 | 93 ++++++++++++++++++++-----------
 app/models/issue.rb                       |  2 +-
 spec/models/concerns/issuable_spec.rb     |  2 +-
 spec/models/internal_id_spec.rb           |  9 ++-
 5 files changed, 95 insertions(+), 43 deletions(-)

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index eef5c0bfcd1..6a0f29806c4 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -1,13 +1,41 @@
+# Include atomic internal id generation scheme for a model
+#
+# This allows to atomically generate internal ids that are
+# unique within a given scope.
+#
+# For example, let's generate internal ids for Issue per Project:
+# ```
+# class Issue < ActiveRecord::Base
+#   has_internal_id :iid, scope: :project, init: ->(s) { s.project.issues.maximum(:iid) }
+# end
+# ```
+#
+# This generates unique internal ids per project for newly created issues.
+# The generated internal id is saved in the `iid` attribute of `Issue`.
+#
+# This concern uses InternalId records to facilitate atomicity.
+# In the absence of a record for the given scope, one will be created automatically.
+# In this situation, the `init` block is called to calculate the initial value.
+# In the example above, we calculate the maximum `iid` of all issues
+# within the given project.
+#
+# Note that a model may have more than one internal id associated with possibly
+# different scopes.
 module AtomicInternalId
   extend ActiveSupport::Concern
 
   included do
     class << self
-      def has_internal_id(on, scope:, usage: nil, init: nil)
+      def has_internal_id(on, scope:, usage: nil, init: nil) # rubocop:disable Naming/PredicateName
         before_validation(on: :create) do
           if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
+
+            scope_attrs = [scope].flatten.compact.each_with_object({}) do |e, h|
+              h[e] = self.public_send(e) # rubocop:disable GitlabSecurity/PublicSend
+            end
+
             usage = (usage || self.class.name.tableize).to_sym
-            new_iid = InternalId.generate_next(self, scope, usage, init)
+            new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
             self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
           end
         end
diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb
index 58e71b623d0..4673dffa842 100644
--- a/app/models/internal_id.rb
+++ b/app/models/internal_id.rb
@@ -1,21 +1,26 @@
 # An InternalId is a strictly monotone sequence of integers
-# for a given project and usage (e.g. issues).
+# generated for a given scope and usage.
 #
-# For possible usages, see InternalId#usage enum.
+# For example, issues use their project to scope internal ids:
+# In that sense, scope is "project" and usage is "issues".
+# Generated internal ids for an issue are unique per project.
+#
+# See InternalId#usage enum for available usages.
+#
+# In order to leverage InternalId for other usages, the idea is to
+# * Add `usage` value to enum
+# * (Optionally) add columns to `internal_ids` if needed for scope.
 class InternalId < ActiveRecord::Base
   belongs_to :project
 
   enum usage: { issues: 0 }
 
   validates :usage, presence: true
-  validates :project_id, presence: true
 
   # Increments #last_value and saves the record
   #
-  # The operation locks the record and gathers
-  # a `ROW SHARE` lock (in PostgreSQL). As such,
-  # the increment is atomic and safe to be called
-  # concurrently.
+  # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL).
+  # As such, the increment is atomic and safe to be called concurrently.
   def increment_and_save!
     lock!
     self.last_value = (last_value || 0) + 1
@@ -24,59 +29,79 @@ class InternalId < ActiveRecord::Base
   end
 
   class << self
-    # Generate next internal id for a given project and usage.
+    def generate_next(subject, scope, usage, init)
+      InternalIdGenerator.new(subject, scope, usage, init).generate
+    end
+  end
+
+  class InternalIdGenerator
+    # Generate next internal id for a given scope and usage.
     #
     # For currently supported usages, see #usage enum.
     #
     # The method implements a locking scheme that has the following properties:
-    # 1) Generated sequence of internal ids is unique per (project, usage)
+    # 1) Generated sequence of internal ids is unique per (scope and usage)
     # 2) The method is thread-safe and may be used in concurrent threads/processes.
     # 3) The generated sequence is gapless.
     # 4) In the absence of a record in the internal_ids table, one will be created
     #    and last_value will be calculated on the fly.
-    def generate_next(subject, scope, usage, init)
-      scope = [scope].flatten.compact
-      raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
-      raise "usage #{usage} is unknown. Supported values are InternalId.usages = #{InternalId.usages.keys.to_s}" unless InternalId.usages.include?(usage.to_sym)
+    #
+    # subject: The instance we're generating an internal id for. Gets passed to init if called.
+    # scope: Attributes that define the scope for id generation.
+    # usage: Symbol to define the usage of the internal id, see InternalId.usages
+    # init: Block that gets called to initialize InternalId record if not yet present (optional)
+    attr_reader :subject, :scope, :init, :scope_attrs, :usage
+    def initialize(subject, scope, usage, init)
+      @subject = subject
+      @scope = scope
+      @init = init || ->(s) { 0 }
+      @usage = usage
 
-      init ||= ->(s) { 0 }
+      raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
 
-      scope_attrs = scope.inject({}) do |h, e|
-        h[e] = subject.public_send(e)
-        h
+      unless InternalId.usages.keys.include?(usage.to_s)
+        raise "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages"
       end
+    end
 
-      transaction do
+    # Generates next internal id and returns it
+    def generate
+      subject.transaction do
         # Create a record in internal_ids if one does not yet exist
-        id = (lookup(scope_attrs, usage) || create_record(scope_attrs, usage, init, subject))
-
-        # This will lock the InternalId record with ROW SHARE
-        # and increment #last_value
-        id.increment_and_save!
+        # and increment it's last value
+        #
+        # Note this will acquire a ROW SHARE lock on the InternalId record
+        (lookup || create_record).increment_and_save!
       end
     end
 
     private
 
     # Retrieve InternalId record for (project, usage) combination, if it exists
-    def lookup(scope_attrs, usage)
-      InternalId.find_by(usage: usages[usage.to_s], **scope_attrs)
+    def lookup
+      InternalId.find_by(**scope, usage: usage_value)
+    end
+
+    def usage_value
+      @usage_value ||= InternalId.usages[usage.to_s]
     end
 
-    # Create InternalId record for (project, usage) combination, if it doesn't exist
+    # Create InternalId record for (scope, usage) combination, if it doesn't exist
     #
-    # We blindly insert without any synchronization. If another process
+    # We blindly insert without synchronization. If another process
     # was faster in doing this, we'll realize once we hit the unique key constraint
     # violation. We can safely roll-back the nested transaction and perform
     # a lookup instead to retrieve the record.
-    def create_record(scope_attrs, usage, init, subject)
-      begin
-        transaction(requires_new: true) do
-          create!(usage: usages[usage.to_s], **scope_attrs, last_value: init.call(subject) || 0)
-        end
-      rescue ActiveRecord::RecordNotUnique
-        lookup(scope_attrs, usage)
+    def create_record
+      subject.transaction(requires_new: true) do
+        InternalId.create!(
+          **scope,
+          usage: usage_value,
+          last_value: init.call(subject) || 0
+        )
       end
+    rescue ActiveRecord::RecordNotUnique
+      lookup
     end
   end
 end
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 509e214c601..7bfc45c1f43 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -24,7 +24,7 @@ class Issue < ActiveRecord::Base
   belongs_to :project
   belongs_to :moved_to, class_name: 'Issue'
 
-  has_internal_id :iid, scope: :project, init: ->(s) { s.project.issues.maximum(:iid) }
+  has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.issues&.maximum(:iid) }
 
   has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
diff --git a/spec/models/concerns/issuable_spec.rb b/spec/models/concerns/issuable_spec.rb
index 4b217df2e8f..f8874d14e3f 100644
--- a/spec/models/concerns/issuable_spec.rb
+++ b/spec/models/concerns/issuable_spec.rb
@@ -34,7 +34,7 @@ describe Issuable do
     subject { build(:issue) }
 
     before do
-      allow(subject).to receive(:set_iid).and_return(false)
+      allow(InternalId).to receive(:generate_next).and_return(nil)
     end
 
     it { is_expected.to validate_presence_of(:project) }
diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb
index 5971d8f47a6..6d5a12c9d06 100644
--- a/spec/models/internal_id_spec.rb
+++ b/spec/models/internal_id_spec.rb
@@ -4,12 +4,11 @@ describe InternalId do
   let(:project) { create(:project) }
   let(:usage) { :issues }
   let(:issue) { build(:issue, project: project) }
-  let(:scope) { :project }
-  let(:init) { ->(s) { project.issues.size } }
+  let(:scope) { { project: project } }
+  let(:init) { ->(s) { s.project.issues.size } }
 
   context 'validations' do
     it { is_expected.to validate_presence_of(:usage) }
-    it { is_expected.to validate_presence_of(:project_id) }
   end
 
   describe '.generate_next' do
@@ -31,8 +30,8 @@ describe InternalId do
 
       context 'with existing issues' do
         before do
-          rand(10).times { create(:issue, project: project) }
-          InternalId.delete_all
+          rand(1..10).times { create(:issue, project: project) }
+          described_class.delete_all
         end
 
         it 'calculates last_value values automatically' do
-- 
cgit v1.2.1


From b1c7ee37ea85dfe4a4fa99342dc09d8ea436769e Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Tue, 6 Jun 2017 16:37:41 -0300
Subject: Add bigserial as a native database type for MySQL adapter

---
 config/initializers/ar_native_database_types.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 config/initializers/ar_native_database_types.rb

diff --git a/config/initializers/ar_native_database_types.rb b/config/initializers/ar_native_database_types.rb
new file mode 100644
index 00000000000..d7b4b348957
--- /dev/null
+++ b/config/initializers/ar_native_database_types.rb
@@ -0,0 +1,11 @@
+require 'active_record/connection_adapters/abstract_mysql_adapter'
+
+module ActiveRecord
+  module ConnectionAdapters
+    class AbstractMysqlAdapter
+      NATIVE_DATABASE_TYPES.merge!(
+        bigserial:  { name: 'bigint(20) auto_increment PRIMARY KEY' }
+      )
+    end
+  end
+end
-- 
cgit v1.2.1


From 3a8207a96120dbb23e9feddd372e720ef0884378 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Mon, 12 Mar 2018 18:12:38 +0100
Subject: Use bigserial for internal_ids table.

---
 db/migrate/20180305095250_create_internal_ids_table.rb | 2 +-
 db/schema.rb                                           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/db/migrate/20180305095250_create_internal_ids_table.rb b/db/migrate/20180305095250_create_internal_ids_table.rb
index 19c1904bb43..d2e79a12c0a 100644
--- a/db/migrate/20180305095250_create_internal_ids_table.rb
+++ b/db/migrate/20180305095250_create_internal_ids_table.rb
@@ -6,7 +6,7 @@ class CreateInternalIdsTable < ActiveRecord::Migration
   disable_ddl_transaction!
 
   def up
-    create_table :internal_ids do |t|
+    create_table :internal_ids, id: :bigserial do |t|
       t.references :project
       t.integer :usage, null: false
       t.integer :last_value, null: false
diff --git a/db/schema.rb b/db/schema.rb
index 3785bf14d5c..aab8ad4753e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -866,7 +866,7 @@ ActiveRecord::Schema.define(version: 20180309160427) do
 
   add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree
 
-  create_table "internal_ids", force: :cascade do |t|
+  create_table "internal_ids", id: :bigserial, force: :cascade do |t|
     t.integer "project_id"
     t.integer "usage", null: false
     t.integer "last_value", null: false
-- 
cgit v1.2.1


From d4bb363f7c2747d15e496aee98bc7cc3fde77a77 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Mon, 12 Mar 2018 18:16:20 +0100
Subject: Add changelog.

Closes #31114.
---
 changelogs/unreleased/31114-internal-ids-are-not-atomic.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 changelogs/unreleased/31114-internal-ids-are-not-atomic.yml

diff --git a/changelogs/unreleased/31114-internal-ids-are-not-atomic.yml b/changelogs/unreleased/31114-internal-ids-are-not-atomic.yml
new file mode 100644
index 00000000000..bc1955bc66f
--- /dev/null
+++ b/changelogs/unreleased/31114-internal-ids-are-not-atomic.yml
@@ -0,0 +1,5 @@
+---
+title: Atomic generation of internal ids for issues.
+merge_request: 17580
+author:
+type: other
-- 
cgit v1.2.1


From d374d0be1af0e6cef4a150425cd73189f9960f54 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Tue, 13 Mar 2018 16:53:55 +0100
Subject: Backwards-compat for migration specs.

The specs are based on a schema version that doesn't know about
`internal_ids` table. However, the actual code being execute relies on
it.
---
 app/models/internal_id.rb       | 20 ++++++++++++++++++--
 spec/models/internal_id_spec.rb | 19 +++++++++++++++++--
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb
index 4673dffa842..6419cdc5b67 100644
--- a/app/models/internal_id.rb
+++ b/app/models/internal_id.rb
@@ -17,6 +17,8 @@ class InternalId < ActiveRecord::Base
 
   validates :usage, presence: true
 
+  REQUIRED_SCHEMA_VERSION = 20180305095250
+
   # Increments #last_value and saves the record
   #
   # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL).
@@ -30,8 +32,22 @@ class InternalId < ActiveRecord::Base
 
   class << self
     def generate_next(subject, scope, usage, init)
+      # Shortcut if `internal_ids` table is not available (yet)
+      # This can be the case in other (unrelated) migration specs
+      return (init.call(subject) || 0) + 1 unless available?
+
       InternalIdGenerator.new(subject, scope, usage, init).generate
     end
+
+    def available?
+      @available_flag ||= ActiveRecord::Migrator.current_version >= REQUIRED_SCHEMA_VERSION # rubocop:disable Gitlab/PredicateMemoization
+    end
+
+    # Flushes cached information about schema
+    def reset_column_information
+      @available_flag = nil
+      super
+    end
   end
 
   class InternalIdGenerator
@@ -49,12 +65,12 @@ class InternalId < ActiveRecord::Base
     # subject: The instance we're generating an internal id for. Gets passed to init if called.
     # scope: Attributes that define the scope for id generation.
     # usage: Symbol to define the usage of the internal id, see InternalId.usages
-    # init: Block that gets called to initialize InternalId record if not yet present (optional)
+    # init: Block that gets called to initialize InternalId record if not present
     attr_reader :subject, :scope, :init, :scope_attrs, :usage
     def initialize(subject, scope, usage, init)
       @subject = subject
       @scope = scope
-      @init = init || ->(s) { 0 }
+      @init = init
       @usage = usage
 
       raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb
index 6d5a12c9d06..ef6db2daa95 100644
--- a/spec/models/internal_id_spec.rb
+++ b/spec/models/internal_id_spec.rb
@@ -12,9 +12,9 @@ describe InternalId do
   end
 
   describe '.generate_next' do
-    context 'in the absence of a record' do
-      subject { described_class.generate_next(issue, scope, usage, init) }
+    subject { described_class.generate_next(issue, scope, usage, init) }
 
+    context 'in the absence of a record' do
       it 'creates a record if not yet present' do
         expect { subject }.to change { described_class.count }.from(0).to(1)
       end
@@ -47,6 +47,21 @@ describe InternalId do
       normalized = seq.map { |i| i - seq.min }
       expect(normalized).to eq((0..seq.size - 1).to_a)
     end
+
+    context 'with an insufficient schema version' do
+      before do
+        described_class.reset_column_information
+        expect(ActiveRecord::Migrator).to receive(:current_version).and_return(InternalId::REQUIRED_SCHEMA_VERSION - 1)
+      end
+
+      let(:init) { double('block') }
+
+      it 'calculates next internal ids on the fly' do
+        val = rand(1..100)
+        expect(init).to receive(:call).with(issue).and_return(val)
+        expect(subject).to eq(val + 1)
+      end
+    end
   end
 
   describe '#increment_and_save!' do
-- 
cgit v1.2.1


From 6568b4a98e25d88a99b7a499bc7965ae47bf4b19 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Tue, 13 Mar 2018 19:43:02 +0100
Subject: Add shared specs for AtomicInternalId concern.

---
 app/models/concerns/atomic_internal_id.rb          |  2 +-
 spec/models/issue_spec.rb                          |  8 ++++-
 .../models/atomic_internal_id_spec.rb              | 38 ++++++++++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)
 create mode 100644 spec/support/shared_examples/models/atomic_internal_id_spec.rb

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index 6a0f29806c4..bba71eee8bd 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -26,7 +26,7 @@ module AtomicInternalId
 
   included do
     class << self
-      def has_internal_id(on, scope:, usage: nil, init: nil) # rubocop:disable Naming/PredicateName
+      def has_internal_id(on, scope:, usage: nil, init:) # rubocop:disable Naming/PredicateName
         before_validation(on: :create) do
           if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
 
diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb
index feed7968f09..11154291368 100644
--- a/spec/models/issue_spec.rb
+++ b/spec/models/issue_spec.rb
@@ -9,11 +9,17 @@ describe Issue do
   describe 'modules' do
     subject { described_class }
 
-    it { is_expected.to include_module(InternalId) }
     it { is_expected.to include_module(Issuable) }
     it { is_expected.to include_module(Referable) }
     it { is_expected.to include_module(Sortable) }
     it { is_expected.to include_module(Taskable) }
+
+    it_behaves_like 'AtomicInternalId' do
+      let(:internal_id_attribute) { :iid }
+      let(:instance) { build(:issue) }
+      let(:scope_attrs) { { project: instance.project } }
+      let(:usage) { :issues }
+    end
   end
 
   subject { create(:issue) }
diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb
new file mode 100644
index 00000000000..671aa314bd6
--- /dev/null
+++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb
@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+shared_examples_for 'AtomicInternalId' do
+  describe '.has_internal_id' do
+    describe 'Module inclusion' do
+      subject { described_class }
+
+      it { is_expected.to include_module(AtomicInternalId) }
+    end
+
+    describe 'Validation' do
+      subject { instance }
+
+      before do
+        allow(InternalId).to receive(:generate_next).and_return(nil)
+      end
+
+      it { is_expected.to validate_presence_of(internal_id_attribute) }
+      it { is_expected.to validate_numericality_of(internal_id_attribute) }
+    end
+
+    describe 'internal id generation' do
+      subject { instance.save! }
+
+      it 'calls InternalId.generate_next and sets internal id attribute' do
+        iid = rand(1..1000)
+        expect(InternalId).to receive(:generate_next).with(instance, scope_attrs, usage, any_args).and_return(iid)
+        subject
+        expect(instance.public_send(internal_id_attribute)).to eq(iid) # rubocop:disable GitlabSecurity/PublicSend
+      end
+
+      it 'does not overwrite an existing internal id' do
+        instance.public_send("#{internal_id_attribute}=", 4711) # rubocop:disable GitlabSecurity/PublicSend
+        expect { subject }.not_to change { instance.public_send(internal_id_attribute) } # rubocop:disable GitlabSecurity/PublicSend
+      end
+    end
+  end
+end
-- 
cgit v1.2.1


From d9a953c847339c3d906be4540bdc2a3c44f13fca Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Tue, 13 Mar 2018 22:17:29 +0100
Subject: Add new model to import/export configuration.

---
 spec/lib/gitlab/import_export/all_models.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index bece82e531a..a204a8f1ffe 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -279,6 +279,7 @@ project:
 - lfs_file_locks
 - project_badges
 - source_of_merge_requests
+- internal_ids
 award_emoji:
 - awardable
 - user
-- 
cgit v1.2.1


From 539bdf73be4a84b1f542801a28f016808d604fe5 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Wed, 14 Mar 2018 13:42:03 +0100
Subject: Address review comments.

---
 app/models/concerns/atomic_internal_id.rb | 27 ++++++++++++---------------
 app/models/internal_id.rb                 |  5 +++--
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index bba71eee8bd..6f2d60a7ee9 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -24,24 +24,21 @@
 module AtomicInternalId
   extend ActiveSupport::Concern
 
-  included do
-    class << self
-      def has_internal_id(on, scope:, usage: nil, init:) # rubocop:disable Naming/PredicateName
-        before_validation(on: :create) do
-          if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
-
-            scope_attrs = [scope].flatten.compact.each_with_object({}) do |e, h|
-              h[e] = self.public_send(e) # rubocop:disable GitlabSecurity/PublicSend
-            end
-
-            usage = (usage || self.class.name.tableize).to_sym
-            new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
-            self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
+  module ClassMethods
+    def has_internal_id(on, scope:, usage: nil, init:) # rubocop:disable Naming/PredicateName
+      before_validation(on: :create) do
+        if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
+          scope_attrs = [scope].flatten.compact.each_with_object({}) do |e, h|
+            h[e] = self.public_send(e) # rubocop:disable GitlabSecurity/PublicSend
           end
-        end
 
-        validates on, presence: true, numericality: true
+          usage = (usage || self.class.table_name).to_sym
+          new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
+          self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
+        end
       end
+
+      validates on, presence: true, numericality: true
     end
   end
 
diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb
index 6419cdc5b67..f3630ed1ac5 100644
--- a/app/models/internal_id.rb
+++ b/app/models/internal_id.rb
@@ -67,16 +67,17 @@ class InternalId < ActiveRecord::Base
     # usage: Symbol to define the usage of the internal id, see InternalId.usages
     # init: Block that gets called to initialize InternalId record if not present
     attr_reader :subject, :scope, :init, :scope_attrs, :usage
+
     def initialize(subject, scope, usage, init)
       @subject = subject
       @scope = scope
       @init = init
       @usage = usage
 
-      raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
+      raise ArgumentError, 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
 
       unless InternalId.usages.keys.include?(usage.to_s)
-        raise "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages"
+        raise ArgumentError, "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages"
       end
     end
 
-- 
cgit v1.2.1


From 4c1e6fd0e83ceb550ec07448d6a37e76cc25e4fa Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Wed, 14 Mar 2018 13:42:24 +0100
Subject: Simplify migration and add NOT NULL to project_id.

---
 .../20180305095250_create_internal_ids_table.rb      | 20 ++------------------
 db/schema.rb                                         |  4 ++--
 2 files changed, 4 insertions(+), 20 deletions(-)

diff --git a/db/migrate/20180305095250_create_internal_ids_table.rb b/db/migrate/20180305095250_create_internal_ids_table.rb
index d2e79a12c0a..e972432fb98 100644
--- a/db/migrate/20180305095250_create_internal_ids_table.rb
+++ b/db/migrate/20180305095250_create_internal_ids_table.rb
@@ -3,33 +3,17 @@ class CreateInternalIdsTable < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  disable_ddl_transaction!
-
   def up
     create_table :internal_ids, id: :bigserial do |t|
-      t.references :project
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
       t.integer :usage, null: false
       t.integer :last_value, null: false
-    end
-
-    unless index_exists?(:internal_ids, [:usage, :project_id])
-      add_index :internal_ids, [:usage, :project_id], unique: true
-    end
 
-    unless foreign_key_exists?(:internal_ids, :project_id)
-      add_concurrent_foreign_key :internal_ids, :projects, column: :project_id, on_delete: :cascade
+      t.index [:usage, :project_id], unique: true
     end
   end
 
   def down
     drop_table :internal_ids
   end
-
-  private
-
-  def foreign_key_exists?(table, column)
-    foreign_keys(table).any? do |key|
-      key.options[:column] == column.to_s
-    end
-  end
 end
diff --git a/db/schema.rb b/db/schema.rb
index aab8ad4753e..3ff1a8754e2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -867,7 +867,7 @@ ActiveRecord::Schema.define(version: 20180309160427) do
   add_index "identities", ["user_id"], name: "index_identities_on_user_id", using: :btree
 
   create_table "internal_ids", id: :bigserial, force: :cascade do |t|
-    t.integer "project_id"
+    t.integer "project_id", null: false
     t.integer "usage", null: false
     t.integer "last_value", null: false
   end
@@ -2066,7 +2066,7 @@ ActiveRecord::Schema.define(version: 20180309160427) do
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
   add_foreign_key "gpg_signatures", "projects", on_delete: :cascade
   add_foreign_key "group_custom_attributes", "namespaces", column: "group_id", on_delete: :cascade
-  add_foreign_key "internal_ids", "projects", name: "fk_f7d46b66c6", on_delete: :cascade
+  add_foreign_key "internal_ids", "projects", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade
   add_foreign_key "issue_assignees", "users", name: "fk_5e0c8d9154", on_delete: :cascade
   add_foreign_key "issue_metrics", "issues", on_delete: :cascade
-- 
cgit v1.2.1


From bc3fc8ec3eec74876a0e2125248c27cde153e32b Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Wed, 14 Mar 2018 14:36:07 +0100
Subject: Only support single scope argument.

We can extend this later, but for now we don't have the use case.
---
 app/models/concerns/atomic_internal_id.rb | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index 6f2d60a7ee9..343edc237c9 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -28,11 +28,9 @@ module AtomicInternalId
     def has_internal_id(on, scope:, usage: nil, init:) # rubocop:disable Naming/PredicateName
       before_validation(on: :create) do
         if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
-          scope_attrs = [scope].flatten.compact.each_with_object({}) do |e, h|
-            h[e] = self.public_send(e) # rubocop:disable GitlabSecurity/PublicSend
-          end
-
+          scope_attrs = { scope => self.public_send(scope) } # rubocop:disable GitlabSecurity/PublicSend
           usage = (usage || self.class.table_name).to_sym
+
           new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
           self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
         end
-- 
cgit v1.2.1


From fb6d6fce5a4d0fd833dc1cd231dd284a6c89471a Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Fri, 16 Mar 2018 13:34:08 +0100
Subject: Address review comments.

---
 app/models/concerns/atomic_internal_id.rb                    | 12 ++++++------
 app/models/internal_id.rb                                    |  7 ++++---
 config/initializers/ar_native_database_types.rb              |  2 +-
 db/migrate/20180305095250_create_internal_ids_table.rb       |  6 +-----
 spec/models/internal_id_spec.rb                              |  6 +++++-
 .../shared_examples/models/atomic_internal_id_spec.rb        |  8 +++++---
 6 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index 343edc237c9..6895c7d7e95 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -1,6 +1,6 @@
 # Include atomic internal id generation scheme for a model
 #
-# This allows to atomically generate internal ids that are
+# This allows us to atomically generate internal ids that are
 # unique within a given scope.
 #
 # For example, let's generate internal ids for Issue per Project:
@@ -25,18 +25,18 @@ module AtomicInternalId
   extend ActiveSupport::Concern
 
   module ClassMethods
-    def has_internal_id(on, scope:, usage: nil, init:) # rubocop:disable Naming/PredicateName
+    def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName
       before_validation(on: :create) do
-        if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
+        if self.public_send(column).blank? # rubocop:disable GitlabSecurity/PublicSend
           scope_attrs = { scope => self.public_send(scope) } # rubocop:disable GitlabSecurity/PublicSend
-          usage = (usage || self.class.table_name).to_sym
+          usage = self.class.table_name.to_sym
 
           new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
-          self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
+          self.public_send("#{column}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
         end
       end
 
-      validates on, presence: true, numericality: true
+      validates column, presence: true, numericality: true
     end
   end
 
diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb
index f3630ed1ac5..cbec735c2dd 100644
--- a/app/models/internal_id.rb
+++ b/app/models/internal_id.rb
@@ -66,6 +66,7 @@ class InternalId < ActiveRecord::Base
     # scope: Attributes that define the scope for id generation.
     # usage: Symbol to define the usage of the internal id, see InternalId.usages
     # init: Block that gets called to initialize InternalId record if not present
+    #       Make sure to not throw exceptions in the absence of records (if this is expected).
     attr_reader :subject, :scope, :init, :scope_attrs, :usage
 
     def initialize(subject, scope, usage, init)
@@ -74,9 +75,9 @@ class InternalId < ActiveRecord::Base
       @init = init
       @usage = usage
 
-      raise ArgumentError, 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
+      raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
 
-      unless InternalId.usages.keys.include?(usage.to_s)
+      unless InternalId.usages.has_key?(usage.to_s)
         raise ArgumentError, "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages"
       end
     end
@@ -85,7 +86,7 @@ class InternalId < ActiveRecord::Base
     def generate
       subject.transaction do
         # Create a record in internal_ids if one does not yet exist
-        # and increment it's last value
+        # and increment its last value
         #
         # Note this will acquire a ROW SHARE lock on the InternalId record
         (lookup || create_record).increment_and_save!
diff --git a/config/initializers/ar_native_database_types.rb b/config/initializers/ar_native_database_types.rb
index d7b4b348957..3522b1db536 100644
--- a/config/initializers/ar_native_database_types.rb
+++ b/config/initializers/ar_native_database_types.rb
@@ -4,7 +4,7 @@ module ActiveRecord
   module ConnectionAdapters
     class AbstractMysqlAdapter
       NATIVE_DATABASE_TYPES.merge!(
-        bigserial:  { name: 'bigint(20) auto_increment PRIMARY KEY' }
+        bigserial: { name: 'bigint(20) auto_increment PRIMARY KEY' }
       )
     end
   end
diff --git a/db/migrate/20180305095250_create_internal_ids_table.rb b/db/migrate/20180305095250_create_internal_ids_table.rb
index e972432fb98..432086fe98b 100644
--- a/db/migrate/20180305095250_create_internal_ids_table.rb
+++ b/db/migrate/20180305095250_create_internal_ids_table.rb
@@ -3,7 +3,7 @@ class CreateInternalIdsTable < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  def up
+  def change
     create_table :internal_ids, id: :bigserial do |t|
       t.references :project, null: false, foreign_key: { on_delete: :cascade }
       t.integer :usage, null: false
@@ -12,8 +12,4 @@ class CreateInternalIdsTable < ActiveRecord::Migration
       t.index [:usage, :project_id], unique: true
     end
   end
-
-  def down
-    drop_table :internal_ids
-  end
 end
diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb
index ef6db2daa95..40d777c46cc 100644
--- a/spec/models/internal_id_spec.rb
+++ b/spec/models/internal_id_spec.rb
@@ -41,10 +41,11 @@ describe InternalId do
     end
 
     it 'generates a strictly monotone, gapless sequence' do
-      seq = (0..rand(1000)).map do
+      seq = (0..rand(100)).map do
         described_class.generate_next(issue, scope, usage, init)
       end
       normalized = seq.map { |i| i - seq.min }
+
       expect(normalized).to eq((0..seq.size - 1).to_a)
     end
 
@@ -58,6 +59,7 @@ describe InternalId do
 
       it 'calculates next internal ids on the fly' do
         val = rand(1..100)
+
         expect(init).to receive(:call).with(issue).and_return(val)
         expect(subject).to eq(val + 1)
       end
@@ -70,11 +72,13 @@ describe InternalId do
 
     it 'returns incremented iid' do
       value = id.last_value
+
       expect(subject).to eq(value + 1)
     end
 
     it 'saves the record' do
       subject
+
       expect(id.changed?).to be_falsey
     end
 
diff --git a/spec/support/shared_examples/models/atomic_internal_id_spec.rb b/spec/support/shared_examples/models/atomic_internal_id_spec.rb
index 671aa314bd6..144af4fc475 100644
--- a/spec/support/shared_examples/models/atomic_internal_id_spec.rb
+++ b/spec/support/shared_examples/models/atomic_internal_id_spec.rb
@@ -24,14 +24,16 @@ shared_examples_for 'AtomicInternalId' do
 
       it 'calls InternalId.generate_next and sets internal id attribute' do
         iid = rand(1..1000)
+
         expect(InternalId).to receive(:generate_next).with(instance, scope_attrs, usage, any_args).and_return(iid)
         subject
-        expect(instance.public_send(internal_id_attribute)).to eq(iid) # rubocop:disable GitlabSecurity/PublicSend
+        expect(instance.public_send(internal_id_attribute)).to eq(iid)
       end
 
       it 'does not overwrite an existing internal id' do
-        instance.public_send("#{internal_id_attribute}=", 4711) # rubocop:disable GitlabSecurity/PublicSend
-        expect { subject }.not_to change { instance.public_send(internal_id_attribute) } # rubocop:disable GitlabSecurity/PublicSend
+        instance.public_send("#{internal_id_attribute}=", 4711)
+
+        expect { subject }.not_to change { instance.public_send(internal_id_attribute) }
       end
     end
   end
-- 
cgit v1.2.1


From e7393191ee209beb1d39727384e4be21434415c6 Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Sun, 18 Mar 2018 17:00:41 +0100
Subject: Replace public_send calls.

---
 app/models/concerns/atomic_internal_id.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb
index 6895c7d7e95..4b66725a3e6 100644
--- a/app/models/concerns/atomic_internal_id.rb
+++ b/app/models/concerns/atomic_internal_id.rb
@@ -27,12 +27,12 @@ module AtomicInternalId
   module ClassMethods
     def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName
       before_validation(on: :create) do
-        if self.public_send(column).blank? # rubocop:disable GitlabSecurity/PublicSend
-          scope_attrs = { scope => self.public_send(scope) } # rubocop:disable GitlabSecurity/PublicSend
+        if read_attribute(column).blank?
+          scope_attrs = { scope => association(scope).reader }
           usage = self.class.table_name.to_sym
 
           new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
-          self.public_send("#{column}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
+          write_attribute(column, new_iid)
         end
       end
 
-- 
cgit v1.2.1


From c3b489bdcb1d6cf8bcdb935c73892dc5465eacbd Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Sun, 18 Mar 2018 17:13:09 +0100
Subject: Add spec for concurrent insert situation.

---
 spec/models/internal_id_spec.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/spec/models/internal_id_spec.rb b/spec/models/internal_id_spec.rb
index 40d777c46cc..581fd0293cc 100644
--- a/spec/models/internal_id_spec.rb
+++ b/spec/models/internal_id_spec.rb
@@ -38,6 +38,19 @@ describe InternalId do
           expect(subject).to eq(project.issues.size + 1)
         end
       end
+
+      context 'with concurrent inserts on table' do
+        it 'looks up the record if it was created concurrently' do
+          args = { **scope, usage: described_class.usages[usage.to_s] }
+          record = double
+          expect(described_class).to receive(:find_by).with(args).and_return(nil)    # first call, record not present
+          expect(described_class).to receive(:find_by).with(args).and_return(record) # second call, record was created by another process
+          expect(described_class).to receive(:create!).and_raise(ActiveRecord::RecordNotUnique, 'record not unique')
+          expect(record).to receive(:increment_and_save!)
+
+          subject
+        end
+      end
     end
 
     it 'generates a strictly monotone, gapless sequence' do
-- 
cgit v1.2.1