Start tracking shards in the database

6 files changed, 97 insertions, 0 deletions

diff --git a/app/models/shard.rb b/app/models/shard.rb
new file mode 100644
index 00000000000..2fa22bd040c
--- /dev/null
+++ b/app/models/shard.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Shard < ActiveRecord::Base
+  # Store shard names from the configuration file in the database. This is not a
+  # list of active shards - we just want to assign an immutable, unique ID to
+  # every shard name for easy indexing / referencing.
+  def self.populate!
+    return unless table_exists?
+
+    # The GitLab config does not change for the lifecycle of the process
+    in_config = Gitlab.config.repositories.storages.keys.map(&:to_s)
+
+    transaction do
+      in_db = all.pluck(:name)
+      missing = in_config - in_db
+
+      missing.map { |name| by_name(name) }
+    end
+  end
+
+  def self.by_name(name)
+    find_or_create_by(name: name)
+  rescue ActiveRecord::RecordNotUnique
+    retry
+  end
+end
diff --git a/config/initializers/fill_shards.rb b/config/initializers/fill_shards.rb
new file mode 100644
index 00000000000..d5071c7d556
--- /dev/null
+++ b/config/initializers/fill_shards.rb
@@ -0,0 +1,3 @@
+return if Gitlab::Database.read_only?
+
+Shard.populate!
diff --git a/db/migrate/20181019032400_add_shards_table.rb b/db/migrate/20181019032400_add_shards_table.rb
new file mode 100644
index 00000000000..5e0a6960548
--- /dev/null
+++ b/db/migrate/20181019032400_add_shards_table.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddShardsTable < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :shards do |t|
+      t.string :name, null: false, index: { unique: true }
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1a8b556228d..439f70b0198 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1931,6 +1931,12 @@ ActiveRecord::Schema.define(version: 20181101144347) do
   add_index "services", ["project_id"], name: "index_services_on_project_id", using: :btree
   add_index "services", ["template"], name: "index_services_on_template", using: :btree
 
+  create_table "shards", force: :cascade do |t|
+    t.string "name", null: false
+  end
+
+  add_index "shards", ["name"], name: "index_shards_on_name", unique: true, using: :btree
+
   create_table "site_statistics", force: :cascade do |t|
     t.integer "repositories_count", default: 0, null: false
   end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index d059854214f..84326724118 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -8,6 +8,7 @@ describe Project do
     it { is_expected.to belong_to(:group) }
     it { is_expected.to belong_to(:namespace) }
     it { is_expected.to belong_to(:creator).class_name('User') }
+    it { is_expected.to belong_to(:pool_repository) }
     it { is_expected.to have_many(:users) }
     it { is_expected.to have_many(:services) }
     it { is_expected.to have_many(:events) }
diff --git a/spec/models/shard_spec.rb b/spec/models/shard_spec.rb
new file mode 100644
index 00000000000..83104711b55
--- /dev/null
+++ b/spec/models/shard_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literals: true
+require 'spec_helper'
+
+describe Shard do
+  describe '.populate!' do
+    it 'creates shards based on the config file' do
+      expect(described_class.all).to be_empty
+
+      stub_storage_settings(foo: {}, bar: {}, baz: {})
+
+      described_class.populate!
+
+      expect(described_class.all.map(&:name)).to match_array(%w[default foo bar baz])
+    end
+  end
+
+  describe '.by_name' do
+    let(:default_shard) { described_class.find_by(name: 'default') }
+
+    before do
+      described_class.populate!
+    end
+
+    it 'returns an existing shard' do
+      expect(described_class.by_name('default')).to eq(default_shard)
+    end
+
+    it 'creates a new shard' do
+      result = described_class.by_name('foo')
+
+      expect(result).not_to eq(default_shard)
+      expect(result.name).to eq('foo')
+    end
+
+    it 'retries if creation races' do
+      expect(described_class)
+        .to receive(:find_or_create_by)
+        .with(name: 'default')
+        .and_raise(ActiveRecord::RecordNotUnique, 'fail')
+        .once
+
+      expect(described_class)
+        .to receive(:find_or_create_by)
+        .with(name: 'default')
+        .and_call_original
+
+      expect(described_class.by_name('default')).to eq(default_shard)
+    end
+  end
+end