Add background migration to populate pages metadata61927-pages-virtual-hosts-api

Background migration to populate project_pages_metadata for each
existing project. Migrate projects on demand if there is no existing
record yet.

6 files changed, 145 insertions, 2 deletions

diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 2b2194aaaa1..248049b5aab 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -319,7 +319,11 @@ class Namespace < ApplicationRecord
   private
 
   def all_projects_with_pages
-    all_projects.migrate_project_pages_metadata
+    if all_projects.project_pages_metadata_not_migrated.exists?
+      BackgroundMigrations::MigratePages.new.perform_on_relation(
+        all_projects.project_pages_metadata_not_migrated)
+    end
+
     all_projects.with_pages_deployed
   end
 
diff --git a/db/post_migrate/20190902153712_schedule_pages_metadata_migration.rb b/db/post_migrate/20190902153712_schedule_pages_metadata_migration.rb
new file mode 100644
index 00000000000..988497d559c
--- /dev/null
+++ b/db/post_migrate/20190902153712_schedule_pages_metadata_migration.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class SchedulePagesMetadataMigration < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 1_000
+  MIGRATION = 'MigratePagesMetadata'
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'projects'
+  end
+
+  def up
+    Project.each_batch(of: BATCH_SIZE) do |relation, index|
+      delay = index * 1.minute
+
+      range = relation.pluck('MIN(id)', 'MAX(id)').first
+
+      BackgroundMigrationWorker.perform_in(delay, MIGRATION, range)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3ea5aad7ab8..f263fa6ad0a 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_02_131045) do
+ActiveRecord::Schema.define(version: 2019_09_02_153712) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb
new file mode 100644
index 00000000000..10296e23dfd
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will insert record into project_pages_metadata
+    # for each existing project
+    class MigratePagesMetadata
+      # Project model
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+      end
+
+      def perform(start_id, stop_id)
+        perform_on_relation(Project.where(id: start_id..stop_id))
+      end
+
+      def perform_on_relation(relation)
+        successful_pages_deploy = <<~SQL
+          SELECT TRUE
+          FROM "ci_builds"
+          WHERE "ci_builds"."type" IN ('GenericCommitStatus')
+            AND ("ci_builds"."status" IN ('success'))
+            AND "ci_builds"."stage" = 'deploy'
+            AND "ci_builds"."name" = 'pages:deploy'
+            AND (ci_builds.project_id = projects.id)
+          LIMIT 1
+        SQL
+
+        select_from = relation
+          .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)")
+          .to_sql
+
+        ActiveRecord::Base.connection_pool.with_connection do |connection|
+          connection.execute <<~SQL
+            INSERT INTO project_pages_metadata (project_id, deployed)
+            #{select_from}
+            ON CONFLICT (project_id) DO NOTHING
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb b/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb
new file mode 100644
index 00000000000..7bd85c252cb
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Gitlab::BackgroundMigration::MigratePagesMetadata, :migration, schema: 20190806112508 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pages_metadata) { table(:project_pages_metadata) }
+
+  let!(:namespace) { namespaces.create(name: 'gitlab', path: 'gitlab-org') }
+  let!(:project_1) { projects.create(namespace_id: namespace.id, name: 'Project 1') }
+  let!(:project_2) { projects.create(namespace_id: namespace.id, name: 'Project 2') }
+  let!(:project_3) { projects.create(namespace_id: namespace.id, name: 'Project 3') }
+
+  subject(:migration) { described_class.new }
+
+  describe '#perform' do
+    it 'creates pages metadata for the specified range of projects' do
+      migration.perform(project_1.id, project_2.id)
+
+      expect(pages_metadata.find_by_project_id(project_1.id)).to be_persisted
+      expect(pages_metadata.find_by_project_id(project_2.id)).to be_persisted
+      expect(pages_metadata.find_by_project_id(project_3.id)).to be_nil
+    end
+  end
+
+  describe '#perform_on_relation' do
+    it 'creates pages metadata for the specified projects' do
+      migration.perform_on_relation(Project.where(id: project_2.id))
+
+      expect(pages_metadata.find_by_project_id(project_1.id)).to be_nil
+      expect(pages_metadata.find_by_project_id(project_2.id)).to be_persisted
+      expect(pages_metadata.find_by_project_id(project_3.id)).to be_nil
+    end
+  end
+end
diff --git a/spec/migrations/schedule_pages_metadata_migration_spec.rb b/spec/migrations/schedule_pages_metadata_migration_spec.rb
new file mode 100644
index 00000000000..b25ec15044e
--- /dev/null
+++ b/spec/migrations/schedule_pages_metadata_migration_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20190902153712_schedule_pages_metadata_migration')
+
+describe SchedulePagesMetadataMigration, :migration, :sidekiq do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    namespaces.create!(id: 11, name: 'gitlab', path: 'gitlab-org')
+    projects.create!(id: 111, namespace_id: 11, name: 'Project 111')
+    projects.create!(id: 114, namespace_id: 11, name: 'Project 114')
+  end
+
+  it 'schedules pages metadata migration' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(1.minute, 111, 111)
+        expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, 114, 114)
+        expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+      end
+    end
+  end
+end