Add latest changes from gitlab-org/gitlab@master

18 files changed, 374 insertions, 176 deletions

diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb
index be4d82bc500..fddbfa735e3 100644
--- a/lib/api/ci/helpers/runner.rb
+++ b/lib/api/ci/helpers/runner.rb
@@ -23,10 +23,17 @@ module API
           return get_runner_ip unless params['info'].present?
 
           attributes_for_keys(%w(name version revision platform architecture executor), params['info'])
+            .merge(get_system_id_from_request)
             .merge(get_runner_config_from_request)
             .merge(get_runner_ip)
         end
 
+        def get_system_id_from_request
+          return { system_id: params[:system_id] } if params.include?(:system_id)
+
+          {}
+        end
+
         def get_runner_ip
           { ip_address: ip_address }
         end
@@ -43,6 +50,15 @@ module API
           end
         end
 
+        def current_runner_machine
+          return if Feature.disabled?(:create_runner_machine)
+          return unless params[:system_id]
+
+          strong_memoize(:current_runner_machine) do
+            current_runner.ensure_machine(machine_xid: params[:system_id]) { |m| m.contacted_at = Time.current }
+          end
+        end
+
         def track_runner_authentication
           if current_runner
             metrics.increment_runner_authentication_success_counter(runner_type: current_runner.runner_type)
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 6b4394114df..ebe66f0a7be 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -115,6 +115,7 @@ module API
         end
         params do
           requires :token, type: String, desc: %q(Runner's authentication token)
+          optional :system_id, type: String, desc: %q(Runner's system identifier)
           optional :last_update, type: String, desc: %q(Runner's queue last_update token)
           optional :info, type: Hash, desc: %q(Runner's metadata) do
             optional :name, type: String, desc: %q(Runner's name)
@@ -166,7 +167,7 @@ module API
           end
 
           new_update = current_runner.ensure_runner_queue_value
-          result = ::Ci::RegisterJobService.new(current_runner).execute(runner_params)
+          result = ::Ci::RegisterJobService.new(current_runner, current_runner_machine).execute(runner_params)
 
           if result.valid?
             if result.build_json
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index 37be6903d8b..fcb7ddb9567 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -88,6 +88,7 @@ module API
 
       expose :emails_disabled, documentation: { type: 'boolean' }
       expose :shared_runners_enabled, documentation: { type: 'boolean' }
+      expose :group_runners_enabled, documentation: { type: 'boolean' }
       expose :lfs_enabled?, as: :lfs_enabled, documentation: { type: 'boolean' }
       expose :creator_id, documentation: { type: 'integer', example: 1 }
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: ->(project, options) do
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 820ad2e9b33..2700ea90d59 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -48,6 +48,7 @@ module API
         optional :warn_about_potentially_unwanted_characters, type: Boolean, desc: 'Warn about Potentially Unwanted Characters'
         optional :enforce_auth_checks_on_uploads, type: Boolean, desc: 'Enforce auth check on uploads'
         optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
+        optional :group_runners_enabled, type: Boolean, desc: 'Flag indication if group runners are enabled for that project'
         optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diff threads on lines changed with a push'
         optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge'
         optional :container_registry_enabled, type: Boolean, desc: 'Deprecated: Use :container_registry_access_level instead. Flag indication if the container registry is enabled for that project'
@@ -170,6 +171,7 @@ module API
           :security_and_compliance_access_level,
           :squash_option,
           :shared_runners_enabled,
+          :group_runners_enabled,
           :snippets_access_level,
           :tag_list,
           :topics,
diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
deleted file mode 100644
index 669e5338dd1..00000000000
--- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
+++ /dev/null
@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2,
-# it's enough at the moment to simply notice if the url is from `atlassian.net`
-module Gitlab
-  module BackgroundMigration
-    # Backfill the deployment_type in jira_tracker_data table
-    class BackfillJiraTrackerDeploymentType2
-      # Migration only version of jira_tracker_data table
-      class JiraTrackerDataTemp < ApplicationRecord
-        self.table_name = 'jira_tracker_data'
-
-        def self.encryption_options
-          {
-            key: Settings.attr_encrypted_db_key_base_32,
-            encode: true,
-            mode: :per_attribute_iv,
-            algorithm: 'aes-256-gcm'
-          }
-        end
-
-        attr_encrypted :url, encryption_options
-        attr_encrypted :api_url, encryption_options
-
-        enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment
-      end
-
-      # Migration only version of services table
-      class JiraServiceTemp < ApplicationRecord
-        self.table_name = 'services'
-        self.inheritance_column = :_type_disabled
-      end
-
-      def perform(start_id, stop_id)
-        @server_ids = []
-        @cloud_ids  = []
-
-        JiraTrackerDataTemp
-          .where(id: start_id..stop_id, deployment_type: 0)
-          .each do |jira_tracker_data|
-            collect_deployment_type(jira_tracker_data)
-          end
-
-        unless cloud_ids.empty?
-          JiraTrackerDataTemp.where(id: cloud_ids)
-            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud])
-        end
-
-        unless server_ids.empty?
-          JiraTrackerDataTemp.where(id: server_ids)
-            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server])
-        end
-
-        mark_jobs_as_succeeded(start_id, stop_id)
-      end
-
-      private
-
-      attr_reader :server_ids, :cloud_ids
-
-      def client_url(jira_tracker_data)
-        jira_tracker_data.api_url.presence || jira_tracker_data.url.presence
-      end
-
-      def server_type(url)
-        url.downcase.include?('.atlassian.net') ? :cloud : :server
-      end
-
-      def collect_deployment_type(jira_tracker_data)
-        url = client_url(jira_tracker_data)
-        return unless url
-
-        case server_type(url)
-        when :cloud
-          cloud_ids << jira_tracker_data.id
-        else
-          server_ids << jira_tracker_data.id
-        end
-      end
-
-      def mark_jobs_as_succeeded(*arguments)
-        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb
index 8e18d57b724..89d681c418d 100644
--- a/lib/gitlab/ci/variables/builder.rb
+++ b/lib/gitlab/ci/variables/builder.rb
@@ -8,6 +8,7 @@ module Gitlab
 
         def initialize(pipeline)
           @pipeline = pipeline
+          @pipeline_variables_builder = Builder::Pipeline.new(pipeline)
           @instance_variables_builder = Builder::Instance.new
           @project_variables_builder = Builder::Project.new(project)
           @group_variables_builder = Builder::Group.new(project&.group)
@@ -18,7 +19,7 @@ module Gitlab
           Gitlab::Ci::Variables::Collection.new.tap do |variables|
             variables.concat(predefined_variables(job))
             variables.concat(project.predefined_variables)
-            variables.concat(pipeline.predefined_variables)
+            variables.concat(pipeline_variables_builder.predefined_variables)
             variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner
             variables.concat(kubernetes_variables(environment: environment, job: job))
             variables.concat(job.yaml_variables)
@@ -38,7 +39,7 @@ module Gitlab
             break variables unless project
 
             variables.concat(project.predefined_variables)
-            variables.concat(pipeline.predefined_variables)
+            variables.concat(pipeline_variables_builder.predefined_variables)
             variables.concat(secret_instance_variables)
             variables.concat(secret_group_variables(environment: nil))
             variables.concat(secret_project_variables(environment: nil))
@@ -117,6 +118,7 @@ module Gitlab
         private
 
         attr_reader :pipeline
+        attr_reader :pipeline_variables_builder
         attr_reader :instance_variables_builder
         attr_reader :project_variables_builder
         attr_reader :group_variables_builder
diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb
new file mode 100644
index 00000000000..96d6f1673b9
--- /dev/null
+++ b/lib/gitlab/ci/variables/builder/pipeline.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Variables
+      class Builder
+        class Pipeline
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(pipeline)
+            @pipeline = pipeline
+          end
+
+          def predefined_variables
+            Gitlab::Ci::Variables::Collection.new.tap do |variables|
+              variables.append(key: 'CI_PIPELINE_IID', value: pipeline.iid.to_s)
+              variables.append(key: 'CI_PIPELINE_SOURCE', value: pipeline.source.to_s)
+              variables.append(key: 'CI_PIPELINE_CREATED_AT', value: pipeline.created_at&.iso8601)
+
+              variables.concat(predefined_commit_variables) if pipeline.sha.present?
+              variables.concat(predefined_commit_tag_variables) if pipeline.tag?
+              variables.concat(predefined_merge_request_variables) if pipeline.merge_request?
+
+              if pipeline.open_merge_requests_refs.any?
+                variables.append(key: 'CI_OPEN_MERGE_REQUESTS', value: pipeline.open_merge_requests_refs.join(','))
+              end
+
+              variables.append(key: 'CI_GITLAB_FIPS_MODE', value: 'true') if Gitlab::FIPS.enabled?
+
+              variables.append(key: 'CI_KUBERNETES_ACTIVE', value: 'true') if pipeline.has_kubernetes_active?
+              variables.append(key: 'CI_DEPLOY_FREEZE', value: 'true') if pipeline.freeze_period?
+
+              if pipeline.external_pull_request_event? && pipeline.external_pull_request
+                variables.concat(pipeline.external_pull_request.predefined_variables)
+              end
+            end
+          end
+
+          private
+
+          attr_reader :pipeline
+
+          def predefined_commit_variables
+            Gitlab::Ci::Variables::Collection.new.tap do |variables|
+              next variables unless pipeline.sha.present?
+
+              variables.append(key: 'CI_COMMIT_SHA', value: pipeline.sha)
+              variables.append(key: 'CI_COMMIT_SHORT_SHA', value: pipeline.short_sha)
+              variables.append(key: 'CI_COMMIT_BEFORE_SHA', value: pipeline.before_sha)
+              variables.append(key: 'CI_COMMIT_REF_NAME', value: pipeline.source_ref)
+              variables.append(key: 'CI_COMMIT_REF_SLUG', value: pipeline.source_ref_slug)
+              variables.append(key: 'CI_COMMIT_BRANCH', value: pipeline.ref) if pipeline.branch?
+              variables.append(key: 'CI_COMMIT_MESSAGE', value: pipeline.git_commit_message.to_s)
+              variables.append(key: 'CI_COMMIT_TITLE', value: pipeline.git_commit_full_title.to_s)
+              variables.append(key: 'CI_COMMIT_DESCRIPTION', value: pipeline.git_commit_description.to_s)
+              variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!pipeline.protected_ref?).to_s)
+              variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s)
+              variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s)
+
+              variables.concat(legacy_predefined_commit_variables)
+            end
+          end
+          strong_memoize_attr :predefined_commit_variables
+
+          def legacy_predefined_commit_variables
+            Gitlab::Ci::Variables::Collection.new.tap do |variables|
+              variables.append(key: 'CI_BUILD_REF', value: pipeline.sha)
+              variables.append(key: 'CI_BUILD_BEFORE_SHA', value: pipeline.before_sha)
+              variables.append(key: 'CI_BUILD_REF_NAME', value: pipeline.source_ref)
+              variables.append(key: 'CI_BUILD_REF_SLUG', value: pipeline.source_ref_slug)
+            end
+          end
+          strong_memoize_attr :legacy_predefined_commit_variables
+
+          def predefined_commit_tag_variables
+            Gitlab::Ci::Variables::Collection.new.tap do |variables|
+              git_tag = pipeline.project.repository.find_tag(pipeline.ref)
+
+              next variables unless git_tag
+
+              variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref)
+              variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message)
+
+              variables.concat(legacy_predefined_commit_tag_variables)
+            end
+          end
+          strong_memoize_attr :predefined_commit_tag_variables
+
+          def legacy_predefined_commit_tag_variables
+            Gitlab::Ci::Variables::Collection.new.tap do |variables|
+              variables.append(key: 'CI_BUILD_TAG', value: pipeline.ref)
+            end
+          end
+          strong_memoize_attr :legacy_predefined_commit_tag_variables
+
+          def predefined_merge_request_variables
+            Gitlab::Ci::Variables::Collection.new.tap do |variables|
+              variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: pipeline.merge_request_event_type.to_s)
+              variables.append(key: 'CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', value: pipeline.source_sha.to_s)
+              variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_SHA', value: pipeline.target_sha.to_s)
+
+              if merge_request_diff.present?
+                variables.append(key: 'CI_MERGE_REQUEST_DIFF_ID', value: merge_request_diff.id.to_s)
+                variables.append(key: 'CI_MERGE_REQUEST_DIFF_BASE_SHA', value: merge_request_diff.base_commit_sha)
+              end
+
+              variables.concat(pipeline.merge_request.predefined_variables)
+            end
+          end
+          strong_memoize_attr :predefined_merge_request_variables
+
+          def merge_request_diff
+            pipeline.merge_request_diff
+          end
+          strong_memoize_attr :merge_request_diff
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb
new file mode 100644
index 00000000000..76403f9a643
--- /dev/null
+++ b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncForeignKeys
+      class PostgresAsyncForeignKeyValidation < SharedModel
+        self.table_name = 'postgres_async_foreign_key_validations'
+
+        MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
+        MAX_LAST_ERROR_LENGTH = 10_000
+
+        validates :name, presence: true, uniqueness: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :last_error, length: { maximum: MAX_LAST_ERROR_LENGTH }
+
+        scope :ordered, -> { order(attempts: :asc, id: :asc) }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes.rb b/lib/gitlab/database/async_indexes.rb
index 6f301a66803..581c7e7ff94 100644
--- a/lib/gitlab/database/async_indexes.rb
+++ b/lib/gitlab/database/async_indexes.rb
@@ -16,6 +16,15 @@ module Gitlab
           IndexDestructor.new(async_index).perform
         end
       end
+
+      def self.execute_pending_actions!(how_many: DEFAULT_INDEXES_PER_INVOCATION)
+        queue_ids = PostgresAsyncIndex.ordered.limit(how_many).pluck(:id)
+        removal_actions = PostgresAsyncIndex.where(id: queue_ids).to_drop.ordered
+        creation_actions = PostgresAsyncIndex.where(id: queue_ids).to_create.ordered
+
+        removal_actions.each { |async_index| IndexDestructor.new(async_index).perform }
+        creation_actions.each { |async_index| IndexCreator.new(async_index).perform }
+      end
     end
   end
 end
diff --git a/lib/gitlab/database/async_indexes/index_base.rb b/lib/gitlab/database/async_indexes/index_base.rb
new file mode 100644
index 00000000000..2891618e978
--- /dev/null
+++ b/lib/gitlab/database/async_indexes/index_base.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class IndexBase
+        include IndexingExclusiveLeaseGuard
+        extend ::Gitlab::Utils::Override
+
+        TIMEOUT_PER_ACTION = 1.day
+
+        def initialize(async_index)
+          @async_index = async_index
+        end
+
+        def perform
+          try_obtain_lease do
+            if preconditions_met?
+              log_index_info("Starting async index #{action_type}")
+              execute_action_with_error_handling
+              log_index_info("Finished async index #{action_type}")
+            else
+              log_index_info(skip_log_message)
+              async_index.destroy!
+            end
+          end
+        end
+
+        private
+
+        attr_reader :async_index
+
+        delegate :connection, to: :async_index
+
+        def preconditions_met?
+          raise NotImplementedError, 'must implement preconditions_met?'
+        end
+
+        def action_type
+          raise NotImplementedError, 'must implement action_type'
+        end
+
+        def execute_action_with_error_handling
+          around_execution { execute_action }
+        rescue StandardError => error
+          async_index.handle_exception!(error)
+
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
+          Gitlab::AppLogger.error(message: error.message, **logging_options)
+        end
+
+        def around_execution
+          yield
+        end
+
+        def execute_action
+          connection.execute(async_index.definition)
+          async_index.destroy!
+        end
+
+        def index_exists?
+          connection.indexes(async_index.table_name).any? do |index|
+            index.name == async_index.name
+          end
+        end
+
+        def lease_timeout
+          TIMEOUT_PER_ACTION
+        end
+
+        def log_index_info(message)
+          Gitlab::AppLogger.info(message: message, **logging_options)
+        end
+
+        def skip_log_message
+          "Skipping index #{action_type} since preconditions are not met. " \
+            "The queuing entry will be deleted"
+        end
+
+        def logging_options
+          {
+            table_name: async_index.table_name,
+            index_name: async_index.name,
+            class: self.class.name.to_s
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb
index 3ae2bb7b3e5..c5f4c5f30ad 100644
--- a/lib/gitlab/database/async_indexes/index_creator.rb
+++ b/lib/gitlab/database/async_indexes/index_creator.rb
@@ -3,48 +3,24 @@
 module Gitlab
   module Database
     module AsyncIndexes
-      class IndexCreator
-        include IndexingExclusiveLeaseGuard
-
-        TIMEOUT_PER_ACTION = 1.day
+      class IndexCreator < AsyncIndexes::IndexBase
         STATEMENT_TIMEOUT = 20.hours
 
-        def initialize(async_index)
-          @async_index = async_index
-        end
-
-        def perform
-          try_obtain_lease do
-            if index_exists?
-              log_index_info('Skipping index creation as the index exists')
-            else
-              log_index_info('Creating async index')
-
-              set_statement_timeout do
-                connection.execute(async_index.definition)
-              end
-
-              log_index_info('Finished creating async index')
-            end
-
-            async_index.destroy
-          end
-        end
-
         private
 
-        attr_reader :async_index
-
-        def index_exists?
-          connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name }
+        override :preconditions_met?
+        def preconditions_met?
+          !index_exists?
         end
 
-        def connection
-          @connection ||= async_index.connection
+        override :action_type
+        def action_type
+          'creation'
         end
 
-        def lease_timeout
-          TIMEOUT_PER_ACTION
+        override :around_execution
+        def around_execution(&block)
+          set_statement_timeout(&block)
         end
 
         def set_statement_timeout
@@ -53,10 +29,6 @@ module Gitlab
         ensure
           connection.execute('RESET statement_timeout')
         end
-
-        def log_index_info(message)
-          Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name)
-        end
       end
     end
   end
diff --git a/lib/gitlab/database/async_indexes/index_destructor.rb b/lib/gitlab/database/async_indexes/index_destructor.rb
index 66955df9d04..5596e099cb6 100644
--- a/lib/gitlab/database/async_indexes/index_destructor.rb
+++ b/lib/gitlab/database/async_indexes/index_destructor.rb
@@ -3,58 +3,29 @@
 module Gitlab
   module Database
     module AsyncIndexes
-      class IndexDestructor
-        include IndexingExclusiveLeaseGuard
-
-        TIMEOUT_PER_ACTION = 1.day
-
-        def initialize(async_index)
-          @async_index = async_index
-        end
-
-        def perform
-          try_obtain_lease do
-            if !index_exists?
-              log_index_info('Skipping dropping as the index does not exist')
-            else
-              log_index_info('Dropping async index')
-
-              retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
-                connection: connection,
-                timing_configuration: Gitlab::Database::Reindexing::REMOVE_INDEX_RETRY_CONFIG,
-                klass: self.class,
-                logger: Gitlab::AppLogger
-              )
-
-              retries.run(raise_on_exhaustion: false) do
-                connection.execute(async_index.definition)
-              end
-
-              log_index_info('Finished dropping async index')
-            end
-
-            async_index.destroy
-          end
-        end
-
+      class IndexDestructor < AsyncIndexes::IndexBase
         private
 
-        attr_reader :async_index
-
-        def index_exists?
-          connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name }
+        override :preconditions_met?
+        def preconditions_met?
+          index_exists?
         end
 
-        def connection
-          @connection ||= async_index.connection
+        override :action_type
+        def action_type
+          'removal'
         end
 
-        def lease_timeout
-          TIMEOUT_PER_ACTION
-        end
+        override :around_execution
+        def around_execution(&block)
+          retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
+            connection: connection,
+            timing_configuration: Gitlab::Database::Reindexing::REMOVE_INDEX_RETRY_CONFIG,
+            klass: self.class,
+            logger: Gitlab::AppLogger
+          )
 
-        def log_index_info(message)
-          Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name)
+          retries.run(raise_on_exhaustion: false, &block)
         end
       end
     end
diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb
index c8f6761534c..f459c43e0ee 100644
--- a/lib/gitlab/database/async_indexes/migration_helpers.rb
+++ b/lib/gitlab/database/async_indexes/migration_helpers.rb
@@ -22,7 +22,7 @@ module Gitlab
           return unless async_index_creation_available?
 
           PostgresAsyncIndex.find_by(name: index_name).try do |async_index|
-            async_index.destroy
+            async_index.destroy!
           end
         end
 
diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb
index dc932482d40..94ff4a69c62 100644
--- a/lib/gitlab/database/async_indexes/postgres_async_index.rb
+++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb
@@ -8,17 +8,37 @@ module Gitlab
 
         MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
         MAX_DEFINITION_LENGTH = 2048
+        MAX_LAST_ERROR_LENGTH = 10_000
 
         validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
         validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
         validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH }
+        validates :last_error, length: { maximum: MAX_LAST_ERROR_LENGTH },
+          if: ->(index) { index.respond_to?(:last_error) }
 
         scope :to_create, -> { where("definition ILIKE 'CREATE%'") }
         scope :to_drop, -> { where("definition ILIKE 'DROP%'") }
+        scope :ordered, -> { order(attempts: :asc, id: :asc) }
 
         def to_s
           definition
         end
+
+        def handle_exception!(error)
+          transaction do
+            increment!(:attempts)
+            update!(last_error: format_last_error(error))
+          end
+        end
+
+        private
+
+        def format_last_error(error)
+          [error.message]
+            .concat(error.backtrace)
+            .join("\n")
+            .truncate(MAX_LAST_ERROR_LENGTH)
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb
index 8e5dc98e96e..f5cb83e398a 100644
--- a/lib/gitlab/database/load_balancing/sticking.rb
+++ b/lib/gitlab/database/load_balancing/sticking.rb
@@ -121,19 +121,19 @@ module Gitlab
         end
 
         def unstick(namespace, id)
-          Gitlab::Redis::SharedState.with do |redis|
+          with_redis do |redis|
             redis.del(redis_key_for(namespace, id))
           end
         end
 
         def set_write_location_for(namespace, id, location)
-          Gitlab::Redis::SharedState.with do |redis|
+          with_redis do |redis|
             redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION)
           end
         end
 
         def last_write_location_for(namespace, id)
-          Gitlab::Redis::SharedState.with do |redis|
+          with_redis do |redis|
             redis.get(redis_key_for(namespace, id))
           end
         end
@@ -143,6 +143,12 @@ module Gitlab
 
           "database-load-balancing/write-location/#{name}/#{namespace}/#{id}"
         end
+
+        private
+
+        def with_redis(&block)
+          Gitlab::Redis::DbLoadBalancing.with(&block)
+        end
       end
     end
   end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index fff086c38a8..4d15022cca5 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -9,6 +9,7 @@ module Gitlab
     # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks.
     ALL_CLASSES = [
       Gitlab::Redis::Cache,
+      Gitlab::Redis::DbLoadBalancing,
       Gitlab::Redis::Queues,
       Gitlab::Redis::RateLimiting,
       Gitlab::Redis::RepositoryCache,
diff --git a/lib/gitlab/redis/db_load_balancing.rb b/lib/gitlab/redis/db_load_balancing.rb
new file mode 100644
index 00000000000..01276445611
--- /dev/null
+++ b/lib/gitlab/redis/db_load_balancing.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class DbLoadBalancing < ::Gitlab::Redis::Wrapper
+      class << self
+        # The data we store on DbLoadBalancing used to be stored on SharedState.
+        def config_fallback
+          SharedState
+        end
+
+        private
+
+        def redis
+          primary_store = ::Redis.new(params)
+          secondary_store = ::Redis.new(config_fallback.params)
+
+          MultiStore.new(primary_store, secondary_store, store_name)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 5757db30b2d..dd4fbc7c8f6 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -283,6 +283,36 @@ namespace :gitlab do
       end
     end
 
+    namespace :execute_async_index_operations do
+      each_database(databases) do |database_name|
+        task database_name, [:pick] => :environment do |_, args|
+          args.with_defaults(pick: 2)
+
+          if Feature.disabled?(:database_async_index_operations, type: :ops)
+            puts <<~NOTE.color(:yellow)
+              Note: database async index operations feature is currently disabled.
+
+              Enable with: Feature.enable(:database_async_index_operations)
+            NOTE
+            exit
+          end
+
+          Gitlab::Database::EachDatabase.each_database_connection(only: database_name) do
+            Gitlab::Database::AsyncIndexes.execute_pending_actions!(how_many: args[:pick].to_i)
+          end
+        end
+      end
+
+      task :all, [:pick] => :environment do |_, args|
+        default_pick = Gitlab.dev_or_test_env? ? 1000 : 2
+        args.with_defaults(pick: default_pick)
+
+        each_database(databases) do |database_name|
+          Rake::Task["gitlab:db:execute_async_index_operations:#{database_name}"].invoke(args[:pick])
+        end
+      end
+    end
+
     desc 'Check if there have been user additions to the database'
     task active: :environment do
       if ActiveRecord::Base.connection.migration_context.needs_migration?