Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

4 files changed, 156 insertions, 8 deletions

diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb
new file mode 100644
index 00000000000..3e6d7ac3c9f
--- /dev/null
+++ b/lib/gitlab/database/background_migration/batch_metrics.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      class BatchMetrics
+        attr_reader :timings
+
+        def initialize
+          @timings = {}
+        end
+
+        def time_operation(label)
+          start_time = monotonic_time
+
+          yield
+
+          timings_for_label(label) << monotonic_time - start_time
+        end
+
+        private
+
+        def timings_for_label(label)
+          timings[label] ||= []
+        end
+
+        def monotonic_time
+          Gitlab::Metrics::System.monotonic_time
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index 0c9add9b355..4aa33ed7946 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -5,7 +5,7 @@ module Gitlab
     module BackgroundMigration
       class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord
         JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration'
-        BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze
+        BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies"
 
         self.table_name = :batched_background_migrations
 
@@ -23,8 +23,15 @@ module Gitlab
           finished: 3
         }
 
-        def interval_elapsed?
-          last_job.nil? || last_job.created_at <= Time.current - interval
+        def self.active_migration
+          active.queue_order.first
+        end
+
+        def interval_elapsed?(variance: 0)
+          return true unless last_job
+
+          interval_with_variance = interval - variance
+          last_job.created_at <= Time.current - interval_with_variance
         end
 
         def create_batched_job!(min, max)
@@ -50,6 +57,13 @@ module Gitlab
         def batch_class_name=(class_name)
           write_attribute(:batch_class_name, class_name.demodulize)
         end
+
+        def prometheus_labels
+          @prometheus_labels ||= {
+            migration_id: id,
+            migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name]
+          }
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb
index 5f8a5ec06a5..cf8b61f5feb 100644
--- a/lib/gitlab/database/background_migration/scheduler.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb
@@ -3,12 +3,22 @@
 module Gitlab
   module Database
     module BackgroundMigration
-      class Scheduler
-        def perform(migration_wrapper: BatchedMigrationWrapper.new)
-          active_migration = BatchedMigration.active.queue_order.first
-
-          return unless active_migration&.interval_elapsed?
+      class BatchedMigrationRunner
+        def initialize(migration_wrapper = BatchedMigrationWrapper.new)
+          @migration_wrapper = migration_wrapper
+        end
 
+        # Runs the next batched_job for a batched_background_migration.
+        #
+        # The batch bounds of the next job are calculated at runtime, based on the migration
+        # configuration and the bounds of the most recently created batched_job. Updating the
+        # migration configuration will cause future jobs to use the updated batch sizes.
+        #
+        # The job instance will automatically receive a set of arguments based on the migration
+        # configuration. For more details, see the BatchedMigrationWrapper class.
+        #
+        # Note that this method is primarily intended to called by a scheduled worker.
+        def run_migration_job(active_migration)
           if next_batched_job = create_next_batched_job!(active_migration)
             migration_wrapper.perform(next_batched_job)
           else
@@ -16,8 +26,26 @@ module Gitlab
           end
         end
 
+        # Runs all remaining batched_jobs for a batched_background_migration.
+        #
+        # This method is intended to be used in a test/dev environment to execute the background
+        # migration inline. It should NOT be used in a real environment for any non-trivial migrations.
+        def run_entire_migration(migration)
+          unless Rails.env.development? || Rails.env.test?
+            raise 'this method is not intended for use in real environments'
+          end
+
+          while migration.active?
+            run_migration_job(migration)
+
+            migration.reload_last_job
+          end
+        end
+
         private
 
+        attr_reader :migration_wrapper
+
         def create_next_batched_job!(active_migration)
           next_batch_range = find_next_batch_range(active_migration)
 
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index 299bd992197..c276f8ce75b 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -4,6 +4,15 @@ module Gitlab
   module Database
     module BackgroundMigration
       class BatchedMigrationWrapper
+        extend Gitlab::Utils::StrongMemoize
+
+        # Wraps the execution of a batched_background_migration.
+        #
+        # Updates the job's tracking records with the status of the migration
+        # when starting and finishing execution, and optionally saves batch_metrics
+        # the migration provides, if any are given.
+        #
+        # The job's batch_metrics are serialized to JSON for storage.
         def perform(batch_tracking_record)
           start_tracking_execution(batch_tracking_record)
 
@@ -16,6 +25,7 @@ module Gitlab
           raise e
         ensure
           finish_tracking_execution(batch_tracking_record)
+          track_prometheus_metrics(batch_tracking_record)
         end
 
         private
@@ -34,12 +44,75 @@ module Gitlab
             tracking_record.migration_column_name,
             tracking_record.sub_batch_size,
             *tracking_record.migration_job_arguments)
+
+          if job_instance.respond_to?(:batch_metrics)
+            tracking_record.metrics = job_instance.batch_metrics
+          end
         end
 
         def finish_tracking_execution(tracking_record)
           tracking_record.finished_at = Time.current
           tracking_record.save!
         end
+
+        def track_prometheus_metrics(tracking_record)
+          migration = tracking_record.batched_migration
+          base_labels = migration.prometheus_labels
+
+          metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size)
+          metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size)
+          metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size)
+
+          # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1)
+          efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f
+          metric_for(:histogram_time_efficiency).observe(base_labels, efficiency)
+
+          if metrics = tracking_record.metrics
+            metrics['timings']&.each do |key, timings|
+              summary = metric_for(:histogram_timings)
+              labels = base_labels.merge(operation: key)
+
+              timings.each do |timing|
+                summary.observe(labels, timing)
+              end
+            end
+          end
+        end
+
+        def metric_for(name)
+          self.class.metrics[name]
+        end
+
+        def self.metrics
+          strong_memoize(:metrics) do
+            {
+              gauge_batch_size: Gitlab::Metrics.gauge(
+                :batched_migration_job_batch_size,
+                'Batch size for a batched migration job'
+              ),
+              gauge_sub_batch_size: Gitlab::Metrics.gauge(
+                :batched_migration_job_sub_batch_size,
+                'Sub-batch size for a batched migration job'
+              ),
+              counter_updated_tuples: Gitlab::Metrics.counter(
+                :batched_migration_job_updated_tuples_total,
+                'Number of tuples updated by batched migration job'
+              ),
+              histogram_timings: Gitlab::Metrics.histogram(
+                :batched_migration_job_duration_seconds,
+                'Timings for a batched migration job',
+                {},
+                [0.1, 0.25, 0.5, 1, 5].freeze
+              ),
+              histogram_time_efficiency: Gitlab::Metrics.histogram(
+                :batched_migration_job_time_efficiency,
+                'Ratio of job duration to interval',
+                {},
+                [0.5, 0.9, 1, 1.5, 2].freeze
+              )
+            }
+          end
+        end
       end
     end
   end