Merge branch 'set-real-next-run-at-for-preventing-duplciate-pipeline-creations' into 'master'

Make pipeline schedule worker resilient

Closes gitlab-com/gl-infra/production#805 and #61955

See merge request gitlab-org/gitlab-ce!28407

4 files changed, 58 insertions, 47 deletions

diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index c0a0ca9acf6..c40ad39be61 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -27,9 +27,13 @@ module Ci
 
     scope :active, -> { where(active: true) }
     scope :inactive, -> { where(active: false) }
+    scope :runnable_schedules, -> { active.where("next_run_at < ?", Time.now) }
+    scope :preloaded, -> { preload(:owner, :project) }
 
     accepts_nested_attributes_for :variables, allow_destroy: true
 
+    alias_attribute :real_next_run, :next_run_at
+
     def owned_by?(current_user)
       owner == current_user
     end
@@ -46,8 +50,14 @@ module Ci
       update_attribute(:active, false)
     end
 
+    ##
+    # The `next_run_at` column is set to the actual execution date of `PipelineScheduleWorker`.
+    # This way, a schedule like `*/1 * * * *` won't be triggered in a short interval
+    # when PipelineScheduleWorker runs irregularly by Sidekiq Memory Killer.
     def set_next_run_at
-      self.next_run_at = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now)
+      self.next_run_at = Gitlab::Ci::CronParser.new(Settings.cron_jobs['pipeline_schedule_worker']['cron'],
+                                                    Time.zone.name)
+                                               .next_time_from(ideal_next_run_at)
     end
 
     def schedule_next_run!
@@ -56,15 +66,14 @@ module Ci
       update_attribute(:next_run_at, nil) # update without validation
     end
 
-    def real_next_run(
-        worker_cron: Settings.cron_jobs['pipeline_schedule_worker']['cron'],
-        worker_time_zone: Time.zone.name)
-      Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone)
-                            .next_time_from(next_run_at)
-    end
-
     def job_variables
       variables&.map(&:to_runner_variable) || []
     end
+
+    private
+
+    def ideal_next_run_at
+      Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now)
+    end
   end
 end
diff --git a/app/services/ci/pipeline_schedule_service.rb b/app/services/ci/pipeline_schedule_service.rb
new file mode 100644
index 00000000000..387d0351490
--- /dev/null
+++ b/app/services/ci/pipeline_schedule_service.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Ci
+  class PipelineScheduleService < BaseService
+    def execute(schedule)
+      # Ensure `next_run_at` is set properly before creating a pipeline.
+      # Otherwise, multiple pipelines could be created in a short interval.
+      schedule.schedule_next_run!
+
+      RunPipelineScheduleWorker.perform_async(schedule.id, schedule.owner.id)
+    end
+  end
+end
diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb
index 8a9ee7808e4..9410fd1a786 100644
--- a/app/workers/pipeline_schedule_worker.rb
+++ b/app/workers/pipeline_schedule_worker.rb
@@ -3,47 +3,12 @@
 class PipelineScheduleWorker
   include ApplicationWorker
   include CronjobQueue
-  include ::Gitlab::ExclusiveLeaseHelpers
 
-  EXCLUSIVE_LOCK_KEY = 'pipeline_schedules:run:lock'
-  LOCK_TIMEOUT = 50.minutes
-
-  # rubocop: disable CodeReuse/ActiveRecord
   def perform
-    in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
-      Ci::PipelineSchedule.active.where("next_run_at < ?", Time.now)
-        .preload(:owner, :project).find_each do |schedule|
-
-        schedule.schedule_next_run!
-
-        Ci::CreatePipelineService.new(schedule.project,
-                                      schedule.owner,
-                                      ref: schedule.ref)
-          .execute!(:schedule, ignore_skip_ci: true, save_on_errors: true, schedule: schedule)
-      rescue => e
-        error(schedule, e)
+    Ci::PipelineSchedule.runnable_schedules.preloaded.find_in_batches do |schedules|
+      schedules.each do |schedule|
+        Ci::PipelineScheduleService.new(schedule.project, schedule.owner).execute(schedule)
       end
     end
   end
-  # rubocop: enable CodeReuse/ActiveRecord
-
-  private
-
-  def error(schedule, error)
-    failed_creation_counter.increment
-
-    Rails.logger.error "Failed to create a scheduled pipeline. " \
-                       "schedule_id: #{schedule.id} message: #{error.message}"
-
-    Gitlab::Sentry
-      .track_exception(error,
-                       issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/41231',
-                       extra: { schedule_id: schedule.id })
-  end
-
-  def failed_creation_counter
-    @failed_creation_counter ||=
-      Gitlab::Metrics.counter(:pipeline_schedule_creation_failed_total,
-                              "Counter of failed attempts of pipeline schedule creation")
-  end
 end
diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb
index f72331c003a..43e0b9db22f 100644
--- a/app/workers/run_pipeline_schedule_worker.rb
+++ b/app/workers/run_pipeline_schedule_worker.rb
@@ -21,6 +21,30 @@ class RunPipelineScheduleWorker
     Ci::CreatePipelineService.new(schedule.project,
                                   user,
                                   ref: schedule.ref)
-      .execute(:schedule, ignore_skip_ci: true, save_on_errors: false, schedule: schedule)
+      .execute!(:schedule, ignore_skip_ci: true, save_on_errors: false, schedule: schedule)
+  rescue Ci::CreatePipelineService::CreateError
+    # no-op. This is a user operation error such as corrupted .gitlab-ci.yml.
+  rescue => e
+    error(schedule, e)
+  end
+
+  private
+
+  def error(schedule, error)
+    failed_creation_counter.increment
+
+    Rails.logger.error "Failed to create a scheduled pipeline. " \
+                       "schedule_id: #{schedule.id} message: #{error.message}"
+
+    Gitlab::Sentry
+      .track_exception(error,
+                       issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/41231',
+                       extra: { schedule_id: schedule.id })
+  end
+
+  def failed_creation_counter
+    @failed_creation_counter ||=
+      Gitlab::Metrics.counter(:pipeline_schedule_creation_failed_total,
+                              "Counter of failed attempts of pipeline schedule creation")
   end
 end