Merge branch 'lock-pipeline-schedule-worker' into 'master'

Prevent concurrent execution of PipelineScheduleWorker

Closes gitlab-com/gl-infra/production#805

See merge request gitlab-org/gitlab-ce!27781

1 files changed, 17 insertions, 11 deletions

diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb
index 02a69ea3b54..8a9ee7808e4 100644
--- a/app/workers/pipeline_schedule_worker.rb
+++ b/app/workers/pipeline_schedule_worker.rb
@@ -3,20 +3,26 @@
 class PipelineScheduleWorker
   include ApplicationWorker
   include CronjobQueue
+  include ::Gitlab::ExclusiveLeaseHelpers
+
+  EXCLUSIVE_LOCK_KEY = 'pipeline_schedules:run:lock'
+  LOCK_TIMEOUT = 50.minutes
 
   # rubocop: disable CodeReuse/ActiveRecord
   def perform
-    Ci::PipelineSchedule.active.where("next_run_at < ?", Time.now)
-      .preload(:owner, :project).find_each do |schedule|
-
-      Ci::CreatePipelineService.new(schedule.project,
-                                    schedule.owner,
-                                    ref: schedule.ref)
-        .execute!(:schedule, ignore_skip_ci: true, save_on_errors: true, schedule: schedule)
-    rescue => e
-      error(schedule, e)
-    ensure
-      schedule.schedule_next_run!
+    in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do
+      Ci::PipelineSchedule.active.where("next_run_at < ?", Time.now)
+        .preload(:owner, :project).find_each do |schedule|
+
+        schedule.schedule_next_run!
+
+        Ci::CreatePipelineService.new(schedule.project,
+                                      schedule.owner,
+                                      ref: schedule.ref)
+          .execute!(:schedule, ignore_skip_ci: true, save_on_errors: true, schedule: schedule)
+      rescue => e
+        error(schedule, e)
+      end
     end
   end
   # rubocop: enable CodeReuse/ActiveRecord