Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (170 commits)
  support ordering of project notes in notes api
  Redirect to an already forked project if it exists
  Reschedule the migration to populate fork networks
  Create fork networks for forks for which the source was deleted.
  Fix item name and namespace text overflow in Projects dropdown
  Minor backport from EE
  fix link that was linking to `html` instead of `md`
  Backport epic tasklist
  Add timeouts for Gitaly calls
  SSHUploadPack over Gitaly is now OptOut
  fix icon colors in commit list
  Fix star icon color/stroke
  Backport border inline edit
  Add checkboxes to automatically run AutoDevops pipeline
  BE for automatic pipeline when enabling Auto DevOps
  I am certainly weary of debugging sidekiq but I don't think that's what was meant
  Ensure MRs always use branch refs for comparison
  Fix issue comment submit button disabled on GFM paste
  Lock seed-fu at the correct version in Gemfile.lock
  Improve indexes on merge_request_diffs
  ...

4 files changed, 53 insertions, 5 deletions

diff --git a/app/workers/create_pipeline_worker.rb b/app/workers/create_pipeline_worker.rb
new file mode 100644
index 00000000000..865ad1ba420
--- /dev/null
+++ b/app/workers/create_pipeline_worker.rb
@@ -0,0 +1,16 @@
+class CreatePipelineWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  enqueue_in group: :creation
+
+  def perform(project_id, user_id, ref, source, params = {})
+    project = Project.find(project_id)
+    user = User.find(user_id)
+    params = params.deep_symbolize_keys
+
+    Ci::CreatePipelineService
+      .new(project, user, ref: ref)
+      .execute(source, **params)
+  end
+end
diff --git a/app/workers/pipeline_schedule_worker.rb b/app/workers/pipeline_schedule_worker.rb
index d7087f20dfc..7320db1065e 100644
--- a/app/workers/pipeline_schedule_worker.rb
+++ b/app/workers/pipeline_schedule_worker.rb
@@ -9,7 +9,7 @@ class PipelineScheduleWorker
         pipeline = Ci::CreatePipelineService.new(schedule.project,
                                                  schedule.owner,
                                                  ref: schedule.ref)
-          .execute(:schedule, save_on_errors: false, schedule: schedule)
+          .execute(:schedule, ignore_skip_ci: true, save_on_errors: false, schedule: schedule)
 
         schedule.deactivate! unless pipeline.persisted?
       rescue => e
diff --git a/app/workers/project_migrate_hashed_storage_worker.rb b/app/workers/project_migrate_hashed_storage_worker.rb
index ca276d7801c..127aa6b9d7d 100644
--- a/app/workers/project_migrate_hashed_storage_worker.rb
+++ b/app/workers/project_migrate_hashed_storage_worker.rb
@@ -2,10 +2,34 @@ class ProjectMigrateHashedStorageWorker
   include Sidekiq::Worker
   include DedicatedSidekiqQueue
 
+  LEASE_TIMEOUT = 30.seconds.to_i
+
   def perform(project_id)
     project = Project.find_by(id: project_id)
     return if project.nil? || project.pending_delete?
 
-    ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    uuid = lease_for(project_id).try_obtain
+    if uuid
+      ::Projects::HashedStorageMigrationService.new(project, logger).execute
+    else
+      false
+    end
+  rescue => ex
+    cancel_lease_for(project_id, uuid) if uuid
+    raise ex
+  end
+
+  def lease_for(project_id)
+    Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
+  end
+
+  private
+
+  def lease_key(project_id)
+    "project_migrate_hashed_storage_worker:#{project_id}"
+  end
+
+  def cancel_lease_for(project_id, uuid)
+    Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
   end
 end
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb
index fdbc049c2df..367e227f680 100644
--- a/app/workers/stuck_ci_jobs_worker.rb
+++ b/app/workers/stuck_ci_jobs_worker.rb
@@ -45,9 +45,17 @@ class StuckCiJobsWorker
   end
 
   def search(status, timeout)
-    builds = Ci::Build.where(status: status).where('ci_builds.updated_at < ?', timeout.ago)
-    builds.joins(:project).merge(Project.without_deleted).includes(:tags, :runner, project: :namespace).find_each(batch_size: 50).each do |build|
-      yield(build)
+    loop do
+      jobs = Ci::Build.where(status: status)
+        .where('ci_builds.updated_at < ?', timeout.ago)
+        .includes(:tags, :runner, project: :namespace)
+        .limit(100)
+        .to_a
+      break if jobs.empty?
+
+      jobs.each do |job|
+        yield(job)
+      end
     end
   end