Merge branch 'master' of gitlab_gitlab:gitlab-org/gitlab-cerunner-metrics-extractor

41 files changed, 186 insertions, 94 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 3d34bfc05c7..991a177018e 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -3,6 +3,12 @@
 
 - auto_merge:auto_merge_process
 
+- chaos:chaos_cpu_spin
+- chaos:chaos_db_spin
+- chaos:chaos_kill
+- chaos:chaos_leak_mem
+- chaos:chaos_sleep
+
 - cronjob:admin_email
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb
index 4a9becf0ca7..66f9b8d9e80 100644
--- a/app/workers/archive_trace_worker.rb
+++ b/app/workers/archive_trace_worker.rb
@@ -7,7 +7,7 @@ class ArchiveTraceWorker
   # rubocop: disable CodeReuse/ActiveRecord
   def perform(job_id)
     Ci::Build.without_archived_trace.find_by(id: job_id).try do |job|
-      Ci::ArchiveTraceService.new.execute(job)
+      Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name)
     end
   end
   # rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/background_migration_worker.rb b/app/workers/background_migration_worker.rb
index 688b600649a..b83412b5e6e 100644
--- a/app/workers/background_migration_worker.rb
+++ b/app/workers/background_migration_worker.rb
@@ -76,8 +76,6 @@ class BackgroundMigrationWorker
   # class_name - The name of the background migration that we might want to
   #              run.
   def healthy_database?
-    return true unless Gitlab::Database.postgresql?
-
     !Postgresql::ReplicationSlot.lag_too_great?
   end
 
diff --git a/app/workers/chaos/cpu_spin_worker.rb b/app/workers/chaos/cpu_spin_worker.rb
new file mode 100644
index 00000000000..43a32c3274f
--- /dev/null
+++ b/app/workers/chaos/cpu_spin_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Chaos
+  class CpuSpinWorker
+    include ApplicationWorker
+    include ChaosQueue
+
+    def perform(duration_s)
+      Gitlab::Chaos.cpu_spin(duration_s)
+    end
+  end
+end
diff --git a/app/workers/chaos/db_spin_worker.rb b/app/workers/chaos/db_spin_worker.rb
new file mode 100644
index 00000000000..217ddabbcb6
--- /dev/null
+++ b/app/workers/chaos/db_spin_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Chaos
+  class DbSpinWorker
+    include ApplicationWorker
+    include ChaosQueue
+
+    def perform(duration_s, interval_s)
+      Gitlab::Chaos.db_spin(duration_s, interval_s)
+    end
+  end
+end
diff --git a/app/workers/chaos/kill_worker.rb b/app/workers/chaos/kill_worker.rb
new file mode 100644
index 00000000000..80f04db1be4
--- /dev/null
+++ b/app/workers/chaos/kill_worker.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Chaos
+  class KillWorker
+    include ApplicationWorker
+    include ChaosQueue
+
+    sidekiq_options retry: false
+
+    def perform
+      Gitlab::Chaos.kill
+    end
+  end
+end
diff --git a/app/workers/chaos/leak_mem_worker.rb b/app/workers/chaos/leak_mem_worker.rb
new file mode 100644
index 00000000000..0caa99e0de9
--- /dev/null
+++ b/app/workers/chaos/leak_mem_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Chaos
+  class LeakMemWorker
+    include ApplicationWorker
+    include ChaosQueue
+
+    def perform(memory_mb, duration_s)
+      Gitlab::Chaos.leak_mem(memory_mb, duration_s)
+    end
+  end
+end
diff --git a/app/workers/chaos/sleep_worker.rb b/app/workers/chaos/sleep_worker.rb
new file mode 100644
index 00000000000..7c724c4cb4e
--- /dev/null
+++ b/app/workers/chaos/sleep_worker.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Chaos
+  class SleepWorker
+    include ApplicationWorker
+    include ChaosQueue
+
+    def perform(duration_s)
+      Gitlab::Chaos.sleep(duration_s)
+    end
+  end
+end
diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb
index f65ff239866..ef2da729705 100644
--- a/app/workers/ci/archive_traces_cron_worker.rb
+++ b/app/workers/ci/archive_traces_cron_worker.rb
@@ -10,8 +10,8 @@ module Ci
       # Archive stale live traces which still resides in redis or database
       # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL
       # More details in https://gitlab.com/gitlab-org/gitlab-ce/issues/36791
-      Ci::Build.finished.with_live_trace.find_each(batch_size: 100) do |build|
-        Ci::ArchiveTraceService.new.execute(build)
+      Ci::Build.with_stale_live_trace.find_each(batch_size: 100) do |build|
+        Ci::ArchiveTraceService.new.execute(build, worker_name: self.class.name)
       end
     end
     # rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/cluster_configure_worker.rb b/app/workers/cluster_configure_worker.rb
index 6f64b7ea0ab..b0e551d4e03 100644
--- a/app/workers/cluster_configure_worker.rb
+++ b/app/workers/cluster_configure_worker.rb
@@ -5,10 +5,6 @@ class ClusterConfigureWorker
   include ClusterQueue
 
   def perform(cluster_id)
-    Clusters::Cluster.managed.find_by_id(cluster_id).try do |cluster|
-      if cluster.project_type?
-        Clusters::RefreshService.create_or_update_namespaces_for_cluster(cluster)
-      end
-    end
+    # Scheduled for removal in https://gitlab.com/gitlab-org/gitlab-ce/issues/59319
   end
 end
diff --git a/app/workers/cluster_project_configure_worker.rb b/app/workers/cluster_project_configure_worker.rb
index 497e57c0d0b..8f48eca4d86 100644
--- a/app/workers/cluster_project_configure_worker.rb
+++ b/app/workers/cluster_project_configure_worker.rb
@@ -5,8 +5,6 @@ class ClusterProjectConfigureWorker
   include ClusterQueue
 
   def perform(project_id)
-    project = Project.find(project_id)
-
-    ::Clusters::RefreshService.create_or_update_namespaces_for_project(project)
+    # Scheduled for removal in https://gitlab.com/gitlab-org/gitlab-ce/issues/59319
   end
 end
diff --git a/app/workers/concerns/chaos_queue.rb b/app/workers/concerns/chaos_queue.rb
new file mode 100644
index 00000000000..e406509d12d
--- /dev/null
+++ b/app/workers/concerns/chaos_queue.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+#
+module ChaosQueue
+  extend ActiveSupport::Concern
+
+  included do
+    queue_namespace :chaos
+  end
+end
diff --git a/app/workers/concerns/new_issuable.rb b/app/workers/concerns/new_issuable.rb
index a89451a4475..22ba7c97309 100644
--- a/app/workers/concerns/new_issuable.rb
+++ b/app/workers/concerns/new_issuable.rb
@@ -27,6 +27,6 @@ module NewIssuable
   # rubocop: enable CodeReuse/ActiveRecord
 
   def log_error(record_class, record_id)
-    Rails.logger.error("#{self.class}: couldn't find #{record_class} with ID=#{record_id}, skipping job")
+    Rails.logger.error("#{self.class}: couldn't find #{record_class} with ID=#{record_id}, skipping job") # rubocop:disable Gitlab/RailsLogger
   end
 end
diff --git a/app/workers/create_gpg_signature_worker.rb b/app/workers/create_gpg_signature_worker.rb
index 7fac7822cf7..e3fb5d479ae 100644
--- a/app/workers/create_gpg_signature_worker.rb
+++ b/app/workers/create_gpg_signature_worker.rb
@@ -22,7 +22,7 @@ class CreateGpgSignatureWorker
     commits.each do |commit|
       Gitlab::Gpg::Commit.new(commit).signature
     rescue => e
-      Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}")
+      Rails.logger.error("Failed to create signature for commit #{commit.id}. Error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
     end
   end
   # rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/delete_user_worker.rb b/app/workers/delete_user_worker.rb
index 4d0295f8d2e..efa8794b214 100644
--- a/app/workers/delete_user_worker.rb
+++ b/app/workers/delete_user_worker.rb
@@ -9,6 +9,6 @@ class DeleteUserWorker
 
     Users::DestroyService.new(current_user).execute(delete_user, options.symbolize_keys)
   rescue Gitlab::Access::AccessDeniedError => e
-    Rails.logger.warn("User could not be destroyed: #{e}")
+    Rails.logger.warn("User could not be destroyed: #{e}") # rubocop:disable Gitlab/RailsLogger
   end
 end
diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb
index c4bcda2da16..e70bf17d5a9 100644
--- a/app/workers/email_receiver_worker.rb
+++ b/app/workers/email_receiver_worker.rb
@@ -16,7 +16,7 @@ class EmailReceiverWorker
   private
 
   def handle_failure(raw, error)
-    Rails.logger.warn("Email can not be processed: #{error}\n\n#{raw}")
+    Rails.logger.warn("Email can not be processed: #{error}\n\n#{raw}") # rubocop:disable Gitlab/RailsLogger
 
     return unless raw.present?
 
diff --git a/app/workers/expire_build_artifacts_worker.rb b/app/workers/expire_build_artifacts_worker.rb
index 251e95c68d5..6f0e0fd33f7 100644
--- a/app/workers/expire_build_artifacts_worker.rb
+++ b/app/workers/expire_build_artifacts_worker.rb
@@ -18,7 +18,7 @@ class ExpireBuildArtifactsWorker
 
   # rubocop: disable CodeReuse/ActiveRecord
   def perform_legacy_artifacts_removal
-    Rails.logger.info 'Scheduling removal of build artifacts'
+    Rails.logger.info 'Scheduling removal of build artifacts' # rubocop:disable Gitlab/RailsLogger
 
     build_ids = Ci::Build.with_expired_artifacts.pluck(:id)
     build_ids = build_ids.map { |build_id| [build_id] }
diff --git a/app/workers/expire_build_instance_artifacts_worker.rb b/app/workers/expire_build_instance_artifacts_worker.rb
index 94426dcf921..71e61dcb878 100644
--- a/app/workers/expire_build_instance_artifacts_worker.rb
+++ b/app/workers/expire_build_instance_artifacts_worker.rb
@@ -12,7 +12,7 @@ class ExpireBuildInstanceArtifactsWorker
 
     return unless build&.project && !build.project.pending_delete
 
-    Rails.logger.info "Removing artifacts for build #{build.id}..."
+    Rails.logger.info "Removing artifacts for build #{build.id}..." # rubocop:disable Gitlab/RailsLogger
     build.erase_erasable_artifacts!
   end
   # rubocop: enable CodeReuse/ActiveRecord
diff --git a/app/workers/git_garbage_collect_worker.rb b/app/workers/git_garbage_collect_worker.rb
index 489d6215774..5499e12e49b 100644
--- a/app/workers/git_garbage_collect_worker.rb
+++ b/app/workers/git_garbage_collect_worker.rb
@@ -24,7 +24,7 @@ class GitGarbageCollectWorker
 
     task = task.to_sym
 
-    ::Projects::GitDeduplicationService.new(project).execute
+    ::Projects::GitDeduplicationService.new(project).execute if task == :gc
 
     gitaly_call(task, project.repository.raw_repository)
 
diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb
index b75e724ca98..a5e22f88a3b 100644
--- a/app/workers/gitlab_usage_ping_worker.rb
+++ b/app/workers/gitlab_usage_ping_worker.rb
@@ -6,10 +6,16 @@ class GitlabUsagePingWorker
   include ApplicationWorker
   include CronjobQueue
 
+  # Retry for up to approximately three hours then give up.
+  sidekiq_options retry: 10, dead: false
+
   def perform
     # Multiple Sidekiq workers could run this. We should only do this at most once a day.
     return unless try_obtain_lease
 
+    # Splay the request over a minute to avoid thundering herd problems.
+    sleep(rand(0.0..60.0).round(3))
+
     SubmitUsagePingService.new.execute
   end
 
diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb
index 48876825564..0c1ca5eb975 100644
--- a/app/workers/namespaces/root_statistics_worker.rb
+++ b/app/workers/namespaces/root_statistics_worker.rb
@@ -9,7 +9,7 @@ module Namespaces
     def perform(namespace_id)
       namespace = Namespace.find(namespace_id)
 
-      return unless update_statistics_enabled_for?(namespace) && namespace.aggregation_scheduled?
+      return unless namespace.aggregation_scheduled?
 
       Namespaces::StatisticsRefresherService.new.execute(namespace)
 
@@ -23,9 +23,5 @@ module Namespaces
     def log_error(namespace_path, error_message)
       Gitlab::SidekiqLogger.error("Namespace statistics can't be updated for #{namespace_path}: #{error_message}")
     end
-
-    def update_statistics_enabled_for?(namespace)
-      Feature.enabled?(:update_statistics_namespace, namespace)
-    end
   end
 end
diff --git a/app/workers/namespaces/schedule_aggregation_worker.rb b/app/workers/namespaces/schedule_aggregation_worker.rb
index a4594b84b13..983ce4bef4a 100644
--- a/app/workers/namespaces/schedule_aggregation_worker.rb
+++ b/app/workers/namespaces/schedule_aggregation_worker.rb
@@ -12,7 +12,7 @@ module Namespaces
       namespace = Namespace.find(namespace_id)
       root_ancestor = namespace.root_ancestor
 
-      return unless update_statistics_enabled_for?(root_ancestor) && !root_ancestor.aggregation_scheduled?
+      return if root_ancestor.aggregation_scheduled?
 
       Namespace::AggregationSchedule.safe_find_or_create_by!(namespace_id: root_ancestor.id)
     rescue ActiveRecord::RecordNotFound
@@ -37,9 +37,5 @@ module Namespaces
     def log_error(root_ancestor_id)
       Gitlab::SidekiqLogger.error("Namespace can't be scheduled for aggregation: #{root_ancestor_id} does not exist")
     end
-
-    def update_statistics_enabled_for?(root_ancestor)
-      Feature.enabled?(:update_statistics_namespace, root_ancestor)
-    end
   end
 end
diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb
index 98f9f45e608..1d1ea926c21 100644
--- a/app/workers/new_note_worker.rb
+++ b/app/workers/new_note_worker.rb
@@ -11,7 +11,7 @@ class NewNoteWorker
       NotificationService.new.new_note(note) unless skip_notification?(note)
       Notes::PostProcessService.new(note).execute
     else
-      Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job")
+      Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job") # rubocop:disable Gitlab/RailsLogger
     end
   end
 
diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb
index 12400d4e025..55ac7cd9b3c 100644
--- a/app/workers/object_storage/migrate_uploads_worker.rb
+++ b/app/workers/object_storage/migrate_uploads_worker.rb
@@ -37,6 +37,7 @@ module ObjectStorage
         end
       end
 
+      # rubocop:disable Gitlab/RailsLogger
       def report!(results)
         success, failures = results.partition(&:success?)
 
@@ -45,6 +46,7 @@ module ObjectStorage
 
         raise MigrationFailures.new(failures.map(&:error)) if failures.any?
       end
+      # rubocop:enable Gitlab/RailsLogger
 
       def header(success, failures)
         _("Migrated %{success_count}/%{total_count} files.") % { success_count: success.count, total_count: success.count + failures.count }
@@ -98,7 +100,7 @@ module ObjectStorage
       report!(results)
     rescue SanityCheckError => e
       # do not retry: the job is insane
-      Rails.logger.warn "#{self.class}: Sanity check error (#{e.message})"
+      Rails.logger.warn "#{self.class}: Sanity check error (#{e.message})" # rubocop:disable Gitlab/RailsLogger
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/app/workers/pages_domain_ssl_renewal_cron_worker.rb b/app/workers/pages_domain_ssl_renewal_cron_worker.rb
index 40c34d29970..e5dde07a648 100644
--- a/app/workers/pages_domain_ssl_renewal_cron_worker.rb
+++ b/app/workers/pages_domain_ssl_renewal_cron_worker.rb
@@ -5,9 +5,9 @@ class PagesDomainSslRenewalCronWorker
   include CronjobQueue
 
   def perform
-    PagesDomain.need_auto_ssl_renewal.find_each do |domain|
-      next unless ::Gitlab::LetsEncrypt.enabled?(domain)
+    return unless ::Gitlab::LetsEncrypt.enabled?
 
+    PagesDomain.need_auto_ssl_renewal.find_each do |domain|
       PagesDomainSslRenewalWorker.perform_async(domain.id)
     end
   end
diff --git a/app/workers/pages_domain_ssl_renewal_worker.rb b/app/workers/pages_domain_ssl_renewal_worker.rb
index b32458ca777..87fd8059946 100644
--- a/app/workers/pages_domain_ssl_renewal_worker.rb
+++ b/app/workers/pages_domain_ssl_renewal_worker.rb
@@ -6,7 +6,7 @@ class PagesDomainSslRenewalWorker
   def perform(domain_id)
     domain = PagesDomain.find_by_id(domain_id)
     return unless domain&.enabled?
-    return unless ::Gitlab::LetsEncrypt.enabled?(domain)
+    return unless ::Gitlab::LetsEncrypt.enabled?
 
     ::PagesDomains::ObtainLetsEncryptCertificateService.new(domain).execute
   end
diff --git a/app/workers/pipeline_process_worker.rb b/app/workers/pipeline_process_worker.rb
index f2aa17acb51..96524d93f8d 100644
--- a/app/workers/pipeline_process_worker.rb
+++ b/app/workers/pipeline_process_worker.rb
@@ -7,9 +7,10 @@ class PipelineProcessWorker
   queue_namespace :pipeline_processing
 
   # rubocop: disable CodeReuse/ActiveRecord
-  def perform(pipeline_id)
-    Ci::Pipeline.find_by(id: pipeline_id)
-      .try(:process!)
+  def perform(pipeline_id, build_ids = nil)
+    Ci::Pipeline.find_by(id: pipeline_id).try do |pipeline|
+      pipeline.process!(build_ids)
+    end
   end
   # rubocop: enable CodeReuse/ActiveRecord
 end
diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb
index dba7837bd12..61d34981458 100644
--- a/app/workers/post_receive.rb
+++ b/app/workers/post_receive.rb
@@ -42,6 +42,9 @@ class PostReceive
     user = identify_user(post_received)
     return false unless user
 
+    # We only need to expire certain caches once per push
+    expire_caches(post_received)
+
     post_received.enum_for(:changes_refs).with_index do |(oldrev, newrev, ref), index|
       service_klass =
         if Gitlab::Git.tag_ref?(ref)
@@ -69,9 +72,34 @@ class PostReceive
     after_project_changes_hooks(post_received, user, refs.to_a, changes)
   end
 
+  # Expire the project, branch, and tag cache once per push. Schedule an
+  # update for the repository size and commit count if necessary.
+  def expire_caches(post_received)
+    project = post_received.project
+
+    project.repository.expire_status_cache if project.empty_repo?
+    project.repository.expire_branches_cache if post_received.includes_branches?
+    project.repository.expire_caches_for_tags if post_received.includes_tags?
+
+    enqueue_repository_cache_update(post_received)
+  end
+
+  def enqueue_repository_cache_update(post_received)
+    stats_to_invalidate = [:repository_size]
+    stats_to_invalidate << :commit_count if post_received.includes_default_branch?
+
+    ProjectCacheWorker.perform_async(
+      post_received.project.id,
+      [],
+      stats_to_invalidate,
+      true
+    )
+  end
+
   def after_project_changes_hooks(post_received, user, refs, changes)
     hook_data = Gitlab::DataBuilder::Repository.update(post_received.project, user, changes, refs)
     SystemHooksService.new.execute_hooks(hook_data, :repository_update_hooks)
+    Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes)
   end
 
   def process_wiki_changes(post_received)
diff --git a/app/workers/process_commit_worker.rb b/app/workers/process_commit_worker.rb
index 3efb5343a96..f6ebe4ab006 100644
--- a/app/workers/process_commit_worker.rb
+++ b/app/workers/process_commit_worker.rb
@@ -2,7 +2,8 @@
 
 # Worker for processing individual commit messages pushed to a repository.
 #
-# Jobs for this worker are scheduled for every commit that is being pushed. As a
+# Jobs for this worker are scheduled for every commit that contains mentionable
+# references in its message and does not exist in the upstream project. As a
 # result of this the workload of this worker should be kept to a bare minimum.
 # Consider using an extra worker if you need to add any extra (and potentially
 # slow) processing of commits.
@@ -19,7 +20,6 @@ class ProcessCommitWorker
     project = Project.find_by(id: project_id)
 
     return unless project
-    return if commit_exists_in_upstream?(project, commit_hash)
 
     user = User.find_by(id: user_id)
 
@@ -77,17 +77,4 @@ class ProcessCommitWorker
 
     Commit.from_hash(hash, project)
   end
-
-  private
-
-  # Avoid reprocessing commits that already exist in the upstream
-  # when project is forked. This will also prevent duplicated system notes.
-  def commit_exists_in_upstream?(project, commit_hash)
-    upstream_project = project.fork_source
-
-    return false unless upstream_project
-
-    commit_id = commit_hash.with_indifferent_access[:id]
-    upstream_project.commit(commit_id).present?
-  end
 end
diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb
index 4e8ea903139..5ac860c93e0 100644
--- a/app/workers/project_cache_worker.rb
+++ b/app/workers/project_cache_worker.rb
@@ -12,13 +12,15 @@ class ProjectCacheWorker
   #         CHANGELOG.
   # statistics - An Array containing columns from ProjectStatistics to
   #              refresh, if empty all columns will be refreshed
+  # refresh_statistics - A boolean that determines whether project statistics should
+  #                     be updated.
   # rubocop: disable CodeReuse/ActiveRecord
-  def perform(project_id, files = [], statistics = [])
+  def perform(project_id, files = [], statistics = [], refresh_statistics = true)
     project = Project.find_by(id: project_id)
 
     return unless project
 
-    update_statistics(project, statistics)
+    update_statistics(project, statistics) if refresh_statistics
 
     return unless project.repository.exists?
 
diff --git a/app/workers/remote_mirror_notification_worker.rb b/app/workers/remote_mirror_notification_worker.rb
index 5bafe8e2046..368abfeda99 100644
--- a/app/workers/remote_mirror_notification_worker.rb
+++ b/app/workers/remote_mirror_notification_worker.rb
@@ -4,7 +4,7 @@ class RemoteMirrorNotificationWorker
   include ApplicationWorker
 
   def perform(remote_mirror_id)
-    remote_mirror = RemoteMirrorFinder.new(id: remote_mirror_id).execute
+    remote_mirror = RemoteMirror.find_by_id(remote_mirror_id)
 
     # We check again if there's an error because a newer run since this job was
     # fired could've completed successfully.
diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb
index a9b88a133be..35e9c58eb13 100644
--- a/app/workers/repository_fork_worker.rb
+++ b/app/workers/repository_fork_worker.rb
@@ -35,7 +35,7 @@ class RepositoryForkWorker
   def start_fork(project)
     return true if start(project.import_state)
 
-    Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while forking.")
+    Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while forking.") # rubocop:disable Gitlab/RailsLogger
     false
   end
 end
diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb
index 59691f48a39..dff9c8f50bf 100644
--- a/app/workers/repository_import_worker.rb
+++ b/app/workers/repository_import_worker.rb
@@ -36,7 +36,7 @@ class RepositoryImportWorker
   def start_import
     return true if start(project.import_state)
 
-    Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while importing.")
+    Rails.logger.info("Project #{project.full_path} was in inconsistent state (#{project.import_status}) while importing.") # rubocop:disable Gitlab/RailsLogger
     false
   end
 
diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb
index c0bae08ba85..d13c7641eb3 100644
--- a/app/workers/repository_update_remote_mirror_worker.rb
+++ b/app/workers/repository_update_remote_mirror_worker.rb
@@ -1,50 +1,53 @@
 # frozen_string_literal: true
 
 class RepositoryUpdateRemoteMirrorWorker
-  UpdateAlreadyInProgressError = Class.new(StandardError)
   UpdateError = Class.new(StandardError)
 
   include ApplicationWorker
+  include Gitlab::ExclusiveLeaseHelpers
 
   sidekiq_options retry: 3, dead: false
 
-  sidekiq_retry_in { |count| 30 * count }
+  LOCK_WAIT_TIME = 30.seconds
+  MAX_TRIES = 3
 
-  sidekiq_retries_exhausted do |msg, _|
-    Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
-  end
-
-  def perform(remote_mirror_id, scheduled_time)
-    remote_mirror = RemoteMirrorFinder.new(id: remote_mirror_id).execute
+  def perform(remote_mirror_id, scheduled_time, tries = 0)
+    remote_mirror = RemoteMirror.find_by_id(remote_mirror_id)
+    return unless remote_mirror
     return if remote_mirror.updated_since?(scheduled_time)
 
-    raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
+    # If the update is already running, wait for it to finish before running again
+    # This will wait for a total of 90 seconds in 3 steps
+    in_lock(remote_mirror_update_lock(remote_mirror.id),
+            retries: 3,
+            ttl: remote_mirror.max_runtime,
+            sleep_sec: LOCK_WAIT_TIME) do
+      update_mirror(remote_mirror, scheduled_time, tries)
+    end
+  rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError
+    # If an update runs longer than 1.5 minutes, we'll reschedule it
+    # with a backoff. The next run will check if the previous update would
+    # include the changes that triggered this update and become a no-op.
+    self.class.perform_in(remote_mirror.backoff_delay, remote_mirror.id, scheduled_time, tries)
+  end
 
-    remote_mirror.update_start
+  private
 
-    project = remote_mirror.project
+  def update_mirror(mirror, scheduled_time, tries)
+    project = mirror.project
     current_user = project.creator
-    result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
-    raise UpdateError, result[:message] if result[:status] == :error
-
-    remote_mirror.update_finish
-  rescue UpdateAlreadyInProgressError
-    raise
-  rescue UpdateError => ex
-    fail_remote_mirror(remote_mirror, ex.message)
-    raise
-  rescue => ex
-    return unless remote_mirror
+    result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(mirror, tries)
 
-    fail_remote_mirror(remote_mirror, ex.message)
-    raise UpdateError, "#{ex.class}: #{ex.message}"
+    if result[:status] == :error && mirror.to_retry?
+      schedule_retry(mirror, scheduled_time, tries)
+    end
   end
 
-  private
-
-  def fail_remote_mirror(remote_mirror, message)
-    remote_mirror.mark_as_failed(message)
+  def remote_mirror_update_lock(mirror_id)
+    [self.class.name, mirror_id].join(':')
+  end
 
-    Rails.logger.error(message)
+  def schedule_retry(mirror, scheduled_time, tries)
+    self.class.perform_in(mirror.backoff_delay, mirror.id, scheduled_time, tries + 1)
   end
 end
diff --git a/app/workers/run_pipeline_schedule_worker.rb b/app/workers/run_pipeline_schedule_worker.rb
index 43e0b9db22f..351850e53cb 100644
--- a/app/workers/run_pipeline_schedule_worker.rb
+++ b/app/workers/run_pipeline_schedule_worker.rb
@@ -30,6 +30,7 @@ class RunPipelineScheduleWorker
 
   private
 
+  # rubocop:disable Gitlab/RailsLogger
   def error(schedule, error)
     failed_creation_counter.increment
 
@@ -41,6 +42,7 @@ class RunPipelineScheduleWorker
                        issue_url: 'https://gitlab.com/gitlab-org/gitlab-ce/issues/41231',
                        extra: { schedule_id: schedule.id })
   end
+  # rubocop:enable Gitlab/RailsLogger
 
   def failed_creation_counter
     @failed_creation_counter ||=
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb
index 25809f68080..30fba038937 100644
--- a/app/workers/stuck_ci_jobs_worker.rb
+++ b/app/workers/stuck_ci_jobs_worker.rb
@@ -14,7 +14,7 @@ class StuckCiJobsWorker
   def perform
     return unless try_obtain_lease
 
-    Rails.logger.info "#{self.class}: Cleaning stuck builds"
+    Rails.logger.info "#{self.class}: Cleaning stuck builds" # rubocop:disable Gitlab/RailsLogger
 
     drop :running, BUILD_RUNNING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
     drop :pending, BUILD_PENDING_OUTDATED_TIMEOUT, 'ci_builds.updated_at < ?', :stuck_or_timeout_failure
@@ -66,7 +66,7 @@ class StuckCiJobsWorker
   # rubocop: enable CodeReuse/ActiveRecord
 
   def drop_build(type, build, status, timeout, reason)
-    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})"
+    Rails.logger.info "#{self.class}: Dropping #{type} build #{build.id} for runner #{build.runner_id} (status: #{status}, timeout: #{timeout}, reason: #{reason})" # rubocop:disable Gitlab/RailsLogger
     Gitlab::OptimisticLocking.retry_lock(build, 3) do |b|
       b.drop(reason)
     end
diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
index c8a186ba4ce..a9ff5b22b25 100644
--- a/app/workers/stuck_import_jobs_worker.rb
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -38,7 +38,7 @@ class StuckImportJobsWorker
     completed_import_states = enqueued_import_states_with_jid.where(id: completed_import_state_ids)
 
     completed_import_state_jids = completed_import_states.map { |import_state| import_state.jid }.join(', ')
-    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_import_state_jids}")
+    Rails.logger.info("Marked stuck import jobs as failed. JIDs: #{completed_import_state_jids}") # rubocop:disable Gitlab/RailsLogger
 
     completed_import_states.each do |import_state|
       import_state.mark_as_failed(error_message)
diff --git a/app/workers/stuck_merge_jobs_worker.rb b/app/workers/stuck_merge_jobs_worker.rb
index f34ed6c4844..e840ae47421 100644
--- a/app/workers/stuck_merge_jobs_worker.rb
+++ b/app/workers/stuck_merge_jobs_worker.rb
@@ -5,7 +5,7 @@ class StuckMergeJobsWorker
   include CronjobQueue
 
   def self.logger
-    Rails.logger
+    Rails.logger # rubocop:disable Gitlab/RailsLogger
   end
 
   # rubocop: disable CodeReuse/ActiveRecord
diff --git a/app/workers/trending_projects_worker.rb b/app/workers/trending_projects_worker.rb
index 3297a1fe3d0..55b599ba38f 100644
--- a/app/workers/trending_projects_worker.rb
+++ b/app/workers/trending_projects_worker.rb
@@ -5,7 +5,7 @@ class TrendingProjectsWorker
   include CronjobQueue
 
   def perform
-    Rails.logger.info('Refreshing trending projects')
+    Rails.logger.info('Refreshing trending projects') # rubocop:disable Gitlab/RailsLogger
 
     TrendingProject.refresh!
   end
diff --git a/app/workers/update_merge_requests_worker.rb b/app/workers/update_merge_requests_worker.rb
index c7213df652a..6c0e472e05a 100644
--- a/app/workers/update_merge_requests_worker.rb
+++ b/app/workers/update_merge_requests_worker.rb
@@ -27,7 +27,7 @@ class UpdateMergeRequestsWorker
       "ref=#{ref}"
     ].join(',')
 
-    Rails.logger.info("UpdateMergeRequestsWorker#perform #{args_log}") if time.real > LOG_TIME_THRESHOLD
+    Rails.logger.info("UpdateMergeRequestsWorker#perform #{args_log}") if time.real > LOG_TIME_THRESHOLD # rubocop:disable Gitlab/RailsLogger
   end
   # rubocop: enable CodeReuse/ActiveRecord
 end
diff --git a/app/workers/upload_checksum_worker.rb b/app/workers/upload_checksum_worker.rb
index 2a0536106d7..834dcaa435d 100644
--- a/app/workers/upload_checksum_worker.rb
+++ b/app/workers/upload_checksum_worker.rb
@@ -8,6 +8,6 @@ class UploadChecksumWorker
     upload.calculate_checksum!
     upload.save!
   rescue ActiveRecord::RecordNotFound
-    Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping")
+    Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping") # rubocop:disable Gitlab/RailsLogger
   end
 end