Merge remote-tracking branch 'origin/master' into 38759-fetch-available-parameters-directly-from-gke-when-creating-a-cluster

12 files changed, 169 insertions, 123 deletions

diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb
index bec0a003a1c..044e470141e 100644
--- a/app/workers/admin_email_worker.rb
+++ b/app/workers/admin_email_worker.rb
@@ -3,6 +3,12 @@ class AdminEmailWorker
   include CronjobQueue
 
   def perform
+    send_repository_check_mail if Gitlab::CurrentSettings.repository_checks_enabled
+  end
+
+  private
+
+  def send_repository_check_mail
     repository_check_failed_count = Project.where(last_repository_check_failed: true).count
     return if repository_check_failed_count.zero?
 
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index c5cb2bc0b31..80b488f31f2 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -51,6 +51,7 @@
 - pipeline_creation:create_pipeline
 - pipeline_creation:run_pipeline_schedule
 - pipeline_background:archive_trace
+- pipeline_background:ci_build_trace_chunk_flush
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
 - pipeline_default:pipeline_metrics
@@ -105,9 +106,11 @@
 - rebase
 - repository_fork
 - repository_import
+- repository_remove_remote
 - storage_migrator
 - system_hook_push
 - update_merge_requests
 - update_user_activity
 - upload_checksum
 - web_hook
+- repository_update_remote_mirror
diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb
new file mode 100644
index 00000000000..218d6688bd9
--- /dev/null
+++ b/app/workers/ci/build_trace_chunk_flush_worker.rb
@@ -0,0 +1,12 @@
+module Ci
+  class BuildTraceChunkFlushWorker
+    include ApplicationWorker
+    include PipelineBackgroundQueue
+
+    def perform(build_trace_chunk_id)
+      ::Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk|
+        build_trace_chunk.use_database!
+      end
+    end
+  end
+end
diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb
index f7f498af840..8d708e15a66 100644
--- a/app/workers/gitlab/github_import/advance_stage_worker.rb
+++ b/app/workers/gitlab/github_import/advance_stage_worker.rb
@@ -63,11 +63,10 @@ module Gitlab
       end
 
       def find_project(id)
-        # We only care about the import JID so we can refresh it. We also only
-        # want the project if it hasn't been marked as failed yet. It's possible
-        # the import gets marked as stuck when jobs of the current stage failed
-        # somehow.
-        Project.select(:import_jid).import_started.find_by(id: id)
+        # TODO: Only select the JID
+        # This is due to the fact that the JID could be present in either the project record or
+        # its associated import_state record
+        Project.import_started.find_by(id: id)
       end
     end
   end
diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb
index 7108b531bc2..68d2c5c4331 100644
--- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb
+++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb
@@ -31,7 +31,10 @@ module Gitlab
       end
 
       def find_project(id)
-        Project.select(:import_jid).import_started.find_by(id: id)
+        # TODO: Only select the JID
+        # This is due to the fact that the JID could be present in either the project record or
+        # its associated import_state record
+        Project.import_started.find_by(id: id)
       end
     end
   end
diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb
index b925741934a..67c54fbf10e 100644
--- a/app/workers/new_note_worker.rb
+++ b/app/workers/new_note_worker.rb
@@ -5,7 +5,7 @@ class NewNoteWorker
   # old `NewNoteWorker` jobs (can remove later)
   def perform(note_id, _params = {})
     if note = Note.find_by(id: note_id)
-      NotificationService.new.new_note(note) if note.can_create_notification?
+      NotificationService.new.new_note(note)
       Notes::PostProcessService.new(note).execute
     else
       Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job")
diff --git a/app/workers/object_storage/migrate_uploads_worker.rb b/app/workers/object_storage/migrate_uploads_worker.rb
index a6b2c251254..a3ecfa8e711 100644
--- a/app/workers/object_storage/migrate_uploads_worker.rb
+++ b/app/workers/object_storage/migrate_uploads_worker.rb
@@ -9,85 +9,6 @@ module ObjectStorage
 
     SanityCheckError = Class.new(StandardError)
 
-    class Upload < ActiveRecord::Base
-      # Upper limit for foreground checksum processing
-      CHECKSUM_THRESHOLD = 100.megabytes
-
-      belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
-
-      validates :size, presence: true
-      validates :path, presence: true
-      validates :model, presence: true
-      validates :uploader, presence: true
-
-      before_save  :calculate_checksum!, if: :foreground_checksummable?
-      after_commit :schedule_checksum,   if: :checksummable?
-
-      scope :stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) }
-      scope :stored_remotely, -> { where(store: ObjectStorage::Store::REMOTE) }
-
-      def self.hexdigest(path)
-        Digest::SHA256.file(path).hexdigest
-      end
-
-      def absolute_path
-        raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local?
-        return path unless relative_path?
-
-        uploader_class.absolute_path(self)
-      end
-
-      def calculate_checksum!
-        self.checksum = nil
-        return unless checksummable?
-
-        self.checksum = self.class.hexdigest(absolute_path)
-      end
-
-      def build_uploader(mounted_as = nil)
-        uploader_class.new(model, mounted_as).tap do |uploader|
-          uploader.upload = self
-          uploader.retrieve_from_store!(identifier)
-        end
-      end
-
-      def exist?
-        File.exist?(absolute_path)
-      end
-
-      def local?
-        return true if store.nil?
-
-        store == ObjectStorage::Store::LOCAL
-      end
-
-      private
-
-      def checksummable?
-        checksum.nil? && local? && exist?
-      end
-
-      def foreground_checksummable?
-        checksummable? && size <= CHECKSUM_THRESHOLD
-      end
-
-      def schedule_checksum
-        UploadChecksumWorker.perform_async(id)
-      end
-
-      def relative_path?
-        !path.start_with?('/')
-      end
-
-      def identifier
-        File.basename(path)
-      end
-
-      def uploader_class
-        Object.const_get(uploader)
-      end
-    end
-
     class MigrationResult
       attr_reader :upload
       attr_accessor :error
diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb
index 76688cf51c1..72f0a9b0619 100644
--- a/app/workers/repository_check/batch_worker.rb
+++ b/app/workers/repository_check/batch_worker.rb
@@ -4,8 +4,11 @@ module RepositoryCheck
     include CronjobQueue
 
     RUN_TIME = 3600
+    BATCH_SIZE = 10_000
 
     def perform
+      return unless Gitlab::CurrentSettings.repository_checks_enabled
+
       start = Time.now
 
       # This loop will break after a little more than one hour ('a little
@@ -15,7 +18,6 @@ module RepositoryCheck
       # check, only one (or two) will be checked at a time.
       project_ids.each do |project_id|
         break if Time.now - start >= RUN_TIME
-        break unless current_settings.repository_checks_enabled
 
         next unless try_obtain_lease(project_id)
 
@@ -31,12 +33,20 @@ module RepositoryCheck
     # getting ID's from Postgres is not terribly slow, and because no user
     # has to sit and wait for this query to finish.
     def project_ids
-      limit = 10_000
-      never_checked_projects = Project.where('last_repository_check_at IS NULL AND created_at < ?', 24.hours.ago)
-        .limit(limit).pluck(:id)
-      old_check_projects = Project.where('last_repository_check_at < ?', 1.month.ago)
-        .reorder('last_repository_check_at ASC').limit(limit).pluck(:id)
-      never_checked_projects + old_check_projects
+      never_checked_project_ids(BATCH_SIZE) + old_checked_project_ids(BATCH_SIZE)
+    end
+
+    def never_checked_project_ids(batch_size)
+      Project.where(last_repository_check_at: nil)
+        .where('created_at < ?', 24.hours.ago)
+        .limit(batch_size).pluck(:id)
+    end
+
+    def old_checked_project_ids(batch_size)
+      Project.where.not(last_repository_check_at: nil)
+        .where('last_repository_check_at < ?', 1.month.ago)
+        .reorder(last_repository_check_at: :asc)
+        .limit(batch_size).pluck(:id)
     end
 
     def try_obtain_lease(id)
@@ -47,16 +57,5 @@ module RepositoryCheck
         timeout: 24.hours
       ).try_obtain
     end
-
-    def current_settings
-      # No caching of the settings! If we cache them and an admin disables
-      # this feature, an active RepositoryCheckWorker would keep going for up
-      # to 1 hour after the feature was disabled.
-      if Rails.env.test?
-        Gitlab::CurrentSettings.fake_application_settings
-      else
-        ApplicationSetting.current
-      end
-    end
   end
 end
diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb
index 116bc185b38..3cffb8b14e4 100644
--- a/app/workers/repository_check/single_repository_worker.rb
+++ b/app/workers/repository_check/single_repository_worker.rb
@@ -5,27 +5,34 @@ module RepositoryCheck
 
     def perform(project_id)
       project = Project.find(project_id)
+      healthy = project_healthy?(project)
+
+      update_repository_check_status(project, healthy)
+    end
+
+    private
+
+    def update_repository_check_status(project, healthy)
       project.update_columns(
-        last_repository_check_failed: !check(project),
+        last_repository_check_failed: !healthy,
         last_repository_check_at: Time.now
       )
     end
 
-    private
+    def project_healthy?(project)
+      repo_healthy?(project) && wiki_repo_healthy?(project)
+    end
 
-    def check(project)
-      if has_pushes?(project) && !git_fsck(project.repository)
-        false
-      elsif project.wiki_enabled?
-        # Historically some projects never had their wiki repos initialized;
-        # this happens on project creation now. Let's initialize an empty repo
-        # if it is not already there.
-        project.create_wiki
+    def repo_healthy?(project)
+      return true unless has_changes?(project)
 
-        git_fsck(project.wiki.repository)
-      else
-        true
-      end
+      git_fsck(project.repository)
+    end
+
+    def wiki_repo_healthy?(project)
+      return true unless has_wiki_changes?(project)
+
+      git_fsck(project.wiki.repository)
     end
 
     def git_fsck(repository)
@@ -39,8 +46,19 @@ module RepositoryCheck
       false
     end
 
-    def has_pushes?(project)
+    def has_changes?(project)
       Project.with_push.exists?(project.id)
     end
+
+    def has_wiki_changes?(project)
+      return false unless project.wiki_enabled?
+
+      # Historically some projects never had their wiki repos initialized;
+      # this happens on project creation now. Let's initialize an empty repo
+      # if it is not already there.
+      return false unless project.create_wiki
+
+      has_changes?(project)
+    end
   end
 end
diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb
new file mode 100644
index 00000000000..1c19b604b77
--- /dev/null
+++ b/app/workers/repository_remove_remote_worker.rb
@@ -0,0 +1,35 @@
+class RepositoryRemoveRemoteWorker
+  include ApplicationWorker
+  include ExclusiveLeaseGuard
+
+  LEASE_TIMEOUT = 1.hour
+
+  attr_reader :project, :remote_name
+
+  def perform(project_id, remote_name)
+    @remote_name = remote_name
+    @project = Project.find_by_id(project_id)
+
+    return unless @project
+
+    logger.info("Removing remote #{remote_name} from project #{project.id}")
+
+    try_obtain_lease do
+      remove_remote = @project.repository.remove_remote(remote_name)
+
+      if remove_remote
+        logger.info("Remote #{remote_name} was successfully removed from project #{project.id}")
+      else
+        logger.error("Could not remove remote #{remote_name} from project #{project.id}")
+      end
+    end
+  end
+
+  def lease_timeout
+    LEASE_TIMEOUT
+  end
+
+  def lease_key
+    "remove_remote_#{project.id}_#{remote_name}"
+  end
+end
diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb
new file mode 100644
index 00000000000..bb963979e88
--- /dev/null
+++ b/app/workers/repository_update_remote_mirror_worker.rb
@@ -0,0 +1,49 @@
+class RepositoryUpdateRemoteMirrorWorker
+  UpdateAlreadyInProgressError = Class.new(StandardError)
+  UpdateError = Class.new(StandardError)
+
+  include ApplicationWorker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options retry: 3, dead: false
+
+  sidekiq_retry_in { |count| 30 * count }
+
+  sidekiq_retries_exhausted do |msg, _|
+    Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
+  end
+
+  def perform(remote_mirror_id, scheduled_time)
+    remote_mirror = RemoteMirror.find(remote_mirror_id)
+    return if remote_mirror.updated_since?(scheduled_time)
+
+    raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
+
+    remote_mirror.update_start
+
+    project = remote_mirror.project
+    current_user = project.creator
+    result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
+    raise UpdateError, result[:message] if result[:status] == :error
+
+    remote_mirror.update_finish
+  rescue UpdateAlreadyInProgressError
+    raise
+  rescue UpdateError => ex
+    fail_remote_mirror(remote_mirror, ex.message)
+    raise
+  rescue => ex
+    return unless remote_mirror
+
+    fail_remote_mirror(remote_mirror, ex.message)
+    raise UpdateError, "#{ex.class}: #{ex.message}"
+  end
+
+  private
+
+  def fail_remote_mirror(remote_mirror, message)
+    remote_mirror.mark_as_failed(message)
+
+    Rails.logger.error(message)
+  end
+end
diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb
index fbb14efc525..6fdd7592e74 100644
--- a/app/workers/stuck_import_jobs_worker.rb
+++ b/app/workers/stuck_import_jobs_worker.rb
@@ -22,7 +22,8 @@ class StuckImportJobsWorker
   end
 
   def mark_projects_with_jid_as_failed!
-    jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h
+    # TODO: Rollback this change to use SQL through #pluck
+    jids_and_ids = enqueued_projects_with_jid.map { |project| [project.import_jid, project.id] }.to_h
 
     # Find the jobs that aren't currently running or that exceeded the threshold.
     completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys)
@@ -42,15 +43,15 @@ class StuckImportJobsWorker
   end
 
   def enqueued_projects
-    Project.with_import_status(:scheduled, :started)
+    Project.joins_import_state.where("(import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started')")
   end
 
   def enqueued_projects_with_jid
-    enqueued_projects.where.not(import_jid: nil)
+    enqueued_projects.where.not("import_state.jid IS NULL AND projects.import_jid IS NULL")
   end
 
   def enqueued_projects_without_jid
-    enqueued_projects.where(import_jid: nil)
+    enqueued_projects.where("import_state.jid IS NULL AND projects.import_jid IS NULL")
   end
 
   def error_message