Add latest changes from gitlab-org/gitlab@master

5 files changed, 103 insertions, 18 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 66b5214cfcb..710998dcd1a 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -38,6 +38,9 @@
 - gcp_cluster:cluster_patch_app
 - gcp_cluster:cluster_upgrade_app
 - gcp_cluster:cluster_provision
+- gcp_cluster:clusters_cleanup_app
+- gcp_cluster:clusters_cleanup_project_namespace
+- gcp_cluster:clusters_cleanup_service_account
 - gcp_cluster:cluster_wait_for_app_installation
 - gcp_cluster:wait_for_cluster_creation
 - gcp_cluster:cluster_wait_for_ingress_ip_address
diff --git a/app/workers/clusters/cleanup/app_worker.rb b/app/workers/clusters/cleanup/app_worker.rb
index 1eedf510ba1..8b2fddd3164 100644
--- a/app/workers/clusters/cleanup/app_worker.rb
+++ b/app/workers/clusters/cleanup/app_worker.rb
@@ -3,13 +3,16 @@
 module Clusters
   module Cleanup
     class AppWorker
-      include ApplicationWorker
-      include ClusterQueue
-      include ClusterApplications
+      include ClusterCleanupMethods
 
-      # TODO: Merge with https://gitlab.com/gitlab-org/gitlab/merge_requests/16954
-      # We're splitting the above MR in smaller chunks to facilitate reviews
-      def perform
+      def perform(cluster_id, execution_count = 0)
+        Clusters::Cluster.with_persisted_applications.find_by_id(cluster_id).try do |cluster|
+          break unless cluster.cleanup_uninstalling_applications?
+
+          break exceeded_execution_limit(cluster) if exceeded_execution_limit?(execution_count)
+
+          ::Clusters::Cleanup::AppService.new(cluster, execution_count).execute
+        end
       end
     end
   end
diff --git a/app/workers/clusters/cleanup/project_namespace_worker.rb b/app/workers/clusters/cleanup/project_namespace_worker.rb
index 09f2abf5d8a..8a7fbf0fde7 100644
--- a/app/workers/clusters/cleanup/project_namespace_worker.rb
+++ b/app/workers/clusters/cleanup/project_namespace_worker.rb
@@ -3,13 +3,16 @@
 module Clusters
   module Cleanup
     class ProjectNamespaceWorker
-      include ApplicationWorker
-      include ClusterQueue
-      include ClusterApplications
+      include ClusterCleanupMethods
 
-      # TODO: Merge with https://gitlab.com/gitlab-org/gitlab/merge_requests/16954
-      # We're splitting the above MR in smaller chunks to facilitate reviews
-      def perform
+      def perform(cluster_id, execution_count = 0)
+        Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
+          break unless cluster.cleanup_removing_project_namespaces?
+
+          break exceeded_execution_limit(cluster) if exceeded_execution_limit?(execution_count)
+
+          Clusters::Cleanup::ProjectNamespaceService.new(cluster, execution_count).execute
+        end
       end
     end
   end
diff --git a/app/workers/clusters/cleanup/service_account_worker.rb b/app/workers/clusters/cleanup/service_account_worker.rb
index fab6318a807..95de56d8ebe 100644
--- a/app/workers/clusters/cleanup/service_account_worker.rb
+++ b/app/workers/clusters/cleanup/service_account_worker.rb
@@ -3,13 +3,14 @@
 module Clusters
   module Cleanup
     class ServiceAccountWorker
-      include ApplicationWorker
-      include ClusterQueue
-      include ClusterApplications
+      include ClusterCleanupMethods
 
-      # TODO: Merge with https://gitlab.com/gitlab-org/gitlab/merge_requests/16954
-      # We're splitting the above MR in smaller chunks to facilitate reviews
-      def perform
+      def perform(cluster_id)
+        Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
+          break unless cluster.cleanup_removing_service_account?
+
+          Clusters::Cleanup::ServiceAccountService.new(cluster).execute
+        end
       end
     end
   end
diff --git a/app/workers/concerns/cluster_cleanup_methods.rb b/app/workers/concerns/cluster_cleanup_methods.rb
new file mode 100644
index 00000000000..04fa4d69666
--- /dev/null
+++ b/app/workers/concerns/cluster_cleanup_methods.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# Concern for setting Sidekiq settings for the various GitLab ObjectStorage workers.
+module ClusterCleanupMethods
+  extend ActiveSupport::Concern
+
+  include ApplicationWorker
+  include ClusterQueue
+
+  DEFAULT_EXECUTION_LIMIT = 10
+  ExceededExecutionLimitError = Class.new(StandardError)
+
+  included do
+    worker_has_external_dependencies!
+
+    sidekiq_options retry: 3
+
+    sidekiq_retries_exhausted do |msg, error|
+      cluster_id = msg['args'][0]
+
+      cluster = Clusters::Cluster.find_by_id(cluster_id)
+
+      cluster.make_cleanup_errored!("#{self.class.name} retried too many times") if cluster
+
+      logger = Gitlab::Kubernetes::Logger.build
+
+      logger.error({
+        exception: error,
+        cluster_id: cluster_id,
+        class_name: msg['class'],
+        event: :sidekiq_retries_exhausted,
+        message: msg['error_message']
+      })
+    end
+  end
+
+  private
+
+  # Override this method to customize the execution_limit
+  def execution_limit
+    DEFAULT_EXECUTION_LIMIT
+  end
+
+  def exceeded_execution_limit?(execution_count)
+    execution_count >= execution_limit
+  end
+
+  def logger
+    @logger ||= Gitlab::Kubernetes::Logger.build
+  end
+
+  def exceeded_execution_limit(cluster)
+    log_exceeded_execution_limit_error(cluster)
+
+    cluster.make_cleanup_errored!("#{self.class.name} exceeded the execution limit")
+  end
+
+  def cluster_applications_and_status(cluster)
+    cluster.persisted_applications
+      .map { |application| "#{application.name}:#{application.status_name}" }
+      .join(",")
+  end
+
+  def log_exceeded_execution_limit_error(cluster)
+    logger.error({
+      exception: ExceededExecutionLimitError.name,
+      cluster_id: cluster.id,
+      class_name: self.class.name,
+      cleanup_status: cluster.cleanup_status_name,
+      applications: cluster_applications_and_status(cluster),
+      event: :failed_to_remove_cluster_and_resources,
+      message: "exceeded execution limit of #{execution_limit} tries"
+    })
+  end
+end