Services to uninstall cluster application

+ to monitor progress of uninstallation pod

7 files changed, 348 insertions, 0 deletions

diff --git a/app/services/clusters/applications/check_uninstall_progress_service.rb b/app/services/clusters/applications/check_uninstall_progress_service.rb
new file mode 100644
index 00000000000..2a2594a30c8
--- /dev/null
+++ b/app/services/clusters/applications/check_uninstall_progress_service.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Applications
+    class CheckUninstallProgressService < BaseHelmService
+      def execute
+        return unless app.uninstalling?
+
+        case installation_phase
+        when Gitlab::Kubernetes::Pod::SUCCEEDED
+          on_success
+        when Gitlab::Kubernetes::Pod::FAILED
+          on_failed
+        else
+          check_timeout
+        end
+      rescue Kubeclient::HttpError => e
+        log_error(e)
+
+        app.make_errored!("Kubernetes error: #{e.error_code}")
+      end
+
+      private
+
+      def on_success
+        app.make_uninstalled!
+      ensure
+        remove_installation_pod
+      end
+
+      def on_failed
+        app.make_errored!("Operation failed. Check pod logs for #{pod_name} for more details.")
+      end
+
+      def check_timeout
+        if timeouted?
+          begin
+            app.make_errored!("Operation timed out. Check pod logs for #{pod_name} for more details.")
+          end
+        else
+          WaitForUninstallAppWorker.perform_in(WaitForUninstallAppWorker::INTERVAL, app.name, app.id)
+        end
+      end
+
+      def pod_name
+        app.uninstall_command.pod_name
+      end
+
+      def timeouted?
+        Time.now.utc - app.updated_at.to_time.utc > WaitForUninstallAppWorker::TIMEOUT
+      end
+
+      def remove_installation_pod
+        helm_api.delete_pod!(pod_name)
+      end
+
+      def installation_phase
+        helm_api.status(pod_name)
+      end
+    end
+  end
+end
diff --git a/app/services/clusters/applications/uninstall_service.rb b/app/services/clusters/applications/uninstall_service.rb
new file mode 100644
index 00000000000..50c8d806c14
--- /dev/null
+++ b/app/services/clusters/applications/uninstall_service.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Applications
+    class UninstallService < BaseHelmService
+      def execute
+        return unless app.scheduled?
+
+        app.make_uninstalling!
+        uninstall
+      end
+
+      private
+
+      def uninstall
+        helm_api.uninstall(app.uninstall_command)
+
+        Clusters::Applications::WaitForUninstallAppWorker.perform_in(
+          Clusters::Applications::WaitForUninstallAppWorker::INTERVAL, app.name, app.id)
+      rescue Kubeclient::HttpError => e
+        log_error(e)
+        app.make_errored!("Kubernetes error: #{e.error_code}")
+      rescue StandardError => e
+        log_error(e)
+        app.make_errored!('Failed to uninstall.')
+      end
+    end
+  end
+end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index d01bbbf269e..0d14d313d21 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -32,6 +32,7 @@
 - gcp_cluster:cluster_wait_for_ingress_ip_address
 - gcp_cluster:cluster_configure
 - gcp_cluster:cluster_project_configure
+- gcp_cluster:clusters_applications_wait_for_uninstall_app
 
 - github_import_advance_stage
 - github_importer:github_import_import_diff_note
diff --git a/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb
new file mode 100644
index 00000000000..163c99d3c3c
--- /dev/null
+++ b/app/workers/clusters/applications/wait_for_uninstall_app_worker.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Applications
+    class WaitForUninstallAppWorker
+      include ApplicationWorker
+      include ClusterQueue
+      include ClusterApplications
+
+      INTERVAL = 10.seconds
+      TIMEOUT = 20.minutes
+
+      def perform(app_name, app_id)
+        find_application(app_name, app_id) do |app|
+          Clusters::Applications::CheckUninstallProgressService.new(app).execute
+        end
+      end
+    end
+  end
+end
diff --git a/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb b/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb
new file mode 100644
index 00000000000..ccae7fd133f
--- /dev/null
+++ b/spec/services/clusters/applications/check_uninstall_progress_service_spec.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Applications::CheckUninstallProgressService do
+  RESCHEDULE_PHASES = Gitlab::Kubernetes::Pod::PHASES - [Gitlab::Kubernetes::Pod::SUCCEEDED, Gitlab::Kubernetes::Pod::FAILED].freeze
+
+  let(:application) { create(:clusters_applications_helm, :uninstalling) }
+  let(:service) { described_class.new(application) }
+  let(:phase) { Gitlab::Kubernetes::Pod::UNKNOWN }
+  let(:errors) { nil }
+  let(:worker_class) { Clusters::Applications::WaitForUninstallAppWorker }
+
+  shared_examples 'a not yet terminated installation' do |a_phase|
+    let(:phase) { a_phase }
+
+    before do
+      expect(service).to receive(:installation_phase).once.and_return(phase)
+    end
+
+    context "when phase is #{a_phase}" do
+      context 'when not timeouted' do
+        it 'reschedule a new check' do
+          expect(worker_class).to receive(:perform_in).once
+          expect(service).not_to receive(:remove_installation_pod)
+
+          expect do
+            service.execute
+
+            application.reload
+          end.not_to change(application, :status)
+
+          expect(application.status_reason).to be_nil
+        end
+      end
+    end
+  end
+
+  before do
+    allow(service).to receive(:installation_errors).and_return(errors)
+    allow(service).to receive(:remove_installation_pod).and_return(nil)
+  end
+
+  context 'when application is installing' do
+    RESCHEDULE_PHASES.each { |phase| it_behaves_like 'a not yet terminated installation', phase }
+
+    context 'when installation POD succeeded' do
+      let(:phase) { Gitlab::Kubernetes::Pod::SUCCEEDED }
+      before do
+        expect(service).to receive(:installation_phase).once.and_return(phase)
+      end
+
+      it 'removes the installation POD' do
+        expect(service).to receive(:remove_installation_pod).once
+
+        service.execute
+      end
+
+      it 'make the application installed' do
+        expect(worker_class).not_to receive(:perform_in)
+
+        service.execute
+
+        expect(application).to be_uninstalled
+        expect(application.status_reason).to be_nil
+      end
+    end
+
+    context 'when installation POD failed' do
+      let(:phase) { Gitlab::Kubernetes::Pod::FAILED }
+      let(:errors) { 'test installation failed' }
+
+      before do
+        expect(service).to receive(:installation_phase).once.and_return(phase)
+      end
+
+      it 'make the application errored' do
+        service.execute
+
+        expect(application).to be_uninstall_errored
+        expect(application.status_reason).to eq('Operation failed. Check pod logs for uninstall-helm for more details.')
+      end
+    end
+
+    context 'when timed out' do
+      let(:application) { create(:clusters_applications_helm, :timeouted, :uninstalling) }
+
+      before do
+        expect(service).to receive(:installation_phase).once.and_return(phase)
+      end
+
+      it 'make the application errored' do
+        expect(worker_class).not_to receive(:perform_in)
+
+        service.execute
+
+        expect(application).to be_uninstall_errored
+        expect(application.status_reason).to eq('Operation timed out. Check pod logs for uninstall-helm for more details.')
+      end
+    end
+
+    context 'when installation raises a Kubeclient::HttpError' do
+      let(:cluster) { create(:cluster, :provided_by_user, :project) }
+      let(:logger) { service.send(:logger) }
+      let(:error) { Kubeclient::HttpError.new(401, 'Unauthorized', nil) }
+
+      before do
+        application.update!(cluster: cluster)
+
+        expect(service).to receive(:installation_phase).and_raise(error)
+      end
+
+      include_examples 'logs kubernetes errors' do
+        let(:error_name) { 'Kubeclient::HttpError' }
+        let(:error_message) { 'Unauthorized' }
+        let(:error_code) { 401 }
+      end
+
+      it 'shows the response code from the error' do
+        service.execute
+
+        expect(application).to be_uninstall_errored
+        expect(application.status_reason).to eq('Kubernetes error: 401')
+      end
+    end
+  end
+end
diff --git a/spec/services/clusters/applications/uninstall_service_spec.rb b/spec/services/clusters/applications/uninstall_service_spec.rb
new file mode 100644
index 00000000000..d1d0e923e18
--- /dev/null
+++ b/spec/services/clusters/applications/uninstall_service_spec.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Applications::UninstallService, '#execute' do
+  let(:application) { create(:clusters_applications_helm, :scheduled) }
+  let(:service) { described_class.new(application) }
+  let(:helm_client) { instance_double(Gitlab::Kubernetes::Helm::Api) }
+  let(:worker_class) { Clusters::Applications::WaitForUninstallAppWorker }
+
+  before do
+    allow(service).to receive(:helm_api).and_return(helm_client)
+  end
+
+  context 'when there are no errors' do
+    before do
+      expect(helm_client).to receive(:uninstall).with(kind_of(Gitlab::Kubernetes::Helm::ResetCommand))
+      allow(worker_class).to receive(:perform_in).and_return(nil)
+    end
+
+    it 'make the application to be uninstalling' do
+      expect(application.cluster).not_to be_nil
+      service.execute
+
+      expect(application).to be_uninstalling
+    end
+
+    it 'schedule async installation status check' do
+      expect(worker_class).to receive(:perform_in).once
+
+      service.execute
+    end
+  end
+
+  context 'when k8s cluster communication fails' do
+    let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
+
+    before do
+      expect(helm_client).to receive(:uninstall).with(kind_of(Gitlab::Kubernetes::Helm::ResetCommand)).and_raise(error)
+    end
+
+    include_examples 'logs kubernetes errors' do
+      let(:error_name) { 'Kubeclient::HttpError' }
+      let(:error_message) { 'system failure' }
+      let(:error_code) { 500 }
+    end
+
+    it 'make the application errored' do
+      service.execute
+
+      expect(application).to be_uninstall_errored
+      expect(application.status_reason).to match('Kubernetes error: 500')
+    end
+  end
+
+  context 'a non kubernetes error happens' do
+    let(:application) { create(:clusters_applications_helm, :scheduled) }
+    let(:error) { StandardError.new('something bad happened') }
+
+    before do
+      expect(helm_client).to receive(:uninstall).with(kind_of(Gitlab::Kubernetes::Helm::ResetCommand)).and_raise(error)
+    end
+
+    include_examples 'logs kubernetes errors' do
+      let(:error_name) { 'StandardError' }
+      let(:error_message) { 'something bad happened' }
+      let(:error_code) { nil }
+    end
+
+    it 'make the application errored' do
+      service.execute
+
+      expect(application).to be_uninstall_errored
+      expect(application.status_reason).to eq('Failed to uninstall.')
+    end
+  end
+end
diff --git a/spec/workers/clusters/applications/wait_for_uninstall_app_worker_spec.rb b/spec/workers/clusters/applications/wait_for_uninstall_app_worker_spec.rb
new file mode 100644
index 00000000000..aaf5c9defc4
--- /dev/null
+++ b/spec/workers/clusters/applications/wait_for_uninstall_app_worker_spec.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Applications::WaitForUninstallAppWorker, '#perform' do
+  let(:app) { create(:clusters_applications_helm) }
+  let(:app_name) { app.name }
+  let(:app_id) { app.id }
+
+  subject { described_class.new.perform(app_name, app_id) }
+
+  context 'app exists' do
+    let(:service) { instance_double(Clusters::Applications::CheckUninstallProgressService) }
+
+    it 'calls the check service' do
+      expect(Clusters::Applications::CheckUninstallProgressService).to receive(:new).with(app).and_return(service)
+      expect(service).to receive(:execute).once
+
+      subject
+    end
+  end
+
+  context 'app does not exist' do
+    let(:app_id) { 0 }
+
+    it 'does not call the check service' do
+      expect(Clusters::Applications::CheckUninstallProgressService).not_to receive(:new)
+
+      expect { subject }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+  end
+end