Merge branch '59276-more-k8s-logging' into 'master'

Resolve "Improvements of kubernetes.log"

Closes #59276

See merge request gitlab-org/gitlab-ce!26895

8 files changed, 78 insertions, 2 deletions

diff --git a/app/services/clusters/applications/base_helm_service.rb b/app/services/clusters/applications/base_helm_service.rb
index adaa68b1efb..3e7f55f0c63 100644
--- a/app/services/clusters/applications/base_helm_service.rb
+++ b/app/services/clusters/applications/base_helm_service.rb
@@ -16,6 +16,7 @@ module Clusters
           error_code: error.respond_to?(:error_code) ? error.error_code : nil,
           service: self.class.name,
           app_id: app.id,
+          app_name: app.name,
           project_ids: app.cluster.project_ids,
           group_ids: app.cluster.group_ids
         }
@@ -30,6 +31,19 @@ module Clusters
         Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
       end
 
+      def log_event(event)
+        meta = {
+          service: self.class.name,
+          app_id: app.id,
+          app_name: app.name,
+          project_ids: app.cluster.project_ids,
+          group_ids: app.cluster.group_ids,
+          event: event
+        }
+
+        logger.info(meta)
+      end
+
       def logger
         @logger ||= Gitlab::Kubernetes::Logger.build
       end
diff --git a/app/services/clusters/applications/install_service.rb b/app/services/clusters/applications/install_service.rb
index 5bd3623a558..1f62b3eb4de 100644
--- a/app/services/clusters/applications/install_service.rb
+++ b/app/services/clusters/applications/install_service.rb
@@ -7,8 +7,10 @@ module Clusters
         return unless app.scheduled?
 
         app.make_installing!
+        log_event(:begin_install)
         helm_api.install(install_command)
 
+        log_event(:schedule_wait_for_installation)
         ClusterWaitForAppInstallationWorker.perform_in(
           ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
       rescue Kubeclient::HttpError => e
diff --git a/app/services/clusters/applications/patch_service.rb b/app/services/clusters/applications/patch_service.rb
index 20c739af7a2..c3d317e226b 100644
--- a/app/services/clusters/applications/patch_service.rb
+++ b/app/services/clusters/applications/patch_service.rb
@@ -8,8 +8,10 @@ module Clusters
 
         app.make_updating!
 
+        log_event(:begin_patch)
         helm_api.update(update_command)
 
+        log_event(:schedule_wait_for_patch)
         ClusterWaitForAppInstallationWorker.perform_in(
           ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
       rescue Kubeclient::HttpError => e
diff --git a/app/services/clusters/applications/upgrade_service.rb b/app/services/clusters/applications/upgrade_service.rb
index a0ece1d2635..c34391bc8ad 100644
--- a/app/services/clusters/applications/upgrade_service.rb
+++ b/app/services/clusters/applications/upgrade_service.rb
@@ -9,10 +9,12 @@ module Clusters
         begin
           app.make_updating!
 
+          log_event(:begin_upgrade)
           # install_command works with upgrades too
           # as it basically does `helm upgrade --install`
           helm_api.update(install_command)
 
+          log_event(:schedule_wait_for_upgrade)
           ClusterWaitForAppInstallationWorker.perform_in(
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
         rescue Kubeclient::HttpError => e
diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md
index 141fe488357..1983513174c 100644
--- a/doc/user/project/clusters/index.md
+++ b/doc/user/project/clusters/index.md
@@ -573,7 +573,7 @@ However, sometimes GitLab can not create them. In such instances, your job will
 This job failed because the necessary resources were not successfully created.
 ```
 
-To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#sidekiqlog).
+To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#kuberneteslog).
 
 Common reasons for failure include:
 
diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb
index 919f19c86d7..8a3bea95a04 100644
--- a/lib/gitlab/kubernetes/namespace.rb
+++ b/lib/gitlab/kubernetes/namespace.rb
@@ -19,11 +19,40 @@ module Gitlab
       def create!
         resource = ::Kubeclient::Resource.new(metadata: { name: name })
 
+        log_event(:begin_create)
         @client.create_namespace(resource)
       end
 
       def ensure_exists!
         exists? || create!
+      rescue ::Kubeclient::HttpError => error
+        log_create_failed(error)
+        raise
+      end
+
+      private
+
+      def log_create_failed(error)
+        logger.error({
+          exception: error.class.name,
+          status_code: error.error_code,
+          namespace: name,
+          class_name: self.class.name,
+          event: :failed_to_create_namespace,
+          message: error.message
+        })
+      end
+
+      def log_event(event)
+        logger.info(
+          namespace: name,
+          class_name: self.class.name,
+          event: event
+        )
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
       end
     end
   end
diff --git a/spec/lib/gitlab/kubernetes/namespace_spec.rb b/spec/lib/gitlab/kubernetes/namespace_spec.rb
index e1c35c355f4..e91a755aa03 100644
--- a/spec/lib/gitlab/kubernetes/namespace_spec.rb
+++ b/spec/lib/gitlab/kubernetes/namespace_spec.rb
@@ -62,5 +62,32 @@ describe Gitlab::Kubernetes::Namespace do
 
       subject.ensure_exists!
     end
+
+    context 'when client errors' do
+      let(:exception) { Kubeclient::HttpError.new(500, 'system failure', nil) }
+
+      before do
+        allow(client).to receive(:get_namespace).with(name).once.and_raise(exception)
+      end
+
+      it 'raises the exception' do
+        expect { subject.ensure_exists! }.to raise_error(exception)
+      end
+
+      it 'logs the error' do
+        expect(subject.send(:logger)).to receive(:error).with(
+          hash_including(
+            exception: 'Kubeclient::HttpError',
+            status_code: 500,
+            namespace: 'a_namespace',
+            class_name: 'Gitlab::Kubernetes::Namespace',
+            event: :failed_to_create_namespace,
+            message: 'system failure'
+          )
+        )
+
+        expect { subject.ensure_exists! }.to raise_error(exception)
+      end
+    end
   end
 end
diff --git a/spec/support/shared_examples/services/base_helm_service_shared_examples.rb b/spec/support/shared_examples/services/base_helm_service_shared_examples.rb
index 78a8e49fd76..fa76b95f768 100644
--- a/spec/support/shared_examples/services/base_helm_service_shared_examples.rb
+++ b/spec/support/shared_examples/services/base_helm_service_shared_examples.rb
@@ -20,7 +20,7 @@ shared_examples 'logs kubernetes errors' do
   end
 
   it 'logs into kubernetes.log and Sentry' do
-    expect(service.send(:logger)).to receive(:error).with(logger_hash)
+    expect(service.send(:logger)).to receive(:error).with(hash_including(logger_hash))
 
     expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
       error,