Use JSON logging for helm install servicesjson-logging-for-k8s

7 files changed, 130 insertions, 9 deletions

diff --git a/app/services/clusters/applications/base_helm_service.rb b/app/services/clusters/applications/base_helm_service.rb
index 270a8eb24f4..68ea3550eb4 100644
--- a/app/services/clusters/applications/base_helm_service.rb
+++ b/app/services/clusters/applications/base_helm_service.rb
@@ -11,6 +11,26 @@ module Clusters
 
       protected
 
+      def log_error(error)
+        meta = {
+          exception: error.class.name,
+          error_code: error.respond_to?(:error_code) ? error.error_code : nil,
+          scope: 'kubernetes',
+          service: self.class.name,
+          app_id: app.id,
+          project_ids: app.cluster.project_ids,
+          group_ids: app.cluster.group_ids,
+          message: error.message
+        }
+
+        logger.error(meta)
+        Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
+      end
+
       def cluster
         app.cluster
       end
diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb
index ca0f7b30053..0e094c6ae1a 100644
--- a/app/services/clusters/applications/check_installation_progress_service.rb
+++ b/app/services/clusters/applications/check_installation_progress_service.rb
@@ -15,8 +15,7 @@ module Clusters
           check_timeout
         end
       rescue Kubeclient::HttpError => e
-        Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
-        Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+        log_error(e)
         app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored?
       end
 
diff --git a/app/services/clusters/applications/install_service.rb b/app/services/clusters/applications/install_service.rb
index f4385748c43..5a65dc4ef59 100644
--- a/app/services/clusters/applications/install_service.rb
+++ b/app/services/clusters/applications/install_service.rb
@@ -13,12 +13,10 @@ module Clusters
           ClusterWaitForAppInstallationWorker.perform_in(
             ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
         rescue Kubeclient::HttpError => e
-          Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
-          Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+          log_error(e)
           app.make_errored!("Kubernetes error: #{e.error_code}")
         rescue StandardError => e
-          Rails.logger.error "Can't start installation process: #{e.class.name} #{e.message}"
-          Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+          log_error(e)
           app.make_errored!("Can't start installation process.")
         end
       end
diff --git a/doc/administration/logs.md b/doc/administration/logs.md
index 7e5a3eb9ccd..e7e1e3ecf64 100644
--- a/doc/administration/logs.md
+++ b/doc/administration/logs.md
@@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec
 {"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"}
 ```
 
+## `kubernetes.log`
+
+Introduced in GitLab 11.6. This file lives in
+`/var/log/gitlab/gitlab-rails/kubernetes.log` for Omnibus GitLab
+packages or in `/home/git/gitlab/log/kubernetes.log` for
+installations from source.
+
+It logs information related to the Kubernetes Integration including errors
+during installing cluster applications on your GitLab managed Kubernetes
+clusters.
+
+Each line contains a JSON line that can be ingested by Elasticsearch, Splunk,
+etc. For example:
+
+```json
+{"severity":"ERROR","time":"2018-11-23T15:14:54.652Z","exception":"Kubeclient::HttpError","error_code":401,"scope":"kubernetes","service":"Clusters::Applications::CheckInstallationProgressService","app_id":14,"project_ids":[1],"group_ids":[],"message":"Unauthorized"}
+{"severity":"ERROR","time":"2018-11-23T15:42:11.647Z","exception":"Kubeclient::HttpError","error_code":null,"scope":"kubernetes","service":"Clusters::Applications::InstallService","app_id":2,"project_ids":[19],"group_ids":[],"message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)"}
+```
+
 ## `githost.log`
 
 This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for
diff --git a/lib/gitlab/kubernetes/logger.rb b/lib/gitlab/kubernetes/logger.rb
new file mode 100644
index 00000000000..5e59482419b
--- /dev/null
+++ b/lib/gitlab/kubernetes/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class Logger < ::Gitlab::JsonLogger
+      def self.file_name_noext
+        'kubernetes'
+      end
+    end
+  end
+end
diff --git a/spec/services/clusters/applications/check_installation_progress_service_spec.rb b/spec/services/clusters/applications/check_installation_progress_service_spec.rb
index ea17f2bb423..70ccf779d87 100644
--- a/spec/services/clusters/applications/check_installation_progress_service_spec.rb
+++ b/spec/services/clusters/applications/check_installation_progress_service_spec.rb
@@ -113,6 +113,12 @@ describe Clusters::Applications::CheckInstallationProgressService do
         expect(application).to be_errored
         expect(application.status_reason).to eq('Kubernetes error: 401')
       end
+
+      it 'should log error' do
+        expect(service.send(:logger)).to receive(:error)
+
+        service.execute
+      end
     end
   end
 end
diff --git a/spec/services/clusters/applications/install_service_spec.rb b/spec/services/clusters/applications/install_service_spec.rb
index 2f801d019fe..23184a0d9b4 100644
--- a/spec/services/clusters/applications/install_service_spec.rb
+++ b/spec/services/clusters/applications/install_service_spec.rb
@@ -33,8 +33,9 @@ describe Clusters::Applications::InstallService do
     end
 
     context 'when k8s cluster communication fails' do
+      let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
+
       before do
-        error = Kubeclient::HttpError.new(500, 'system failure', nil)
         expect(helm_client).to receive(:install).with(install_command).and_raise(error)
       end
 
@@ -44,18 +45,85 @@ describe Clusters::Applications::InstallService do
         expect(application).to be_errored
         expect(application.status_reason).to match('Kubernetes error: 500')
       end
+
+      it 'logs errors' do
+        expect(service.send(:logger)).to receive(:error).with(
+          {
+            exception: 'Kubeclient::HttpError',
+            message: 'system failure',
+            scope: 'kubernetes',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.project_ids,
+            group_ids: [],
+            error_code: 500
+          }
+        )
+
+        expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
+          error,
+          extra: {
+            exception: 'Kubeclient::HttpError',
+            message: 'system failure',
+            scope: 'kubernetes',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.project_ids,
+            group_ids: [],
+            error_code: 500
+          }
+        )
+
+        service.execute
+      end
     end
 
-    context 'when application cannot be persisted' do
+    context 'a non kubernetes error happens' do
       let(:application) { create(:clusters_applications_helm, :scheduled) }
+      let(:error) { StandardError.new("something bad happened") }
+
+      before do
+        expect(application).to receive(:make_installing!).once.and_raise(error)
+      end
 
       it 'make the application errored' do
-        expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid)
         expect(helm_client).not_to receive(:install)
 
         service.execute
 
         expect(application).to be_errored
+        expect(application.status_reason).to eq("Can't start installation process.")
+      end
+
+      it 'logs errors' do
+        expect(service.send(:logger)).to receive(:error).with(
+          {
+            exception: 'StandardError',
+            error_code: nil,
+            message: 'something bad happened',
+            scope: 'kubernetes',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.projects.pluck(:id),
+            group_ids: []
+          }
+        )
+
+        expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
+          error,
+          extra: {
+            exception: 'StandardError',
+            error_code: nil,
+            message: 'something bad happened',
+            scope: 'kubernetes',
+            service: 'Clusters::Applications::InstallService',
+            app_id: application.id,
+            project_ids: application.cluster.projects.pluck(:id),
+            group_ids: []
+          }
+        )
+
+        service.execute
       end
     end
   end