summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/services/clusters/applications/base_helm_service.rb19
-rw-r--r--app/services/clusters/applications/check_installation_progress_service.rb3
-rw-r--r--app/services/clusters/applications/install_service.rb6
-rw-r--r--doc/administration/logs.md19
-rw-r--r--doc/development/logging.md10
-rw-r--r--lib/gitlab/kubernetes/logger.rb11
-rw-r--r--spec/services/clusters/applications/check_installation_progress_service_spec.rb6
-rw-r--r--spec/services/clusters/applications/install_service_spec.rb70
8 files changed, 130 insertions, 14 deletions
diff --git a/app/services/clusters/applications/base_helm_service.rb b/app/services/clusters/applications/base_helm_service.rb
index 270a8eb24f4..e86ca8cf1d0 100644
--- a/app/services/clusters/applications/base_helm_service.rb
+++ b/app/services/clusters/applications/base_helm_service.rb
@@ -11,6 +11,25 @@ module Clusters
protected
+ def log_error(error)
+ meta = {
+ exception: error.class.name,
+ error_code: error.respond_to?(:error_code) ? error.error_code : nil,
+ service: self.class.name,
+ app_id: app.id,
+ project_ids: app.cluster.project_ids,
+ group_ids: app.cluster.group_ids,
+ message: error.message
+ }
+
+ logger.error(meta)
+ Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
+ end
+
+ def logger
+ @logger ||= Gitlab::Kubernetes::Logger.build
+ end
+
def cluster
app.cluster
end
diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb
index 6794580e1e8..21ec26ea233 100644
--- a/app/services/clusters/applications/check_installation_progress_service.rb
+++ b/app/services/clusters/applications/check_installation_progress_service.rb
@@ -15,8 +15,7 @@ module Clusters
check_timeout
end
rescue Kubeclient::HttpError => e
- Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
- Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+ log_error(e)
app.make_errored!("Kubernetes error: #{e.error_code}") unless app.errored?
end
diff --git a/app/services/clusters/applications/install_service.rb b/app/services/clusters/applications/install_service.rb
index f4385748c43..5a65dc4ef59 100644
--- a/app/services/clusters/applications/install_service.rb
+++ b/app/services/clusters/applications/install_service.rb
@@ -13,12 +13,10 @@ module Clusters
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
- Rails.logger.error("Kubernetes error: #{e.error_code} #{e.message}")
- Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+ log_error(e)
app.make_errored!("Kubernetes error: #{e.error_code}")
rescue StandardError => e
- Rails.logger.error "Can't start installation process: #{e.class.name} #{e.message}"
- Gitlab::Sentry.track_acceptable_exception(e, extra: { scope: 'kubernetes', app_id: app.id })
+ log_error(e)
app.make_errored!("Can't start installation process.")
end
end
diff --git a/doc/administration/logs.md b/doc/administration/logs.md
index 7e5a3eb9ccd..698f4caab3a 100644
--- a/doc/administration/logs.md
+++ b/doc/administration/logs.md
@@ -126,6 +126,25 @@ It contains information about [integrations](../user/project/integrations/projec
{"severity":"INFO","time":"2018-09-06T17:15:16.365Z","service_class":"JiraService","project_id":3,"project_path":"namespace2/project2","message":"Successfully posted","client_url":"http://jira.example.net"}
```
+## `kubernetes.log`
+
+Introduced in GitLab 11.6. This file lives in
+`/var/log/gitlab/gitlab-rails/kubernetes.log` for Omnibus GitLab
+packages or in `/home/git/gitlab/log/kubernetes.log` for
+installations from source.
+
+It logs information related to the Kubernetes Integration including errors
+during installing cluster applications on your GitLab managed Kubernetes
+clusters.
+
+Each line contains a JSON line that can be ingested by Elasticsearch, Splunk,
+etc. For example:
+
+```json
+{"severity":"ERROR","time":"2018-11-23T15:14:54.652Z","exception":"Kubeclient::HttpError","error_code":401,"service":"Clusters::Applications::CheckInstallationProgressService","app_id":14,"project_ids":[1],"group_ids":[],"message":"Unauthorized"}
+{"severity":"ERROR","time":"2018-11-23T15:42:11.647Z","exception":"Kubeclient::HttpError","error_code":null,"service":"Clusters::Applications::InstallService","app_id":2,"project_ids":[19],"group_ids":[],"message":"SSL_connect returned=1 errno=0 state=error: certificate verify failed (unable to get local issuer certificate)"}
+```
+
## `githost.log`
This file lives in `/var/log/gitlab/gitlab-rails/githost.log` for
diff --git a/doc/development/logging.md b/doc/development/logging.md
index abd08c420da..5c1d96b9e0c 100644
--- a/doc/development/logging.md
+++ b/doc/development/logging.md
@@ -75,7 +75,7 @@ To create a new file:
module Import
class Logger < ::Gitlab::JsonLogger
def self.file_name_noext
- 'importer_json'
+ 'importer'
end
end
end
@@ -105,7 +105,7 @@ To create a new file:
```ruby
# GOOD
- logger.info("Unable to create project", project_id: project.id)
+ logger.info(message: "Unable to create project", project_id: project.id)
```
1. Be sure to create a common base structure of your log messages. For example,
@@ -118,13 +118,13 @@ To create a new file:
```ruby
# BAD
- logger.info("Import error", error: 1)
- logger.info("Import error", error: "I/O failure")
+ logger.info(message: "Import error", error: 1)
+ logger.info(message: "Import error", error: "I/O failure")
```
```ruby
# GOOD
- logger.info("Import error", error_code: 1, error: "I/O failure")
+ logger.info(message: "Import error", error_code: 1, error: "I/O failure")
```
## Additional steps with new log files
diff --git a/lib/gitlab/kubernetes/logger.rb b/lib/gitlab/kubernetes/logger.rb
new file mode 100644
index 00000000000..5e59482419b
--- /dev/null
+++ b/lib/gitlab/kubernetes/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ class Logger < ::Gitlab::JsonLogger
+ def self.file_name_noext
+ 'kubernetes'
+ end
+ end
+ end
+end
diff --git a/spec/services/clusters/applications/check_installation_progress_service_spec.rb b/spec/services/clusters/applications/check_installation_progress_service_spec.rb
index 9452a9e38fb..45b8ce94815 100644
--- a/spec/services/clusters/applications/check_installation_progress_service_spec.rb
+++ b/spec/services/clusters/applications/check_installation_progress_service_spec.rb
@@ -105,6 +105,12 @@ describe Clusters::Applications::CheckInstallationProgressService do
expect(application).to be_errored
expect(application.status_reason).to eq('Kubernetes error: 401')
end
+
+ it 'should log error' do
+ expect(service.send(:logger)).to receive(:error)
+
+ service.execute
+ end
end
end
end
diff --git a/spec/services/clusters/applications/install_service_spec.rb b/spec/services/clusters/applications/install_service_spec.rb
index 2f801d019fe..018d9822d3e 100644
--- a/spec/services/clusters/applications/install_service_spec.rb
+++ b/spec/services/clusters/applications/install_service_spec.rb
@@ -33,8 +33,9 @@ describe Clusters::Applications::InstallService do
end
context 'when k8s cluster communication fails' do
+ let(:error) { Kubeclient::HttpError.new(500, 'system failure', nil) }
+
before do
- error = Kubeclient::HttpError.new(500, 'system failure', nil)
expect(helm_client).to receive(:install).with(install_command).and_raise(error)
end
@@ -44,18 +45,81 @@ describe Clusters::Applications::InstallService do
expect(application).to be_errored
expect(application.status_reason).to match('Kubernetes error: 500')
end
+
+ it 'logs errors' do
+ expect(service.send(:logger)).to receive(:error).with(
+ {
+ exception: 'Kubeclient::HttpError',
+ message: 'system failure',
+ service: 'Clusters::Applications::InstallService',
+ app_id: application.id,
+ project_ids: application.cluster.project_ids,
+ group_ids: [],
+ error_code: 500
+ }
+ )
+
+ expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
+ error,
+ extra: {
+ exception: 'Kubeclient::HttpError',
+ message: 'system failure',
+ service: 'Clusters::Applications::InstallService',
+ app_id: application.id,
+ project_ids: application.cluster.project_ids,
+ group_ids: [],
+ error_code: 500
+ }
+ )
+
+ service.execute
+ end
end
- context 'when application cannot be persisted' do
+ context 'a non kubernetes error happens' do
let(:application) { create(:clusters_applications_helm, :scheduled) }
+ let(:error) { StandardError.new("something bad happened") }
+
+ before do
+ expect(application).to receive(:make_installing!).once.and_raise(error)
+ end
it 'make the application errored' do
- expect(application).to receive(:make_installing!).once.and_raise(ActiveRecord::RecordInvalid)
expect(helm_client).not_to receive(:install)
service.execute
expect(application).to be_errored
+ expect(application.status_reason).to eq("Can't start installation process.")
+ end
+
+ it 'logs errors' do
+ expect(service.send(:logger)).to receive(:error).with(
+ {
+ exception: 'StandardError',
+ error_code: nil,
+ message: 'something bad happened',
+ service: 'Clusters::Applications::InstallService',
+ app_id: application.id,
+ project_ids: application.cluster.projects.pluck(:id),
+ group_ids: []
+ }
+ )
+
+ expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
+ error,
+ extra: {
+ exception: 'StandardError',
+ error_code: nil,
+ message: 'something bad happened',
+ service: 'Clusters::Applications::InstallService',
+ app_id: application.id,
+ project_ids: application.cluster.projects.pluck(:id),
+ group_ids: []
+ }
+ )
+
+ service.execute
end
end
end