diff options
author | James Lopez <james@gitlab.com> | 2019-04-10 06:38:28 +0000 |
---|---|---|
committer | James Lopez <james@gitlab.com> | 2019-04-10 06:38:28 +0000 |
commit | 112741ef635a4c4f9504bf63a3b0207a438e8ee8 (patch) | |
tree | 2e0a95a1a741e0dc76e5f166cc29a65549aaa41e | |
parent | b5af30bba6a7ffdd4b054fa9c8c9b42dd91a51d7 (diff) | |
parent | c239bfcb1750794ec1bf8172dfa380dea64fe4c1 (diff) | |
download | gitlab-ce-112741ef635a4c4f9504bf63a3b0207a438e8ee8.tar.gz |
Merge branch '59276-more-k8s-logging' into 'master'
Resolve "Improvements of kubernetes.log"
Closes #59276
See merge request gitlab-org/gitlab-ce!26895
8 files changed, 78 insertions, 2 deletions
diff --git a/app/services/clusters/applications/base_helm_service.rb b/app/services/clusters/applications/base_helm_service.rb index adaa68b1efb..3e7f55f0c63 100644 --- a/app/services/clusters/applications/base_helm_service.rb +++ b/app/services/clusters/applications/base_helm_service.rb @@ -16,6 +16,7 @@ module Clusters error_code: error.respond_to?(:error_code) ? error.error_code : nil, service: self.class.name, app_id: app.id, + app_name: app.name, project_ids: app.cluster.project_ids, group_ids: app.cluster.group_ids } @@ -30,6 +31,19 @@ module Clusters Gitlab::Sentry.track_acceptable_exception(error, extra: meta) end + def log_event(event) + meta = { + service: self.class.name, + app_id: app.id, + app_name: app.name, + project_ids: app.cluster.project_ids, + group_ids: app.cluster.group_ids, + event: event + } + + logger.info(meta) + end + def logger @logger ||= Gitlab::Kubernetes::Logger.build end diff --git a/app/services/clusters/applications/install_service.rb b/app/services/clusters/applications/install_service.rb index 5bd3623a558..1f62b3eb4de 100644 --- a/app/services/clusters/applications/install_service.rb +++ b/app/services/clusters/applications/install_service.rb @@ -7,8 +7,10 @@ module Clusters return unless app.scheduled? app.make_installing! + log_event(:begin_install) helm_api.install(install_command) + log_event(:schedule_wait_for_installation) ClusterWaitForAppInstallationWorker.perform_in( ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) rescue Kubeclient::HttpError => e diff --git a/app/services/clusters/applications/patch_service.rb b/app/services/clusters/applications/patch_service.rb index 20c739af7a2..c3d317e226b 100644 --- a/app/services/clusters/applications/patch_service.rb +++ b/app/services/clusters/applications/patch_service.rb @@ -8,8 +8,10 @@ module Clusters app.make_updating! + log_event(:begin_patch) helm_api.update(update_command) + log_event(:schedule_wait_for_patch) ClusterWaitForAppInstallationWorker.perform_in( ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) rescue Kubeclient::HttpError => e diff --git a/app/services/clusters/applications/upgrade_service.rb b/app/services/clusters/applications/upgrade_service.rb index a0ece1d2635..c34391bc8ad 100644 --- a/app/services/clusters/applications/upgrade_service.rb +++ b/app/services/clusters/applications/upgrade_service.rb @@ -9,10 +9,12 @@ module Clusters begin app.make_updating! + log_event(:begin_upgrade) # install_command works with upgrades too # as it basically does `helm upgrade --install` helm_api.update(install_command) + log_event(:schedule_wait_for_upgrade) ClusterWaitForAppInstallationWorker.perform_in( ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id) rescue Kubeclient::HttpError => e diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 141fe488357..1983513174c 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -573,7 +573,7 @@ However, sometimes GitLab can not create them. In such instances, your job will This job failed because the necessary resources were not successfully created. ``` -To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#sidekiqlog). +To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#kuberneteslog). Common reasons for failure include: diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index 919f19c86d7..8a3bea95a04 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -19,11 +19,40 @@ module Gitlab def create! resource = ::Kubeclient::Resource.new(metadata: { name: name }) + log_event(:begin_create) @client.create_namespace(resource) end def ensure_exists! exists? || create! + rescue ::Kubeclient::HttpError => error + log_create_failed(error) + raise + end + + private + + def log_create_failed(error) + logger.error({ + exception: error.class.name, + status_code: error.error_code, + namespace: name, + class_name: self.class.name, + event: :failed_to_create_namespace, + message: error.message + }) + end + + def log_event(event) + logger.info( + namespace: name, + class_name: self.class.name, + event: event + ) + end + + def logger + @logger ||= Gitlab::Kubernetes::Logger.build end end end diff --git a/spec/lib/gitlab/kubernetes/namespace_spec.rb b/spec/lib/gitlab/kubernetes/namespace_spec.rb index e1c35c355f4..e91a755aa03 100644 --- a/spec/lib/gitlab/kubernetes/namespace_spec.rb +++ b/spec/lib/gitlab/kubernetes/namespace_spec.rb @@ -62,5 +62,32 @@ describe Gitlab::Kubernetes::Namespace do subject.ensure_exists! end + + context 'when client errors' do + let(:exception) { Kubeclient::HttpError.new(500, 'system failure', nil) } + + before do + allow(client).to receive(:get_namespace).with(name).once.and_raise(exception) + end + + it 'raises the exception' do + expect { subject.ensure_exists! }.to raise_error(exception) + end + + it 'logs the error' do + expect(subject.send(:logger)).to receive(:error).with( + hash_including( + exception: 'Kubeclient::HttpError', + status_code: 500, + namespace: 'a_namespace', + class_name: 'Gitlab::Kubernetes::Namespace', + event: :failed_to_create_namespace, + message: 'system failure' + ) + ) + + expect { subject.ensure_exists! }.to raise_error(exception) + end + end end end diff --git a/spec/support/shared_examples/services/base_helm_service_shared_examples.rb b/spec/support/shared_examples/services/base_helm_service_shared_examples.rb index 78a8e49fd76..fa76b95f768 100644 --- a/spec/support/shared_examples/services/base_helm_service_shared_examples.rb +++ b/spec/support/shared_examples/services/base_helm_service_shared_examples.rb @@ -20,7 +20,7 @@ shared_examples 'logs kubernetes errors' do end it 'logs into kubernetes.log and Sentry' do - expect(service.send(:logger)).to receive(:error).with(logger_hash) + expect(service.send(:logger)).to receive(:error).with(hash_including(logger_hash)) expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with( error, |