diff options
author | Shinya Maeda <shinya@gitlab.com> | 2019-07-18 16:22:46 +0700 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2019-07-23 17:26:08 +0700 |
commit | c2e0e689f355555db231ac6db40ab1b654c90233 (patch) | |
tree | 1ab8b2e6561598a61a10a0197a975019ed13f464 /app/services | |
parent | 1a3fda63a5f9756cde19bc7e221651b0c33cb5dc (diff) | |
download | gitlab-ce-c2e0e689f355555db231ac6db40ab1b654c90233.tar.gz |
Validate the existence of archived traces before removing live tracesafe-archiving-for-traces
Often live traces are removed even though the archived trace
doesn't exist. This commit checkes the existence strictly.
Diffstat (limited to 'app/services')
-rw-r--r-- | app/services/ci/archive_trace_service.rb | 28 |
1 files changed, 24 insertions, 4 deletions
diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index b5cfa1d019c..700d78361a4 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -2,8 +2,25 @@ module Ci class ArchiveTraceService - def execute(job) + def execute(job, worker_name:) + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_live_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have live trace but going to be archived.', + job_id: job.id) + return + end + job.trace.archive! + + # TODO: Remove this logging once we confirmed new live trace architecture is functional. + # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. + unless job.has_archived_trace? + Sidekiq.logger.warn(class: worker_name, + message: 'The job does not have archived trace after archiving.', + job_id: job.id) + end rescue ::Gitlab::Ci::Trace::AlreadyArchivedError # It's already archived, thus we can safely ignore this exception. rescue => e @@ -11,7 +28,7 @@ module Ci # If `archive!` keeps failing for over a week, that could incur data loss. # (See more https://docs.gitlab.com/ee/administration/job_traces.html#new-live-trace-architecture) # In order to avoid interrupting the system, we do not raise an exception here. - archive_error(e, job) + archive_error(e, job, worker_name) end private @@ -22,9 +39,12 @@ module Ci "Counter of failed attempts of trace archiving") end - def archive_error(error, job) + def archive_error(error, job, worker_name) failed_archive_counter.increment - Rails.logger.error "Failed to archive trace. id: #{job.id} message: #{error.message}" # rubocop:disable Gitlab/RailsLogger + + Sidekiq.logger.warn(class: worker_name, + message: "Failed to archive trace. message: #{error.message}.", + job_id: job.id) Gitlab::Sentry .track_exception(error, |