diff options
| -rw-r--r-- | app/models/ci/build_trace_chunk.rb | 15 | ||||
| -rw-r--r-- | app/workers/ci/rescue_stale_live_trace_worker.rb | 23 | ||||
| -rw-r--r-- | app/workers/rescue_stale_live_trace_worker.rb | 26 |
3 files changed, 38 insertions, 26 deletions
diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 4856f10846c..4884e355c36 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -50,6 +50,21 @@ module Ci def finalize_fast_destroy(keys) redis_delete_data(keys) end + + # Find stale live traces and return their build ids + def find_stale(finished_before: 1.hour.ago) + include(EachBatch) + .select(:build_id) + .group(:build_id) + .joins(:build) + .merge(Ci::Build.finished) + .where('ci_builds.finished_at < ?', finished_before) + .each_batch(column: :build_id) do |chunks| + build_ids = chunks.map { |chunk| [chunk.build_id] } + + yield build_ids + end + end end ## diff --git a/app/workers/ci/rescue_stale_live_trace_worker.rb b/app/workers/ci/rescue_stale_live_trace_worker.rb new file mode 100644 index 00000000000..b60c212c468 --- /dev/null +++ b/app/workers/ci/rescue_stale_live_trace_worker.rb @@ -0,0 +1,23 @@ +module Ci + class RescueStaleLiveTraceWorker + include ApplicationWorker + include CronjobQueue + + def perform + # Reschedule to archive live traces + # + # The targets are jobs with the following conditions + # - It had been finished 1 hour ago, but it has not had an acthived trace yet + # This case happens when sidekiq-jobs of archiving traces are lost in order to restart sidekiq instace which hit RSS limit + Ci::BuildTraceChunk.find_stale(finished_before: 1.hour.ago) do |build_ids| + Ci::Build.where(id: build_ids).find_each do |build| + begin + build.trace.archive! + rescue => e + Rails.logger.info "Failed to archive stale live trace. id: #{build.id} message: #{e.message}" + end + end + end + end + end +end diff --git a/app/workers/rescue_stale_live_trace_worker.rb b/app/workers/rescue_stale_live_trace_worker.rb deleted file mode 100644 index 15f9561baf2..00000000000 --- a/app/workers/rescue_stale_live_trace_worker.rb +++ /dev/null @@ -1,26 +0,0 @@ -class RescueStaleLiveTraceWorker - include ApplicationWorker - include CronjobQueue - - def perform - # Reschedule to archive live traces - # - # The targets are jobs with the following conditions - # - It had been finished 1 hour ago, but it has not had an acthived trace yet - # This case happens when sidekiq-jobs of archiving traces are lost in order to restart sidekiq instace which hit RSS limit - Ci::BuildTraceChunk - .include(EachBatch) - .select(:build_id) - .group(:build_id) - .joins(:build) - .merge(Ci::Build.finished) - .where('ci_builds.finished_at < ?', 1.hour.ago) - .each_batch(column: :build_id) do |chunks| - build_ids = chunks.map { |chunk| [chunk.build_id] } - - ArchiveTraceWorker.bulk_perform_async(build_ids) - - Rails.logger.info "Scheduled to archive stale live traces from #{build_ids.min} to #{build_ids.max}" - end - end -end |