Merge branch 'feature/sm/artifacts-trace' into 'master'

CE: Trace as artifacts (FileStorage only)

Closes gitlab-ee#4180

See merge request gitlab-org/gitlab-ce!16702

8 files changed, 45 insertions, 5 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 78906e7a968..20534b8eed0 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -21,6 +21,7 @@ module Ci
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 84fc6863567..0a599f72bc7 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -9,9 +9,12 @@ module Ci
 
     mount_uploader :file, JobArtifactUploader
 
+    delegate :open, :exists?, to: :file
+
     enum file_type: {
       archive: 1,
-      metadata: 2
+      metadata: 2,
+      trace: 3
     }
 
     def self.artifacts_size_for(project)
diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb
index 0460439e9e6..ff52ca64459 100644
--- a/app/models/concerns/artifact_migratable.rb
+++ b/app/models/concerns/artifact_migratable.rb
@@ -39,7 +39,6 @@ module ArtifactMigratable
   end
 
   def artifacts_size
-    read_attribute(:artifacts_size).to_i +
-      job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i
+    read_attribute(:artifacts_size).to_i + job_artifacts.sum(:size).to_i
   end
 end
diff --git a/app/services/ci/create_trace_artifact_service.rb b/app/services/ci/create_trace_artifact_service.rb
new file mode 100644
index 00000000000..280a2c3afa4
--- /dev/null
+++ b/app/services/ci/create_trace_artifact_service.rb
@@ -0,0 +1,16 @@
+module Ci
+  class CreateTraceArtifactService < BaseService
+    def execute(job)
+      return if job.job_artifacts_trace
+
+      job.trace.read do |stream|
+        if stream.file?
+          job.create_job_artifacts_trace!(
+            project: job.project,
+            file_type: :trace,
+            file: stream)
+        end
+      end
+    end
+  end
+end
diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index 0abb462ab7d..ad5385f45a4 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -13,6 +13,12 @@ class JobArtifactUploader < GitlabUploader
     dynamic_segment
   end
 
+  def open
+    raise 'Only File System is supported' unless file_storage?
+
+    File.open(path, "rb") if path
+  end
+
   private
 
   def dynamic_segment
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 50e876b1d19..f2c20114534 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -43,6 +43,7 @@
 - pipeline_creation:run_pipeline_schedule
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
+- pipeline_default:create_trace_artifact
 - pipeline_default:pipeline_metrics
 - pipeline_default:pipeline_notification
 - pipeline_default:update_head_pipeline_for_merge_request
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index 97d80305bec..b5ed8d607b3 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -6,9 +6,13 @@ class BuildFinishedWorker
 
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
-      BuildTraceSectionsWorker.perform_async(build.id)
+      # We execute that in sync as this access the files in order to access local file, and reduce IO
+      BuildTraceSectionsWorker.new.perform(build.id)
       BuildCoverageWorker.new.perform(build.id)
-      BuildHooksWorker.new.perform(build.id)
+
+      # We execute that async as this are two indepentent operations that can be executed after TraceSections and Coverage
+      BuildHooksWorker.perform_async(build.id)
+      CreateTraceArtifactWorker.perform_async(build.id)
     end
   end
 end
diff --git a/app/workers/create_trace_artifact_worker.rb b/app/workers/create_trace_artifact_worker.rb
new file mode 100644
index 00000000000..11cda58021e
--- /dev/null
+++ b/app/workers/create_trace_artifact_worker.rb
@@ -0,0 +1,10 @@
+class CreateTraceArtifactWorker
+  include ApplicationWorker
+  include PipelineQueue
+
+  def perform(job_id)
+    Ci::Build.preload(:project, :user).find_by(id: job_id).try do |job|
+      Ci::CreateTraceArtifactService.new(job.project, job.user).execute(job)
+    end
+  end
+end