From 63a9d582aa88d774af5eff124b693df6271ae7bc Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Thu, 25 Jan 2018 18:50:56 +0900
Subject: Trace as artifacts

---
 app/models/ci/build.rb                 | 1 +
 app/models/ci/job_artifact.rb          | 5 ++++-
 app/uploaders/job_artifact_uploader.rb | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 78906e7a968..20534b8eed0 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -21,6 +21,7 @@ module Ci
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index 84fc6863567..0a599f72bc7 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -9,9 +9,12 @@ module Ci
 
     mount_uploader :file, JobArtifactUploader
 
+    delegate :open, :exists?, to: :file
+
     enum file_type: {
       archive: 1,
-      metadata: 2
+      metadata: 2,
+      trace: 3
     }
 
     def self.artifacts_size_for(project)
diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index 0abb462ab7d..8766db94079 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -13,6 +13,10 @@ class JobArtifactUploader < GitlabUploader
     dynamic_segment
   end
 
+  def open
+    File.open(path, "rb")
+  end
+
   private
 
   def dynamic_segment
-- 
cgit v1.2.1


From 14d1715610b7df7f8f518a8b57ee4176f208c4a8 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Thu, 25 Jan 2018 23:04:53 +0900
Subject: Inlcude the size of trace artifacts

---
 app/models/concerns/artifact_migratable.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb
index 0460439e9e6..ff52ca64459 100644
--- a/app/models/concerns/artifact_migratable.rb
+++ b/app/models/concerns/artifact_migratable.rb
@@ -39,7 +39,6 @@ module ArtifactMigratable
   end
 
   def artifacts_size
-    read_attribute(:artifacts_size).to_i +
-      job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i
+    read_attribute(:artifacts_size).to_i + job_artifacts.sum(:size).to_i
   end
 end
-- 
cgit v1.2.1


From 0d960fac9115c765896450daf625752f5e9db185 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Thu, 25 Jan 2018 23:08:25 +0900
Subject: JobArtifactUploader#open raise execption if its not Filestorage

---
 app/uploaders/job_artifact_uploader.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index 8766db94079..841168438d8 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -14,7 +14,9 @@ class JobArtifactUploader < GitlabUploader
   end
 
   def open
-    File.open(path, "rb")
+    raise 'Only File System is supported' unless file_storage?
+
+    File.open(path) if path
   end
 
   private
-- 
cgit v1.2.1


From 002f314f320c5731681297225fff5b528de88ed2 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Fri, 26 Jan 2018 02:21:31 +0900
Subject: Expose current_path

---
 app/uploaders/job_artifact_uploader.rb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'app')

diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index 841168438d8..4c814c6f501 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -14,9 +14,11 @@ class JobArtifactUploader < GitlabUploader
   end
 
   def open
-    raise 'Only File System is supported' unless file_storage?
-
-    File.open(path) if path
+    if file_storage?
+      File.open(path, "rb")
+    else
+      raise 'Only File System is supported'
+    end
   end
 
   private
-- 
cgit v1.2.1


From 5f6d826165aa975735cd543dba2b91999c115545 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Sat, 27 Jan 2018 02:00:29 +0900
Subject: Add CreateArtifactsTraceWorker

---
 app/services/ci/create_artifacts_trace_service.rb | 16 ++++++++++++++++
 app/uploaders/job_artifact_uploader.rb            |  8 +++-----
 app/workers/build_finished_worker.rb              |  1 +
 app/workers/create_artifacts_trace_worker.rb      |  8 ++++++++
 4 files changed, 28 insertions(+), 5 deletions(-)
 create mode 100644 app/services/ci/create_artifacts_trace_service.rb
 create mode 100644 app/workers/create_artifacts_trace_worker.rb

(limited to 'app')

diff --git a/app/services/ci/create_artifacts_trace_service.rb b/app/services/ci/create_artifacts_trace_service.rb
new file mode 100644
index 00000000000..eefc2ae13ea
--- /dev/null
+++ b/app/services/ci/create_artifacts_trace_service.rb
@@ -0,0 +1,16 @@
+module Ci
+  class CreateArtifactsTraceService < BaseService
+    def execute(job_id)
+      Ci::Build.find_by(id: job_id).try do |job|
+        return if job.job_artifacts_trace
+
+        job.trace.read do |stream|
+          job.create_job_artifacts_trace!(
+            project: job.project,
+            file_type: :trace,
+            file: stream.path) if stream.file?
+        end
+      end
+    end
+  end
+end
diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index 4c814c6f501..57678dae9ca 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -14,11 +14,9 @@ class JobArtifactUploader < GitlabUploader
   end
 
   def open
-    if file_storage?
-      File.open(path, "rb")
-    else
-      raise 'Only File System is supported'
-    end
+    raise 'Only File System is supported' unless file_storage?
+
+    File.open(path, "rb")
   end
 
   private
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index 97d80305bec..842ab7bbbe4 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -7,6 +7,7 @@ class BuildFinishedWorker
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
       BuildTraceSectionsWorker.perform_async(build.id)
+      CreateArtifactsTraceWorker.perform_async(build.id)
       BuildCoverageWorker.new.perform(build.id)
       BuildHooksWorker.new.perform(build.id)
     end
diff --git a/app/workers/create_artifacts_trace_worker.rb b/app/workers/create_artifacts_trace_worker.rb
new file mode 100644
index 00000000000..d73951b0905
--- /dev/null
+++ b/app/workers/create_artifacts_trace_worker.rb
@@ -0,0 +1,8 @@
+class CreateArtifactsTraceWorker
+  include ApplicationWorker
+  include PipelineQueue
+
+  def perform(job_id)
+    Ci::CreateArtifactsTraceService.new.execute(job_id)
+  end
+end
-- 
cgit v1.2.1


From abc64da90cc4341607b48c09b0920296b5fb9663 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Mon, 29 Jan 2018 00:03:22 +0900
Subject: Rename CreateArtifactsTraceService to CreateTraceArtifactService

---
 app/services/ci/create_artifacts_trace_service.rb | 16 ----------------
 app/services/ci/create_trace_artifact_service.rb  | 14 ++++++++++++++
 app/workers/build_finished_worker.rb              |  2 +-
 app/workers/create_artifacts_trace_worker.rb      |  8 --------
 app/workers/create_trace_artifact_worker.rb       | 10 ++++++++++
 5 files changed, 25 insertions(+), 25 deletions(-)
 delete mode 100644 app/services/ci/create_artifacts_trace_service.rb
 create mode 100644 app/services/ci/create_trace_artifact_service.rb
 delete mode 100644 app/workers/create_artifacts_trace_worker.rb
 create mode 100644 app/workers/create_trace_artifact_worker.rb

(limited to 'app')

diff --git a/app/services/ci/create_artifacts_trace_service.rb b/app/services/ci/create_artifacts_trace_service.rb
deleted file mode 100644
index eefc2ae13ea..00000000000
--- a/app/services/ci/create_artifacts_trace_service.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-module Ci
-  class CreateArtifactsTraceService < BaseService
-    def execute(job_id)
-      Ci::Build.find_by(id: job_id).try do |job|
-        return if job.job_artifacts_trace
-
-        job.trace.read do |stream|
-          job.create_job_artifacts_trace!(
-            project: job.project,
-            file_type: :trace,
-            file: stream.path) if stream.file?
-        end
-      end
-    end
-  end
-end
diff --git a/app/services/ci/create_trace_artifact_service.rb b/app/services/ci/create_trace_artifact_service.rb
new file mode 100644
index 00000000000..80e41856719
--- /dev/null
+++ b/app/services/ci/create_trace_artifact_service.rb
@@ -0,0 +1,14 @@
+module Ci
+  class CreateTraceArtifactService < BaseService
+    def execute(job)
+      return if job.job_artifacts_trace
+
+      job.trace.read do |stream|
+        job.create_job_artifacts_trace!(
+          project: job.project,
+          file_type: :trace,
+          file: stream.path) if stream.file?
+      end
+    end
+  end
+end
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index 842ab7bbbe4..d5a4b6c1553 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -7,7 +7,7 @@ class BuildFinishedWorker
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
       BuildTraceSectionsWorker.perform_async(build.id)
-      CreateArtifactsTraceWorker.perform_async(build.id)
+      CreateTraceArtifactWorker.perform_async(build.id)
       BuildCoverageWorker.new.perform(build.id)
       BuildHooksWorker.new.perform(build.id)
     end
diff --git a/app/workers/create_artifacts_trace_worker.rb b/app/workers/create_artifacts_trace_worker.rb
deleted file mode 100644
index d73951b0905..00000000000
--- a/app/workers/create_artifacts_trace_worker.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-class CreateArtifactsTraceWorker
-  include ApplicationWorker
-  include PipelineQueue
-
-  def perform(job_id)
-    Ci::CreateArtifactsTraceService.new.execute(job_id)
-  end
-end
diff --git a/app/workers/create_trace_artifact_worker.rb b/app/workers/create_trace_artifact_worker.rb
new file mode 100644
index 00000000000..40d8a086cfb
--- /dev/null
+++ b/app/workers/create_trace_artifact_worker.rb
@@ -0,0 +1,10 @@
+class CreateTraceArtifactWorker
+  include ApplicationWorker
+  include PipelineQueue
+
+  def perform(job_id)
+    Ci::Build.find_by(id: job_id).try do |job|
+      Ci::CreateTraceArtifactService.new.execute(job)
+    end
+  end
+end
-- 
cgit v1.2.1


From 93386411a0e304293a962f123d84d127fecd6ffb Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Mon, 29 Jan 2018 00:39:17 +0900
Subject: Add worker queue

---
 app/services/ci/create_trace_artifact_service.rb | 2 +-
 app/workers/all_queues.yml                       | 1 +
 app/workers/create_trace_artifact_worker.rb      | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/services/ci/create_trace_artifact_service.rb b/app/services/ci/create_trace_artifact_service.rb
index 80e41856719..a92ce1a4d52 100644
--- a/app/services/ci/create_trace_artifact_service.rb
+++ b/app/services/ci/create_trace_artifact_service.rb
@@ -7,7 +7,7 @@ module Ci
         job.create_job_artifacts_trace!(
           project: job.project,
           file_type: :trace,
-          file: stream.path) if stream.file?
+          file: stream) if stream.file?
       end
     end
   end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 50e876b1d19..f2c20114534 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -43,6 +43,7 @@
 - pipeline_creation:run_pipeline_schedule
 - pipeline_default:build_coverage
 - pipeline_default:build_trace_sections
+- pipeline_default:create_trace_artifact
 - pipeline_default:pipeline_metrics
 - pipeline_default:pipeline_notification
 - pipeline_default:update_head_pipeline_for_merge_request
diff --git a/app/workers/create_trace_artifact_worker.rb b/app/workers/create_trace_artifact_worker.rb
index 40d8a086cfb..37866da9255 100644
--- a/app/workers/create_trace_artifact_worker.rb
+++ b/app/workers/create_trace_artifact_worker.rb
@@ -4,7 +4,7 @@ class CreateTraceArtifactWorker
 
   def perform(job_id)
     Ci::Build.find_by(id: job_id).try do |job|
-      Ci::CreateTraceArtifactService.new.execute(job)
+      Ci::CreateTraceArtifactService.new(nil, nil).execute(job)
     end
   end
 end
-- 
cgit v1.2.1


From edc936cde2730bb7c417343c582f2b8cf5b571c3 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Mon, 29 Jan 2018 18:00:53 +0900
Subject: Fix inital test failures

---
 app/services/ci/create_trace_artifact_service.rb | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'app')

diff --git a/app/services/ci/create_trace_artifact_service.rb b/app/services/ci/create_trace_artifact_service.rb
index a92ce1a4d52..280a2c3afa4 100644
--- a/app/services/ci/create_trace_artifact_service.rb
+++ b/app/services/ci/create_trace_artifact_service.rb
@@ -4,10 +4,12 @@ module Ci
       return if job.job_artifacts_trace
 
       job.trace.read do |stream|
-        job.create_job_artifacts_trace!(
-          project: job.project,
-          file_type: :trace,
-          file: stream) if stream.file?
+        if stream.file?
+          job.create_job_artifacts_trace!(
+            project: job.project,
+            file_type: :trace,
+            file: stream)
+        end
       end
     end
   end
-- 
cgit v1.2.1


From c9ed3b2d4d208b7452fc2e057f11d28356c08887 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Tue, 30 Jan 2018 01:56:12 +0900
Subject: Add essential tests

---
 app/uploaders/job_artifact_uploader.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'app')

diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index 57678dae9ca..bc535e9fb1b 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -16,7 +16,13 @@ class JobArtifactUploader < GitlabUploader
   def open
     raise 'Only File System is supported' unless file_storage?
 
-    File.open(path, "rb")
+    File.open(path, "rb") if path
+  end
+
+  def filename
+    return 'trace.log' if model.trace?
+
+    super
   end
 
   private
-- 
cgit v1.2.1


From 25bf5555937e0ff9c7ca7e86a6090dc4b5f9cceb Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Sat, 3 Feb 2018 00:15:19 +0900
Subject: GB catches: CreateTraceArtifactService project, user pass

---
 app/workers/create_trace_artifact_worker.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/workers/create_trace_artifact_worker.rb b/app/workers/create_trace_artifact_worker.rb
index 37866da9255..11cda58021e 100644
--- a/app/workers/create_trace_artifact_worker.rb
+++ b/app/workers/create_trace_artifact_worker.rb
@@ -3,8 +3,8 @@ class CreateTraceArtifactWorker
   include PipelineQueue
 
   def perform(job_id)
-    Ci::Build.find_by(id: job_id).try do |job|
-      Ci::CreateTraceArtifactService.new(nil, nil).execute(job)
+    Ci::Build.preload(:project, :user).find_by(id: job_id).try do |job|
+      Ci::CreateTraceArtifactService.new(job.project, job.user).execute(job)
     end
   end
 end
-- 
cgit v1.2.1


From 522cadede82ebcad599f196e8b4a212529fe39d9 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Mon, 5 Feb 2018 17:05:01 +0900
Subject: Drop filename enforcement

---
 app/uploaders/job_artifact_uploader.rb | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'app')

diff --git a/app/uploaders/job_artifact_uploader.rb b/app/uploaders/job_artifact_uploader.rb
index bc535e9fb1b..ad5385f45a4 100644
--- a/app/uploaders/job_artifact_uploader.rb
+++ b/app/uploaders/job_artifact_uploader.rb
@@ -19,12 +19,6 @@ class JobArtifactUploader < GitlabUploader
     File.open(path, "rb") if path
   end
 
-  def filename
-    return 'trace.log' if model.trace?
-
-    super
-  end
-
   private
 
   def dynamic_segment
-- 
cgit v1.2.1


From bb4ebd0b708aa299775377eaf5b554fe0333ebc3 Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Mon, 5 Feb 2018 17:06:53 +0900
Subject: Make all workers in BuildFinishedWorker to run async and reorder

---
 app/workers/build_finished_worker.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index d5a4b6c1553..2acf0d14ff5 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -7,9 +7,9 @@ class BuildFinishedWorker
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
       BuildTraceSectionsWorker.perform_async(build.id)
+      BuildCoverageWorker.perform_async(build.id)
+      BuildHooksWorker.perform_async(build.id)
       CreateTraceArtifactWorker.perform_async(build.id)
-      BuildCoverageWorker.new.perform(build.id)
-      BuildHooksWorker.new.perform(build.id)
     end
   end
 end
-- 
cgit v1.2.1


From a2d79e1f2c8186fa69a91717b3d4e71a332d8bbf Mon Sep 17 00:00:00 2001
From: Shinya Maeda <shinya@gitlab.com>
Date: Tue, 6 Feb 2018 01:52:46 +0900
Subject: Reorder async/sync tasks in BuildFinishedWorker to read traces
 efficiently

---
 app/workers/build_finished_worker.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index 2acf0d14ff5..b5ed8d607b3 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -6,8 +6,11 @@ class BuildFinishedWorker
 
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
-      BuildTraceSectionsWorker.perform_async(build.id)
-      BuildCoverageWorker.perform_async(build.id)
+      # We execute that in sync as this access the files in order to access local file, and reduce IO
+      BuildTraceSectionsWorker.new.perform(build.id)
+      BuildCoverageWorker.new.perform(build.id)
+
+      # We execute that async as this are two indepentent operations that can be executed after TraceSections and Coverage
       BuildHooksWorker.perform_async(build.id)
       CreateTraceArtifactWorker.perform_async(build.id)
     end
-- 
cgit v1.2.1