Merge branch '37970-ci-sections-tracking' into 'master'

Add CI build trace sections extractor

Closes #37970

See merge request gitlab-org/gitlab-ce!14511

7 files changed, 67 insertions, 0 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index dd315866e60..6ca46ae89c1 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -11,6 +11,7 @@ module Ci
 
     has_many :deployments, as: :deployable
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
+    has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
@@ -265,6 +266,10 @@ module Ci
       update_attributes(coverage: coverage) if coverage.present?
     end
 
+    def parse_trace_sections!
+      ExtractSectionsFromBuildTraceService.new(project, user).execute(self)
+    end
+
     def trace
       Gitlab::Ci::Trace.new(self)
     end
diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb
new file mode 100644
index 00000000000..ccdb95546c8
--- /dev/null
+++ b/app/models/ci/build_trace_section.rb
@@ -0,0 +1,11 @@
+module Ci
+  class BuildTraceSection < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :build, class_name: 'Ci::Build'
+    belongs_to :project
+    belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName'
+
+    validates :section_name, :build, :project, presence: true, allow_blank: false
+  end
+end
diff --git a/app/models/ci/build_trace_section_name.rb b/app/models/ci/build_trace_section_name.rb
new file mode 100644
index 00000000000..0fdcb1ea329
--- /dev/null
+++ b/app/models/ci/build_trace_section_name.rb
@@ -0,0 +1,11 @@
+module Ci
+  class BuildTraceSectionName < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :project
+    has_many :trace_sections, class_name: 'Ci::BuildTraceSection', foreign_key: :section_name_id
+
+    validates :name, :project, presence: true, allow_blank: false
+    validates :name, uniqueness: { scope: :project_id }
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index e51e70f01b7..608b545f99f 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -180,6 +180,7 @@ class Project < ActiveRecord::Base
   # bulk that doesn't involve loading the rows into memory. As a result we're
   # still using `dependent: :destroy` here.
   has_many :builds, class_name: 'Ci::Build', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName'
   has_many :runner_projects, class_name: 'Ci::RunnerProject'
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'
diff --git a/app/services/ci/extract_sections_from_build_trace_service.rb b/app/services/ci/extract_sections_from_build_trace_service.rb
new file mode 100644
index 00000000000..75f9e0f897d
--- /dev/null
+++ b/app/services/ci/extract_sections_from_build_trace_service.rb
@@ -0,0 +1,30 @@
+module Ci
+  class ExtractSectionsFromBuildTraceService < BaseService
+    def execute(build)
+      return false unless build.trace_sections.empty?
+
+      Gitlab::Database.bulk_insert(BuildTraceSection.table_name, extract_sections(build))
+      true
+    end
+
+    private
+
+    def find_or_create_name(name)
+      project.build_trace_section_names.find_or_create_by!(name: name)
+    rescue ActiveRecord::RecordInvalid
+      project.build_trace_section_names.find_by!(name: name)
+    end
+
+    def extract_sections(build)
+      build.trace.extract_sections.map do |attr|
+        name = attr.delete(:name)
+        name_record = find_or_create_name(name)
+
+        attr.merge(
+          build_id: build.id,
+          project_id: project.id,
+          section_name_id: name_record.id)
+      end
+    end
+  end
+end
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index e2a1b3dcc41..52e7d346e74 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -6,6 +6,7 @@ class BuildFinishedWorker
 
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
+      BuildTraceSectionsWorker.perform_async(build.id)
       BuildCoverageWorker.new.perform(build.id)
       BuildHooksWorker.new.perform(build.id)
     end
diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb
new file mode 100644
index 00000000000..8c57e8f767b
--- /dev/null
+++ b/app/workers/build_trace_sections_worker.rb
@@ -0,0 +1,8 @@
+class BuildTraceSectionsWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  def perform(build_id)
+    Ci::Build.find_by(id: build_id)&.parse_trace_sections!
+  end
+end