Add CI build trace sections extractor

29 files changed, 545 insertions, 2 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index dd315866e60..72e646210a8 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -11,6 +11,7 @@ module Ci
 
     has_many :deployments, as: :deployable
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
+    has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
@@ -265,6 +266,27 @@ module Ci
       update_attributes(coverage: coverage) if coverage.present?
     end
 
+    def parse_trace_sections!
+      return false unless trace_sections.empty?
+
+      sections = trace.extract_sections.map do |attr|
+        name = attr.delete(:name)
+        name_record = begin
+                        project.build_trace_section_names.find_or_create_by!(name: name)
+                      rescue ActiveRecord::RecordInvalid
+                        project.build_trace_section_names.find_by!(name: name)
+                      end
+
+        attr.merge(
+          build_id: self.id,
+          project_id: self.project_id,
+          section_name_id: name_record.id)
+      end
+
+      Gitlab::Database.bulk_insert(Ci::BuildTraceSection.table_name, sections)
+      true
+    end
+
     def trace
       Gitlab::Ci::Trace.new(self)
     end
diff --git a/app/models/ci/build_trace_section.rb b/app/models/ci/build_trace_section.rb
new file mode 100644
index 00000000000..ccdb95546c8
--- /dev/null
+++ b/app/models/ci/build_trace_section.rb
@@ -0,0 +1,11 @@
+module Ci
+  class BuildTraceSection < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :build, class_name: 'Ci::Build'
+    belongs_to :project
+    belongs_to :section_name, class_name: 'Ci::BuildTraceSectionName'
+
+    validates :section_name, :build, :project, presence: true, allow_blank: false
+  end
+end
diff --git a/app/models/ci/build_trace_section_name.rb b/app/models/ci/build_trace_section_name.rb
new file mode 100644
index 00000000000..0fdcb1ea329
--- /dev/null
+++ b/app/models/ci/build_trace_section_name.rb
@@ -0,0 +1,11 @@
+module Ci
+  class BuildTraceSectionName < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :project
+    has_many :trace_sections, class_name: 'Ci::BuildTraceSection', foreign_key: :section_name_id
+
+    validates :name, :project, presence: true, allow_blank: false
+    validates :name, uniqueness: { scope: :project_id }
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 59b5a5b3cd7..4d4d028dd7e 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -179,6 +179,7 @@ class Project < ActiveRecord::Base
   # bulk that doesn't involve loading the rows into memory. As a result we're
   # still using `dependent: :destroy` here.
   has_many :builds, class_name: 'Ci::Build', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+  has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName'
   has_many :runner_projects, class_name: 'Ci::RunnerProject'
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'
diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb
index e2a1b3dcc41..52e7d346e74 100644
--- a/app/workers/build_finished_worker.rb
+++ b/app/workers/build_finished_worker.rb
@@ -6,6 +6,7 @@ class BuildFinishedWorker
 
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
+      BuildTraceSectionsWorker.perform_async(build.id)
       BuildCoverageWorker.new.perform(build.id)
       BuildHooksWorker.new.perform(build.id)
     end
diff --git a/app/workers/build_trace_sections_worker.rb b/app/workers/build_trace_sections_worker.rb
new file mode 100644
index 00000000000..8c57e8f767b
--- /dev/null
+++ b/app/workers/build_trace_sections_worker.rb
@@ -0,0 +1,8 @@
+class BuildTraceSectionsWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  def perform(build_id)
+    Ci::Build.find_by(id: build_id)&.parse_trace_sections!
+  end
+end
diff --git a/changelogs/unreleased/37970-ci-sections-tracking.yml b/changelogs/unreleased/37970-ci-sections-tracking.yml
new file mode 100644
index 00000000000..a9011b22c6c
--- /dev/null
+++ b/changelogs/unreleased/37970-ci-sections-tracking.yml
@@ -0,0 +1,5 @@
+---
+title: Parse and store gitlab-runner timestamped section markers
+merge_request: 14551
+author:
+type: added
diff --git a/db/migrate/20170926102201_create_ci_build_trace_sections.rb b/db/migrate/20170926102201_create_ci_build_trace_sections.rb
new file mode 100644
index 00000000000..ad11b31b75d
--- /dev/null
+++ b/db/migrate/20170926102201_create_ci_build_trace_sections.rb
@@ -0,0 +1,19 @@
+class CreateCiBuildTraceSections < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :ci_build_trace_sections do |t|
+      t.datetime_with_timezone :date_start, null: false
+      t.datetime_with_timezone :date_end, null: false
+      t.integer :byte_start, limit: 8, null: false
+      t.integer :byte_end, limit: 8, null: false
+      t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade }
+      t.integer :build_id, null: false
+      t.integer :section_name_id, null: false
+    end
+
+    add_index :ci_build_trace_sections, [:build_id, :section_name_id], unique: true
+  end
+end
diff --git a/db/migrate/20170926134436_add_build_foreign_key_to_ci_build_trace_sections.rb b/db/migrate/20170926134436_add_build_foreign_key_to_ci_build_trace_sections.rb
new file mode 100644
index 00000000000..d279463eb4b
--- /dev/null
+++ b/db/migrate/20170926134436_add_build_foreign_key_to_ci_build_trace_sections.rb
@@ -0,0 +1,15 @@
+class AddBuildForeignKeyToCiBuildTraceSections < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:ci_build_trace_sections, :ci_builds, column: :build_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_build_trace_sections, column: :build_id)
+  end
+end
diff --git a/db/migrate/20170929123651_create_ci_build_trace_section_names.rb b/db/migrate/20170929123651_create_ci_build_trace_section_names.rb
new file mode 100644
index 00000000000..88f3e60699a
--- /dev/null
+++ b/db/migrate/20170929123651_create_ci_build_trace_section_names.rb
@@ -0,0 +1,19 @@
+class CreateCiBuildTraceSectionNames < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    create_table :ci_build_trace_section_names do |t|
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }
+      t.string :name, null: false
+    end
+
+    add_index :ci_build_trace_section_names, [:project_id, :name], unique: true
+  end
+
+  def down
+    remove_foreign_key :ci_build_trace_section_names, column: :project_id
+    drop_table :ci_build_trace_section_names
+  end
+end
diff --git a/db/migrate/20170929123915_add_name_foreign_key_to_ci_build_trace_sections.rb b/db/migrate/20170929123915_add_name_foreign_key_to_ci_build_trace_sections.rb
new file mode 100644
index 00000000000..08422885a98
--- /dev/null
+++ b/db/migrate/20170929123915_add_name_foreign_key_to_ci_build_trace_sections.rb
@@ -0,0 +1,15 @@
+class AddNameForeignKeyToCiBuildTraceSections < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key(:ci_build_trace_sections, :ci_build_trace_section_names, column: :section_name_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_build_trace_sections, column: :section_name_id)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index fa1aad257db..4f2424ff2ab 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -207,6 +207,26 @@ ActiveRecord::Schema.define(version: 20171004121444) do
 
   add_index "chat_teams", ["namespace_id"], name: "index_chat_teams_on_namespace_id", unique: true, using: :btree
 
+  create_table "ci_build_trace_section_names", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.string "name", null: false
+  end
+
+  add_index "ci_build_trace_section_names", ["project_id", "name"], name: "index_ci_build_trace_section_names_on_project_id_and_name", unique: true, using: :btree
+
+  create_table "ci_build_trace_sections", force: :cascade do |t|
+    t.datetime_with_timezone "date_start", null: false
+    t.datetime_with_timezone "date_end", null: false
+    t.integer "byte_start", limit: 8, null: false
+    t.integer "byte_end", limit: 8, null: false
+    t.integer "project_id", null: false
+    t.integer "build_id", null: false
+    t.integer "section_name_id", null: false
+  end
+
+  add_index "ci_build_trace_sections", ["build_id", "section_name_id"], name: "index_ci_build_trace_sections_on_build_id_and_section_name_id", unique: true, using: :btree
+  add_index "ci_build_trace_sections", ["project_id"], name: "index_ci_build_trace_sections_on_project_id", using: :btree
+
   create_table "ci_builds", force: :cascade do |t|
     t.string "status"
     t.datetime "finished_at"
@@ -1697,6 +1717,10 @@ ActiveRecord::Schema.define(version: 20171004121444) do
 
   add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
+  add_foreign_key "ci_build_trace_section_names", "projects", on_delete: :cascade
+  add_foreign_key "ci_build_trace_sections", "ci_build_trace_section_names", column: "section_name_id", name: "fk_264e112c66", on_delete: :cascade
+  add_foreign_key "ci_build_trace_sections", "ci_builds", column: "build_id", name: "fk_4ebe41f502", on_delete: :cascade
+  add_foreign_key "ci_build_trace_sections", "projects", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
   add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
   add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index 088adbdd267..72b75791bbb 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -155,7 +155,7 @@ module Gitlab
           stream.each_line do |line|
             s = StringScanner.new(line)
             until s.eos?
-              if s.scan(/section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/)
+              if s.scan(Gitlab::Regex.build_trace_section_regex)
                 handle_section(s)
               elsif s.scan(/\e([@-_])(.*?)([@-~])/)
                 handle_sequence(s)
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 5b835bb669a..baf55b1fa07 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -27,6 +27,12 @@ module Gitlab
         end
       end
 
+      def extract_sections
+        read do |stream|
+          stream.extract_sections
+        end
+      end
+
       def set(data)
         write do |stream|
           data = job.hide_secrets(data)
diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb
new file mode 100644
index 00000000000..9bb0166c9e3
--- /dev/null
+++ b/lib/gitlab/ci/trace/section_parser.rb
@@ -0,0 +1,97 @@
+module Gitlab
+  module Ci
+    class Trace
+      class SectionParser
+        def initialize(lines)
+          @lines = lines
+        end
+
+        def parse!
+          @markers = {}
+
+          @lines.each do |line, pos|
+            parse_line(line, pos)
+          end
+        end
+
+        def sections
+          sanitize_markers.map do |name, markers|
+            start_, end_ = markers
+
+            {
+              name: name,
+              byte_start: start_[:marker],
+              byte_end: end_[:marker],
+              date_start: start_[:timestamp],
+              date_end: end_[:timestamp]
+            }
+          end
+        end
+
+        private
+
+        def parse_line(line, line_start_position)
+          s = StringScanner.new(line)
+          until s.eos?
+            find_next_marker(s) do |scanner|
+              marker_begins_at = line_start_position + scanner.pointer
+
+              if scanner.scan(Gitlab::Regex.build_trace_section_regex)
+                marker_ends_at = line_start_position + scanner.pointer
+                handle_line(scanner[1], scanner[2].to_i, scanner[3], marker_begins_at, marker_ends_at)
+                true
+              else
+                false
+              end
+            end
+          end
+        end
+
+        def sanitize_markers
+          @markers.select do |_, markers|
+            markers.size == 2 && markers[0][:action] == :start && markers[1][:action] == :end
+          end
+        end
+
+        def handle_line(action, time, name, marker_start, marker_end)
+          action = action.to_sym
+          timestamp = Time.at(time).utc
+          marker = if action == :start
+                     marker_end
+                   else
+                     marker_start
+                   end
+
+          @markers[name] ||= []
+          @markers[name] << {
+            name: name,
+            action: action,
+            timestamp: timestamp,
+            marker: marker
+          }
+        end
+
+        def beginning_of_section_regex
+          @beginning_of_section_regex ||= /section_/.freeze
+        end
+
+        def find_next_marker(s)
+          beginning_of_section_len = 8
+          maybe_marker = s.exist?(beginning_of_section_regex)
+
+          if maybe_marker.nil?
+            s.terminate
+          else
+            # repositioning at the beginning of the match
+            s.pos += maybe_marker - beginning_of_section_len
+            if block_given?
+              good_marker = yield(s)
+              # if not a good marker: Consuming the matched beginning_of_section_regex
+              s.pos += beginning_of_section_len unless good_marker
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index ab3408f48d6..d52194f688b 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -90,8 +90,25 @@ module Gitlab
           # so we just silently ignore error for now
         end
 
+        def extract_sections
+          return [] unless valid?
+
+          lines = to_enum(:each_line_with_pos)
+          parser = SectionParser.new(lines)
+
+          parser.parse!
+          parser.sections
+        end
+
         private
 
+        def each_line_with_pos
+          stream.seek(0, IO::SEEK_SET)
+          stream.each_line do |line|
+            yield [line, stream.pos - line.bytesize]
+          end
+        end
+
         def read_last_lines(limit)
           to_enum(:reverse_line).first(limit).reverse.join
         end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 58f6245579a..bd677ec4bf3 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -65,5 +65,9 @@ module Gitlab
       "can contain only lowercase letters, digits, and '-'. " \
       "Must start with a letter, and cannot end with '-'"
     end
+
+    def build_trace_section_regex
+      @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([^\r]+)\r\033\[0K/.freeze
+    end
   end
 end
diff --git a/spec/factories/ci/build_trace_section_names.rb b/spec/factories/ci/build_trace_section_names.rb
new file mode 100644
index 00000000000..1c16225f0e5
--- /dev/null
+++ b/spec/factories/ci/build_trace_section_names.rb
@@ -0,0 +1,6 @@
+FactoryGirl.define do
+  factory :ci_build_trace_section_name, class: Ci::BuildTraceSectionName do
+    sequence(:name) { |n| "section_#{n}" }
+    project factory: :project
+  end
+end
diff --git a/spec/fixtures/trace/trace_with_sections b/spec/fixtures/trace/trace_with_sections
new file mode 100644
index 00000000000..21dff3928c3
--- /dev/null
+++ b/spec/fixtures/trace/trace_with_sections
@@ -0,0 +1,15 @@
+Running with gitlab-runner dev (HEAD)
+  on kitsune minikube (a21b584f)
+WARNING: Namespace is empty, therefore assuming 'default'.
+Using Kubernetes namespace: default
+Using Kubernetes executor with image alpine:3.4 ...
+section_start:1506004954:prepare_script
Waiting for pod default/runner-a21b584f-project-1208199-concurrent-0sg03f to be running, status is Pending
+Running on runner-a21b584f-project-1208199-concurrent-0sg03f via kitsune.local...
+section_end:1506004957:prepare_script
section_start:1506004957:get_sources
Cloning repository...
+Cloning into '/nolith/ci-tests'...
+Checking out dddd7a6e as master...
+Skipping Git submodules setup
+section_end:1506004958:get_sources
section_start:1506004958:restore_cache
section_end:1506004958:restore_cache
section_start:1506004958:download_artifacts
section_end:1506004958:download_artifacts
section_start:1506004958:build_script
$ whoami
+root
+section_end:1506004959:build_script
section_start:1506004959:after_script
section_end:1506004959:after_script
section_start:1506004959:archive_cache
section_end:1506004959:archive_cache
section_start:1506004959:upload_artifacts
section_end:1506004959:upload_artifacts
Job succeeded
+
diff --git a/spec/lib/gitlab/ci/trace/section_parser_spec.rb b/spec/lib/gitlab/ci/trace/section_parser_spec.rb
new file mode 100644
index 00000000000..ca53ff87c6f
--- /dev/null
+++ b/spec/lib/gitlab/ci/trace/section_parser_spec.rb
@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::SectionParser do
+  def lines_with_pos(text)
+    pos = 0
+    StringIO.new(text).each_line do |line|
+      yield line, pos
+      pos += line.bytesize + 1 # newline
+    end
+  end
+
+  def build_lines(text)
+    to_enum(:lines_with_pos, text)
+  end
+
+  def section(name, start, duration, text)
+    end_ = start + duration
+    "section_start:#{start.to_i}:#{name}\r\033[0K#{text}section_end:#{end_.to_i}:#{name}\r\033[0K"
+  end
+
+  let(:lines) { build_lines('') }
+  subject { described_class.new(lines) }
+
+  describe '#sections' do
+    before do
+      subject.parse!
+    end
+
+    context 'empty trace' do
+      let(:lines) { build_lines('') }
+
+      it { expect(subject.sections).to be_empty }
+    end
+
+    context 'with a sectionless trace' do
+      let(:lines) { build_lines("line 1\nline 2\n") }
+
+      it { expect(subject.sections).to be_empty }
+    end
+
+    context 'with trace markers' do
+      let(:start_time) { Time.new(2017, 10, 5).utc }
+      let(:section_b_duration) { 1.second }
+      let(:section_a) { section('a', start_time, 0, 'a line') }
+      let(:section_b) { section('b', start_time, section_b_duration, "another line\n") }
+      let(:lines) { build_lines(section_a + section_b) }
+
+      it { expect(subject.sections.size).to eq(2) }
+      it { expect(subject.sections[1][:name]).to eq('b') }
+      it { expect(subject.sections[1][:date_start]).to eq(start_time) }
+      it { expect(subject.sections[1][:date_end]).to eq(start_time + section_b_duration) }
+    end
+  end
+
+  describe '#parse!' do
+    context 'multiple "section_" but no complete markers' do
+      let(:lines) { build_lines('section_section_section_') }
+
+      it 'must find 3 possible section start but no complete sections' do
+        expect(subject).to receive(:find_next_marker).exactly(3).times.and_call_original
+
+        subject.parse!
+
+        expect(subject.sections).to be_empty
+      end
+    end
+
+    context 'trace with UTF-8 chars' do
+      let(:line) { 'GitLab ❤️ 狸 (tanukis)\n' }
+      let(:trace) { section('test_section', Time.new(2017, 10, 5).utc, 3.seconds, line) }
+      let(:lines) { build_lines(trace) }
+
+      it 'must handle correctly byte positioning' do
+        expect(subject).to receive(:find_next_marker).exactly(2).times.and_call_original
+
+        subject.parse!
+
+        sections = subject.sections
+
+        expect(sections.size).to eq(1)
+        s = sections[0]
+        len = s[:byte_end] - s[:byte_start]
+        expect(trace.byteslice(s[:byte_start], len)).to eq(line)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/trace_spec.rb b/spec/lib/gitlab/ci/trace_spec.rb
index 9cb0b62590a..3546532b9b4 100644
--- a/spec/lib/gitlab/ci/trace_spec.rb
+++ b/spec/lib/gitlab/ci/trace_spec.rb
@@ -61,6 +61,93 @@ describe Gitlab::Ci::Trace do
     end
   end
 
+  describe '#extract_sections' do
+    let(:log) { 'No sections' }
+    let(:sections) { trace.extract_sections }
+
+    before do
+      trace.set(log)
+    end
+
+    context 'no sections' do
+      it 'returs []' do
+        expect(trace.extract_sections).to eq([])
+      end
+    end
+
+    context 'multiple sections available' do
+      let(:log) { File.read(expand_fixture_path('trace/trace_with_sections')) }
+      let(:sections_data) do
+        [
+          { name: 'prepare_script', lines: 2, duration: 3.seconds },
+          { name: 'get_sources', lines: 4, duration: 1.second },
+          { name: 'restore_cache', lines: 0, duration: 0.seconds },
+          { name: 'download_artifacts', lines: 0, duration: 0.seconds },
+          { name: 'build_script', lines: 2, duration: 1.second },
+          { name: 'after_script', lines: 0, duration: 0.seconds },
+          { name: 'archive_cache', lines: 0, duration: 0.seconds },
+          { name: 'upload_artifacts', lines: 0, duration: 0.seconds }
+        ]
+      end
+
+      it "returns valid sections" do
+        expect(sections).not_to be_empty
+        expect(sections.size).to eq(sections_data.size),
+                                 "expected #{sections_data.size} sections, got #{sections.size}"
+
+        buff = StringIO.new(log)
+        sections.each_with_index do |s, i|
+          expected = sections_data[i]
+
+          expect(s[:name]).to eq(expected[:name])
+          expect(s[:date_end] - s[:date_start]).to eq(expected[:duration])
+
+          buff.seek(s[:byte_start], IO::SEEK_SET)
+          length = s[:byte_end] - s[:byte_start]
+          lines = buff.read(length).count("\n")
+          expect(lines).to eq(expected[:lines])
+        end
+      end
+    end
+
+    context 'logs contains "section_start"' do
+      let(:log) { "section_start:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_end:1506417477:a_section\r\033[0K"}
+
+      it "returns only one section" do
+        expect(sections).not_to be_empty
+        expect(sections.size).to eq(1)
+
+        section = sections[0]
+        expect(section[:name]).to eq('a_section')
+        expect(section[:byte_start]).not_to eq(section[:byte_end]), "got an empty section"
+      end
+    end
+
+    context 'missing section_end' do
+      let(:log) { "section_start:1506417476:a_section\r\033[0KSome logs\nNo section_end\n"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+
+    context 'missing section_start' do
+      let(:log) { "Some logs\nNo section_start\nsection_end:1506417476:a_section\r\033[0K"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+
+    context 'inverted section_start section_end' do
+      let(:log) { "section_end:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_start:1506417477:a_section\r\033[0K"}
+
+      it "returns no sections" do
+        expect(sections).to be_empty
+      end
+    end
+  end
+
   describe '#set' do
     before do
       trace.set("12")
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index 3fb8edeb701..195997e3501 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -266,6 +266,7 @@ project:
 - container_repositories
 - uploads
 - members_and_requesters
+- build_trace_section_names
 award_emoji:
 - awardable
 - user
diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb
index 451968c7342..7fe12f92a2a 100644
--- a/spec/models/ci/build_spec.rb
+++ b/spec/models/ci/build_spec.rb
@@ -18,6 +18,7 @@ describe Ci::Build do
   it { is_expected.to belong_to(:trigger_request) }
   it { is_expected.to belong_to(:erased_by) }
   it { is_expected.to have_many(:deployments) }
+  it { is_expected.to have_many(:trace_sections)}
   it { is_expected.to validate_presence_of(:ref) }
   it { is_expected.to respond_to(:has_trace?) }
   it { is_expected.to respond_to(:trace) }
@@ -320,6 +321,28 @@ describe Ci::Build do
     end
   end
 
+  describe '#parse_trace_sections!' do
+    context "when the build trace has sections markers," do
+      before do
+        build.trace.set(File.read(expand_fixture_path('trace/trace_with_sections')))
+      end
+
+      it "saves the correct extracted sections" do
+        expect(build.trace_sections).to be_empty
+        expect(build.parse_trace_sections!).to be(true)
+        expect(build.trace_sections).not_to be_empty
+      end
+
+      it "fails if trace_sections isn't empty" do
+        expect(build.parse_trace_sections!).to be(true)
+        expect(build.trace_sections).not_to be_empty
+
+        expect(build.parse_trace_sections!).to be(false)
+        expect(build.trace_sections).not_to be_empty
+      end
+    end
+  end
+
   describe '#trace' do
     subject { build.trace }
 
diff --git a/spec/models/ci/build_trace_section_name_spec.rb b/spec/models/ci/build_trace_section_name_spec.rb
new file mode 100644
index 00000000000..386ee6880cb
--- /dev/null
+++ b/spec/models/ci/build_trace_section_name_spec.rb
@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe Ci::BuildTraceSectionName, model: true do
+  subject { build(:ci_build_trace_section_name) }
+
+  it { is_expected.to belong_to(:project) }
+  it { is_expected.to have_many(:trace_sections)}
+
+  it { is_expected.to validate_presence_of(:project) }
+  it { is_expected.to validate_presence_of(:name) }
+  it { is_expected.to validate_uniqueness_of(:name).scoped_to(:project_id) }
+end
diff --git a/spec/models/ci/build_trace_section_spec.rb b/spec/models/ci/build_trace_section_spec.rb
new file mode 100644
index 00000000000..541a9a36fb8
--- /dev/null
+++ b/spec/models/ci/build_trace_section_spec.rb
@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe Ci::BuildTraceSection, model: true do
+  it { is_expected.to belong_to(:build)}
+  it { is_expected.to belong_to(:project)}
+  it { is_expected.to belong_to(:section_name)}
+
+  it { is_expected.to validate_presence_of(:section_name) }
+  it { is_expected.to validate_presence_of(:build) }
+  it { is_expected.to validate_presence_of(:project) }
+end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 176bb568cbe..1c04cb8cb5f 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -57,6 +57,7 @@ describe Project do
     it { is_expected.to have_many(:commit_statuses) }
     it { is_expected.to have_many(:pipelines) }
     it { is_expected.to have_many(:builds) }
+    it { is_expected.to have_many(:build_trace_section_names)}
     it { is_expected.to have_many(:runner_projects) }
     it { is_expected.to have_many(:runners) }
     it { is_expected.to have_many(:active_runners) }
diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb
index fbb3213f42b..9db3568abee 100644
--- a/spec/services/ci/retry_build_service_spec.rb
+++ b/spec/services/ci/retry_build_service_spec.rb
@@ -20,7 +20,7 @@ describe Ci::RetryBuildService do
        erased_at auto_canceled_by].freeze
 
   IGNORE_ACCESSORS =
-    %i[type lock_version target_url base_tags
+    %i[type lock_version target_url base_tags trace_sections
        commit_id deployments erased_by_id last_deployment project_id
        runner_id tag_taggings taggings tags trigger_request_id
        user_id auto_canceled_by_id retried failure_reason].freeze
diff --git a/spec/workers/build_finished_worker_spec.rb b/spec/workers/build_finished_worker_spec.rb
index 8cc3f37ebe8..1a7ffd5cdbf 100644
--- a/spec/workers/build_finished_worker_spec.rb
+++ b/spec/workers/build_finished_worker_spec.rb
@@ -11,6 +11,8 @@ describe BuildFinishedWorker do
         expect(BuildHooksWorker)
           .to receive(:new).ordered.and_call_original
 
+        expect(BuildTraceSectionsWorker)
+          .to receive(:perform_async)
         expect_any_instance_of(BuildCoverageWorker)
           .to receive(:perform)
         expect_any_instance_of(BuildHooksWorker)
diff --git a/spec/workers/build_trace_sections_worker_spec.rb b/spec/workers/build_trace_sections_worker_spec.rb
new file mode 100644
index 00000000000..45243f45547
--- /dev/null
+++ b/spec/workers/build_trace_sections_worker_spec.rb
@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe BuildTraceSectionsWorker do
+  describe '#perform' do
+    context 'when build exists' do
+      let!(:build) { create(:ci_build) }
+
+      it 'updates trace sections' do
+        expect_any_instance_of(Ci::Build)
+          .to receive(:parse_trace_sections!)
+
+        described_class.new.perform(build.id)
+      end
+    end
+
+    context 'when build does not exist' do
+      it 'does not raise exception' do
+        expect { described_class.new.perform(123) }
+          .not_to raise_error
+      end
+    end
+  end
+end