diff options
author | Alessio Caiazza <acaiazza@gitlab.com> | 2017-09-25 18:54:08 +0200 |
---|---|---|
committer | Alessio Caiazza <acaiazza@gitlab.com> | 2017-10-05 15:42:25 +0200 |
commit | 91f8e734fee1f9e7a16573f7185c48f442e3bb5e (patch) | |
tree | f9eb25a636c95a3bf1bbeaec09b0c36cefc7a7b9 /spec | |
parent | f685c368eaa223035b9458b704cfd6ca768f0f7d (diff) | |
download | gitlab-ce-91f8e734fee1f9e7a16573f7185c48f442e3bb5e.tar.gz |
Add CI build trace sections extractor
Diffstat (limited to 'spec')
-rw-r--r-- | spec/factories/ci/build_trace_section_names.rb | 6 | ||||
-rw-r--r-- | spec/fixtures/trace/trace_with_sections | 15 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/trace/section_parser_spec.rb | 87 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/trace_spec.rb | 87 | ||||
-rw-r--r-- | spec/lib/gitlab/import_export/all_models.yml | 1 | ||||
-rw-r--r-- | spec/models/ci/build_spec.rb | 23 | ||||
-rw-r--r-- | spec/models/ci/build_trace_section_name_spec.rb | 12 | ||||
-rw-r--r-- | spec/models/ci/build_trace_section_spec.rb | 11 | ||||
-rw-r--r-- | spec/models/project_spec.rb | 1 | ||||
-rw-r--r-- | spec/services/ci/retry_build_service_spec.rb | 2 | ||||
-rw-r--r-- | spec/workers/build_finished_worker_spec.rb | 2 | ||||
-rw-r--r-- | spec/workers/build_trace_sections_worker_spec.rb | 23 |
12 files changed, 269 insertions, 1 deletions
diff --git a/spec/factories/ci/build_trace_section_names.rb b/spec/factories/ci/build_trace_section_names.rb new file mode 100644 index 00000000000..1c16225f0e5 --- /dev/null +++ b/spec/factories/ci/build_trace_section_names.rb @@ -0,0 +1,6 @@ +FactoryGirl.define do + factory :ci_build_trace_section_name, class: Ci::BuildTraceSectionName do + sequence(:name) { |n| "section_#{n}" } + project factory: :project + end +end diff --git a/spec/fixtures/trace/trace_with_sections b/spec/fixtures/trace/trace_with_sections new file mode 100644 index 00000000000..21dff3928c3 --- /dev/null +++ b/spec/fixtures/trace/trace_with_sections @@ -0,0 +1,15 @@ +[0KRunning with gitlab-runner dev (HEAD) + on kitsune minikube (a21b584f) +[0;m[0;33mWARNING: Namespace is empty, therefore assuming 'default'. +[0;m[0KUsing Kubernetes namespace: default +[0;m[0KUsing Kubernetes executor with image alpine:3.4 ... +[0;msection_start:1506004954:prepare_script
[0KWaiting for pod default/runner-a21b584f-project-1208199-concurrent-0sg03f to be running, status is Pending +Running on runner-a21b584f-project-1208199-concurrent-0sg03f via kitsune.local... +section_end:1506004957:prepare_script
[0Ksection_start:1506004957:get_sources
[0K[32;1mCloning repository...[0;m +Cloning into '/nolith/ci-tests'... +[32;1mChecking out dddd7a6e as master...[0;m +[32;1mSkipping Git submodules setup[0;m +section_end:1506004958:get_sources
[0Ksection_start:1506004958:restore_cache
[0Ksection_end:1506004958:restore_cache
[0Ksection_start:1506004958:download_artifacts
[0Ksection_end:1506004958:download_artifacts
[0Ksection_start:1506004958:build_script
[0K[32;1m$ whoami[0;m +root +section_end:1506004959:build_script
[0Ksection_start:1506004959:after_script
[0Ksection_end:1506004959:after_script
[0Ksection_start:1506004959:archive_cache
[0Ksection_end:1506004959:archive_cache
[0Ksection_start:1506004959:upload_artifacts
[0Ksection_end:1506004959:upload_artifacts
[0K[32;1mJob succeeded +[0;m diff --git a/spec/lib/gitlab/ci/trace/section_parser_spec.rb b/spec/lib/gitlab/ci/trace/section_parser_spec.rb new file mode 100644 index 00000000000..ca53ff87c6f --- /dev/null +++ b/spec/lib/gitlab/ci/trace/section_parser_spec.rb @@ -0,0 +1,87 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace::SectionParser do + def lines_with_pos(text) + pos = 0 + StringIO.new(text).each_line do |line| + yield line, pos + pos += line.bytesize + 1 # newline + end + end + + def build_lines(text) + to_enum(:lines_with_pos, text) + end + + def section(name, start, duration, text) + end_ = start + duration + "section_start:#{start.to_i}:#{name}\r\033[0K#{text}section_end:#{end_.to_i}:#{name}\r\033[0K" + end + + let(:lines) { build_lines('') } + subject { described_class.new(lines) } + + describe '#sections' do + before do + subject.parse! + end + + context 'empty trace' do + let(:lines) { build_lines('') } + + it { expect(subject.sections).to be_empty } + end + + context 'with a sectionless trace' do + let(:lines) { build_lines("line 1\nline 2\n") } + + it { expect(subject.sections).to be_empty } + end + + context 'with trace markers' do + let(:start_time) { Time.new(2017, 10, 5).utc } + let(:section_b_duration) { 1.second } + let(:section_a) { section('a', start_time, 0, 'a line') } + let(:section_b) { section('b', start_time, section_b_duration, "another line\n") } + let(:lines) { build_lines(section_a + section_b) } + + it { expect(subject.sections.size).to eq(2) } + it { expect(subject.sections[1][:name]).to eq('b') } + it { expect(subject.sections[1][:date_start]).to eq(start_time) } + it { expect(subject.sections[1][:date_end]).to eq(start_time + section_b_duration) } + end + end + + describe '#parse!' do + context 'multiple "section_" but no complete markers' do + let(:lines) { build_lines('section_section_section_') } + + it 'must find 3 possible section start but no complete sections' do + expect(subject).to receive(:find_next_marker).exactly(3).times.and_call_original + + subject.parse! + + expect(subject.sections).to be_empty + end + end + + context 'trace with UTF-8 chars' do + let(:line) { 'GitLab ❤️ 狸 (tanukis)\n' } + let(:trace) { section('test_section', Time.new(2017, 10, 5).utc, 3.seconds, line) } + let(:lines) { build_lines(trace) } + + it 'must handle correctly byte positioning' do + expect(subject).to receive(:find_next_marker).exactly(2).times.and_call_original + + subject.parse! + + sections = subject.sections + + expect(sections.size).to eq(1) + s = sections[0] + len = s[:byte_end] - s[:byte_start] + expect(trace.byteslice(s[:byte_start], len)).to eq(line) + end + end + end +end diff --git a/spec/lib/gitlab/ci/trace_spec.rb b/spec/lib/gitlab/ci/trace_spec.rb index 9cb0b62590a..3546532b9b4 100644 --- a/spec/lib/gitlab/ci/trace_spec.rb +++ b/spec/lib/gitlab/ci/trace_spec.rb @@ -61,6 +61,93 @@ describe Gitlab::Ci::Trace do end end + describe '#extract_sections' do + let(:log) { 'No sections' } + let(:sections) { trace.extract_sections } + + before do + trace.set(log) + end + + context 'no sections' do + it 'returs []' do + expect(trace.extract_sections).to eq([]) + end + end + + context 'multiple sections available' do + let(:log) { File.read(expand_fixture_path('trace/trace_with_sections')) } + let(:sections_data) do + [ + { name: 'prepare_script', lines: 2, duration: 3.seconds }, + { name: 'get_sources', lines: 4, duration: 1.second }, + { name: 'restore_cache', lines: 0, duration: 0.seconds }, + { name: 'download_artifacts', lines: 0, duration: 0.seconds }, + { name: 'build_script', lines: 2, duration: 1.second }, + { name: 'after_script', lines: 0, duration: 0.seconds }, + { name: 'archive_cache', lines: 0, duration: 0.seconds }, + { name: 'upload_artifacts', lines: 0, duration: 0.seconds } + ] + end + + it "returns valid sections" do + expect(sections).not_to be_empty + expect(sections.size).to eq(sections_data.size), + "expected #{sections_data.size} sections, got #{sections.size}" + + buff = StringIO.new(log) + sections.each_with_index do |s, i| + expected = sections_data[i] + + expect(s[:name]).to eq(expected[:name]) + expect(s[:date_end] - s[:date_start]).to eq(expected[:duration]) + + buff.seek(s[:byte_start], IO::SEEK_SET) + length = s[:byte_end] - s[:byte_start] + lines = buff.read(length).count("\n") + expect(lines).to eq(expected[:lines]) + end + end + end + + context 'logs contains "section_start"' do + let(:log) { "section_start:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_end:1506417477:a_section\r\033[0K"} + + it "returns only one section" do + expect(sections).not_to be_empty + expect(sections.size).to eq(1) + + section = sections[0] + expect(section[:name]).to eq('a_section') + expect(section[:byte_start]).not_to eq(section[:byte_end]), "got an empty section" + end + end + + context 'missing section_end' do + let(:log) { "section_start:1506417476:a_section\r\033[0KSome logs\nNo section_end\n"} + + it "returns no sections" do + expect(sections).to be_empty + end + end + + context 'missing section_start' do + let(:log) { "Some logs\nNo section_start\nsection_end:1506417476:a_section\r\033[0K"} + + it "returns no sections" do + expect(sections).to be_empty + end + end + + context 'inverted section_start section_end' do + let(:log) { "section_end:1506417476:a_section\r\033[0Klooks like a section_start:invalid\nsection_start:1506417477:a_section\r\033[0K"} + + it "returns no sections" do + expect(sections).to be_empty + end + end + end + describe '#set' do before do trace.set("12") diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 3fb8edeb701..195997e3501 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -266,6 +266,7 @@ project: - container_repositories - uploads - members_and_requesters +- build_trace_section_names award_emoji: - awardable - user diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 451968c7342..7fe12f92a2a 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -18,6 +18,7 @@ describe Ci::Build do it { is_expected.to belong_to(:trigger_request) } it { is_expected.to belong_to(:erased_by) } it { is_expected.to have_many(:deployments) } + it { is_expected.to have_many(:trace_sections)} it { is_expected.to validate_presence_of(:ref) } it { is_expected.to respond_to(:has_trace?) } it { is_expected.to respond_to(:trace) } @@ -320,6 +321,28 @@ describe Ci::Build do end end + describe '#parse_trace_sections!' do + context "when the build trace has sections markers," do + before do + build.trace.set(File.read(expand_fixture_path('trace/trace_with_sections'))) + end + + it "saves the correct extracted sections" do + expect(build.trace_sections).to be_empty + expect(build.parse_trace_sections!).to be(true) + expect(build.trace_sections).not_to be_empty + end + + it "fails if trace_sections isn't empty" do + expect(build.parse_trace_sections!).to be(true) + expect(build.trace_sections).not_to be_empty + + expect(build.parse_trace_sections!).to be(false) + expect(build.trace_sections).not_to be_empty + end + end + end + describe '#trace' do subject { build.trace } diff --git a/spec/models/ci/build_trace_section_name_spec.rb b/spec/models/ci/build_trace_section_name_spec.rb new file mode 100644 index 00000000000..386ee6880cb --- /dev/null +++ b/spec/models/ci/build_trace_section_name_spec.rb @@ -0,0 +1,12 @@ +require 'spec_helper' + +describe Ci::BuildTraceSectionName, model: true do + subject { build(:ci_build_trace_section_name) } + + it { is_expected.to belong_to(:project) } + it { is_expected.to have_many(:trace_sections)} + + it { is_expected.to validate_presence_of(:project) } + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_uniqueness_of(:name).scoped_to(:project_id) } +end diff --git a/spec/models/ci/build_trace_section_spec.rb b/spec/models/ci/build_trace_section_spec.rb new file mode 100644 index 00000000000..541a9a36fb8 --- /dev/null +++ b/spec/models/ci/build_trace_section_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe Ci::BuildTraceSection, model: true do + it { is_expected.to belong_to(:build)} + it { is_expected.to belong_to(:project)} + it { is_expected.to belong_to(:section_name)} + + it { is_expected.to validate_presence_of(:section_name) } + it { is_expected.to validate_presence_of(:build) } + it { is_expected.to validate_presence_of(:project) } +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 176bb568cbe..1c04cb8cb5f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -57,6 +57,7 @@ describe Project do it { is_expected.to have_many(:commit_statuses) } it { is_expected.to have_many(:pipelines) } it { is_expected.to have_many(:builds) } + it { is_expected.to have_many(:build_trace_section_names)} it { is_expected.to have_many(:runner_projects) } it { is_expected.to have_many(:runners) } it { is_expected.to have_many(:active_runners) } diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index fbb3213f42b..9db3568abee 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -20,7 +20,7 @@ describe Ci::RetryBuildService do erased_at auto_canceled_by].freeze IGNORE_ACCESSORS = - %i[type lock_version target_url base_tags + %i[type lock_version target_url base_tags trace_sections commit_id deployments erased_by_id last_deployment project_id runner_id tag_taggings taggings tags trigger_request_id user_id auto_canceled_by_id retried failure_reason].freeze diff --git a/spec/workers/build_finished_worker_spec.rb b/spec/workers/build_finished_worker_spec.rb index 8cc3f37ebe8..1a7ffd5cdbf 100644 --- a/spec/workers/build_finished_worker_spec.rb +++ b/spec/workers/build_finished_worker_spec.rb @@ -11,6 +11,8 @@ describe BuildFinishedWorker do expect(BuildHooksWorker) .to receive(:new).ordered.and_call_original + expect(BuildTraceSectionsWorker) + .to receive(:perform_async) expect_any_instance_of(BuildCoverageWorker) .to receive(:perform) expect_any_instance_of(BuildHooksWorker) diff --git a/spec/workers/build_trace_sections_worker_spec.rb b/spec/workers/build_trace_sections_worker_spec.rb new file mode 100644 index 00000000000..45243f45547 --- /dev/null +++ b/spec/workers/build_trace_sections_worker_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' + +describe BuildTraceSectionsWorker do + describe '#perform' do + context 'when build exists' do + let!(:build) { create(:ci_build) } + + it 'updates trace sections' do + expect_any_instance_of(Ci::Build) + .to receive(:parse_trace_sections!) + + described_class.new.perform(build.id) + end + end + + context 'when build does not exist' do + it 'does not raise exception' do + expect { described_class.new.perform(123) } + .not_to raise_error + end + end + end +end |