diff options
Diffstat (limited to 'qa/spec/support')
7 files changed, 371 insertions, 113 deletions
diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index 518c7407ba6..ba59588d186 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -8,14 +8,15 @@ describe QA::Support::Formatters::TestStatsFormatter do include QA::Specs::Helpers::RSpec include ActiveSupport::Testing::TimeHelpers - let(:url) { "http://influxdb.net" } - let(:token) { "token" } - let(:ci_timestamp) { "2021-02-23T20:58:41Z" } - let(:ci_job_name) { "test-job 1/5" } - let(:ci_job_url) { "url" } - let(:ci_pipeline_url) { "url" } - let(:ci_pipeline_id) { "123" } + let(:url) { 'http://influxdb.net' } + let(:token) { 'token' } + let(:ci_timestamp) { '2021-02-23T20:58:41Z' } + let(:ci_job_name) { 'test-job 1/5' } + let(:ci_job_url) { 'url' } + let(:ci_pipeline_url) { 'url' } + let(:ci_pipeline_id) { '123' } let(:run_type) { 'staging-full' } + let(:smoke) { 'false' } let(:reliable) { 'false' } let(:quarantined) { 'false' } let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) } @@ -25,6 +26,7 @@ describe QA::Support::Formatters::TestStatsFormatter do let(:ui_fabrication) { 0 } let(:api_fabrication) { 0 } let(:fabrication_resources) { {} } + let(:testcase) { 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' } let(:influx_client_args) do { @@ -42,14 +44,15 @@ describe QA::Support::Formatters::TestStatsFormatter do name: 'stats export spec', file_path: file_path.gsub('./qa/specs/features', ''), status: :passed, + smoke: smoke, reliable: reliable, quarantined: quarantined, - retried: "false", - job_name: "test-job", - merge_request: "false", + retried: 'false', + job_name: 'test-job', + merge_request: 'false', run_type: run_type, stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first, - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' + testcase: testcase }, fields: { id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]', @@ -78,12 +81,6 @@ describe QA::Support::Formatters::TestStatsFormatter do around do |example| RSpec::Core::Sandbox.sandboxed do |config| config.formatter = QA::Support::Formatters::TestStatsFormatter - - config.append_after do |example| - example.metadata[:api_fabrication] = Thread.current[:api_fabrication] - example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication] - end - config.before(:context) { RSpec.current_example = nil } example.run @@ -93,10 +90,11 @@ describe QA::Support::Formatters::TestStatsFormatter do before do allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client } allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources } + allow_any_instance_of(RSpec::Core::Example::ExecutionResult).to receive(:run_time).and_return(0) # rubocop:disable RSpec/AnyInstanceOf end - context "without influxdb variables configured" do - it "skips export without influxdb url" do + context 'without influxdb variables configured' do + it 'skips export without influxdb url' do stub_env('QA_INFLUXDB_URL', nil) stub_env('QA_INFLUXDB_TOKEN', nil) @@ -105,7 +103,7 @@ describe QA::Support::Formatters::TestStatsFormatter do expect(influx_client).not_to have_received(:create_write_api) end - it "skips export without influxdb token" do + it 'skips export without influxdb token' do stub_env('QA_INFLUXDB_URL', url) stub_env('QA_INFLUXDB_TOKEN', nil) @@ -145,6 +143,19 @@ describe QA::Support::Formatters::TestStatsFormatter do end end + context 'with smoke spec' do + let(:smoke) { 'true' } + + it 'exports data to influxdb with correct smoke tag' do + run_spec do + it('spec', :smoke, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} + end + + expect(influx_write_api).to have_received(:write).once + expect(influx_write_api).to have_received(:write).with(data: [data]) + end + end + context 'with quarantined spec' do let(:quarantined) { 'true' } @@ -210,16 +221,18 @@ describe QA::Support::Formatters::TestStatsFormatter do end context 'with fabrication runtimes' do - let(:ui_fabrication) { 10 } let(:api_fabrication) { 4 } - - before do - Thread.current[:api_fabrication] = api_fabrication - Thread.current[:browser_ui_fabrication] = ui_fabrication - end + let(:ui_fabrication) { 10 } + let(:testcase) { nil } it 'exports data to influxdb with fabrication times' do - run_spec + run_spec do + # Main logic tracks fabrication time in thread local variable and injects it as metadata from + # global after hook defined in main spec_helper. + # + # Inject the values directly since we do not load e2e test spec_helper in unit tests + it('spec', api_fabrication: 4, browser_ui_fabrication: 10) {} + end expect(influx_write_api).to have_received(:write).once expect(influx_write_api).to have_received(:write).with(data: [data]) diff --git a/qa/spec/support/loglinking_spec.rb b/qa/spec/support/loglinking_spec.rb index cba8a139767..e02ae45ee93 100644 --- a/qa/spec/support/loglinking_spec.rb +++ b/qa/spec/support/loglinking_spec.rb @@ -28,7 +28,7 @@ RSpec.describe QA::Support::Loglinking do expect(QA::Support::Loglinking.failure_metadata('foo123')).to eql(<<~ERROR.chomp) Correlation Id: foo123 - Kibana Url: https://kibana.address/app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20foo123')) + Kibana Url: https://kibana.address/app/discover#/?_a=(query:(language:kuery,query:'json.correlation_id%20:%20foo123'))&_g=(time:(from:now-24h%2Fh,to:now)) ERROR end end @@ -83,7 +83,7 @@ RSpec.describe QA::Support::Loglinking do describe '.logging_environment' do let(:staging_address) { 'https://staging.gitlab.com' } let(:staging_ref_address) { 'https://staging-ref.gitlab.com' } - let(:production_address) { 'https://www.gitlab.com' } + let(:production_address) { 'https://gitlab.com' } let(:pre_prod_address) { 'https://pre.gitlab.com' } let(:logging_env_array) do [ diff --git a/qa/spec/support/page_error_checker_spec.rb b/qa/spec/support/page_error_checker_spec.rb index b9b085fa7b9..7c8aaeb182a 100644 --- a/qa/spec/support/page_error_checker_spec.rb +++ b/qa/spec/support/page_error_checker_spec.rb @@ -238,6 +238,88 @@ RSpec.describe QA::Support::PageErrorChecker do end end + describe '::log_request_errors' do + let(:page_url) { 'https://baz.foo' } + let(:browser) { double('browser', current_url: page_url) } + let(:driver) { double('driver', browser: browser) } + let(:session) { double('session', driver: driver) } + + before do + allow(Capybara).to receive(:current_session).and_return(session) + end + + it 'logs from the error cache' do + error = { + 'url' => 'https://foo.bar', + 'status' => 500, + 'method' => 'GET', + 'headers' => { 'x-request-id' => '12345' } + } + + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return({ 'errors' => [error] }) + expect(page).to receive(:execute_script) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).to receive(:error).with(<<~ERROR.chomp) + Interceptor Api Errors + [500] GET https://foo.bar -- Correlation Id: 12345 + ERROR + + QA::Support::PageErrorChecker.log_request_errors(page) + end + + it 'removes duplicates' do + error = { + 'url' => 'https://foo.bar', + 'status' => 500, + 'method' => 'GET', + 'headers' => { 'x-request-id' => '12345' } + } + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return({ 'errors' => [error, error, error] }) + expect(page).to receive(:execute_script) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).to receive(:error).with(<<~ERROR.chomp).exactly(1).time + Interceptor Api Errors + [500] GET https://foo.bar -- Correlation Id: 12345 + ERROR + + QA::Support::PageErrorChecker.log_request_errors(page) + end + + it 'chops the url query string' do + error = { + 'url' => 'https://foo.bar?query={ sensitive-data: 12345 }', + 'status' => 500, + 'method' => 'GET', + 'headers' => { 'x-request-id' => '12345' } + } + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return({ 'errors' => [error] }) + expect(page).to receive(:execute_script) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).to receive(:error).with(<<~ERROR.chomp) + Interceptor Api Errors + [500] GET https://foo.bar -- Correlation Id: 12345 + ERROR + + QA::Support::PageErrorChecker.log_request_errors(page) + end + + it 'returns if cache is nil' do + expect(page).to receive(:driver).and_return(driver) + expect(page).to receive(:execute_script).and_return(nil) + + expect(QA::Runtime::Logger).to receive(:debug).with("Fetching API error cache for #{page_url}") + expect(QA::Runtime::Logger).not_to receive(:error) + + QA::Support::PageErrorChecker.log_request_errors(page) + end + end + describe '.logs' do before do logs_class = Class.new do diff --git a/qa/spec/support/shared_contexts/merge_train_spec_with_user_prep.rb b/qa/spec/support/shared_contexts/merge_train_spec_with_user_prep.rb new file mode 100644 index 00000000000..9d1a37cb0b8 --- /dev/null +++ b/qa/spec/support/shared_contexts/merge_train_spec_with_user_prep.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'merge train spec with user prep' do + let(:executor) { "qa-runner-#{Faker::Alphanumeric.alphanumeric(number: 8)}" } + let(:file_name) { Faker::Lorem.word } + let(:mr_title) { Faker::Lorem.sentence } + let(:admin_api_client) { Runtime::API::Client.as_admin } + + let(:user) do + Resource::User.fabricate_via_api! do |user| + user.api_client = admin_api_client + end + end + + let(:project) do + Resource::Project.fabricate_via_api! do |project| + project.name = 'pipeline-for-merge-trains' + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.project = project + runner.name = executor + runner.tags = [executor] + end + end + + let!(:project_files) do + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add .gitlab-ci.yml' + commit.add_files( + [ + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + test_merge_train: + tags: + - #{executor} + script: + - sleep 10 + - echo 'OK!' + only: + - merge_requests + YAML + }, + { + file_path: file_name, + content: Faker::Lorem.sentence + } + ] + ) + end + end + + before do + project.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + + Flow::Login.sign_in + project.visit! + Flow::MergeRequest.enable_merge_trains + + Flow::Login.sign_in(as: user) + + Resource::MergeRequest.fabricate_via_api! do |merge_request| + merge_request.title = mr_title + merge_request.project = project + merge_request.description = Faker::Lorem.sentence + merge_request.target_new_branch = false + merge_request.update_existing_file = true + merge_request.file_name = file_name + merge_request.file_content = Faker::Lorem.sentence + end.visit! + + Page::MergeRequest::Show.perform do |show| + show.has_pipeline_status?('passed') + show.try_to_merge! + end + end + + after do + runner&.remove_via_api! + user&.remove_via_api! + end + end +end diff --git a/qa/spec/support/shared_contexts/variable_inheritance_shared_context.rb b/qa/spec/support/shared_contexts/variable_inheritance_shared_context.rb new file mode 100644 index 00000000000..1dc8870d4d9 --- /dev/null +++ b/qa/spec/support/shared_contexts/variable_inheritance_shared_context.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_context 'variable inheritance test prep' do + let(:random_string) { Faker::Alphanumeric.alphanumeric(number: 8) } + + let(:group) do + Resource::Group.fabricate_via_api! do |group| + group.path = "group-for-variable-inheritance-#{random_string}" + end + end + + let(:upstream_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'upstream-variable-inheritance' + project.description = 'Project for pipeline with variable defined via UI - Upstream' + end + end + + let(:downstream1_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'downstream1-variable-inheritance' + project.description = 'Project for pipeline with variable defined via UI - Downstream' + end + end + + let(:downstream2_project) do + Resource::Project.fabricate_via_api! do |project| + project.group = group + project.name = 'downstream2-variable-inheritance' + project.description = 'Project for pipeline with variable defined via UI - Downstream' + end + end + + let!(:runner) do + Resource::Runner.fabricate! do |runner| + runner.token = group.reload!.runners_token + runner.name = random_string + runner.tags = [random_string] + end + end + + before do + Runtime::Feature.enable(:ci_trigger_forward_variables) + Flow::Login.sign_in + end + + after do + runner.remove_via_api! + Runtime::Feature.disable(:ci_trigger_forward_variables) + end + + def start_pipeline_with_variable + upstream_project.visit! + Flow::Pipeline.wait_for_latest_pipeline + Page::Project::Pipeline::Index.perform(&:click_run_pipeline_button) + Page::Project::Pipeline::New.perform do |new| + new.add_variable('TEST_VAR', 'This is great!') + new.click_run_pipeline_button + end + end + + def add_ci_file(project, files) + Resource::Repository::Commit.fabricate_via_api! do |commit| + commit.project = project + commit.commit_message = 'Add CI config file' + commit.add_files(files) + end + end + + def visit_job_page(pipeline_title, job_name) + Page::Project::Pipeline::Show.perform do |show| + show.expand_child_pipeline(title: pipeline_title) + show.click_job(job_name) + end + end + + def verify_job_log_shows_variable_value + Page::Project::Job::Show.perform do |show| + show.wait_until { show.successful? } + expect(show.output).to have_content('This is great!') + end + end + + def verify_job_log_does_not_show_variable_value + Page::Project::Job::Show.perform do |show| + show.wait_until { show.successful? } + expect(show.output).to have_no_content('This is great!') + end + end + + def upstream_child1_ci_file + { + file_path: '.child1-ci.yml', + content: <<~YAML + child1_job: + stage: test + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + + def upstream_child2_ci_file + { + file_path: '.child2-ci.yml', + content: <<~YAML + child2_job: + stage: test + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + + def downstream1_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + downstream1_job: + stage: deploy + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + + def downstream2_ci_file + { + file_path: '.gitlab-ci.yml', + content: <<~YAML + downstream2_job: + stage: deploy + tags: ["#{random_string}"] + script: + - echo $TEST_VAR + - echo Done! + YAML + } + end + end +end diff --git a/qa/spec/support/shared_examples/scenario_shared_examples.rb b/qa/spec/support/shared_examples/scenario_shared_examples.rb deleted file mode 100644 index 5e448349cf9..00000000000 --- a/qa/spec/support/shared_examples/scenario_shared_examples.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.shared_examples 'a QA scenario class' do - let(:attributes) { spy('Runtime::Scenario') } - let(:runner) { spy('Specs::Runner') } - let(:release) { spy('Runtime::Release') } - let(:feature) { spy('Runtime::Feature') } - - let(:args) { { gitlab_address: 'http://gitlab_address' } } - let(:named_options) { %w[--address http://gitlab_address] } - let(:tags) { [] } - let(:options) { %w[path1 path2] } - - before do - stub_const('QA::Specs::Runner', runner) - stub_const('QA::Runtime::Release', release) - stub_const('QA::Runtime::Scenario', attributes) - stub_const('QA::Runtime::Feature', feature) - - allow(attributes).to receive(:gitlab_address).and_return(args[:gitlab_address]) - allow(runner).to receive(:perform).and_yield(runner) - allow(QA::Runtime::Address).to receive(:valid?).and_return(true) - end - - it 'responds to perform' do - expect(subject).to respond_to(:perform) - end - - it 'sets an address of the subject' do - subject.perform(args) - - expect(attributes).to have_received(:define).with(:gitlab_address, 'http://gitlab_address').at_least(:once) - end - - it 'performs before hooks only once' do - subject.perform(args) - - expect(release).to have_received(:perform_before_hooks).once - end - - it 'sets tags on runner' do - subject.perform(args) - - expect(runner).to have_received(:tags=).with(tags) - end - - context 'specifying RSpec options' do - it 'sets options on runner' do - subject.perform(args, *options) - - expect(runner).to have_received(:options=).with(options) - end - end - - context 'with named command-line options' do - it 'converts options to attributes' do - described_class.launch!(named_options) - - args do |k, v| - expect(attributes).to have_received(:define).with(k, v) - end - end - - it 'raises an error if the option is invalid' do - expect { described_class.launch!(['--foo']) }.to raise_error(OptionParser::InvalidOption) - end - - it 'passes on options after --' do - expect(described_class).to receive(:perform).with(attributes, *%w[--tag quarantine]) - - described_class.launch!(named_options.push(*%w[-- --tag quarantine])) - end - end - end -end diff --git a/qa/spec/support/wait_for_requests_spec.rb b/qa/spec/support/wait_for_requests_spec.rb index 2492820b67f..221d61ea2b4 100644 --- a/qa/spec/support/wait_for_requests_spec.rb +++ b/qa/spec/support/wait_for_requests_spec.rb @@ -5,37 +5,38 @@ RSpec.describe QA::Support::WaitForRequests do before do allow(subject).to receive(:finished_all_ajax_requests?).and_return(true) allow(subject).to receive(:finished_loading?).and_return(true) + allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code) end context 'when skip_finished_loading_check is defaulted to false' do it 'calls finished_loading?' do - expect(subject).to receive(:finished_loading?).with(hash_including(wait: 1)) - subject.wait_for_requests + + expect(subject).to have_received(:finished_loading?).with(hash_including(wait: 1)) end end context 'when skip_finished_loading_check is true' do it 'does not call finished_loading?' do - expect(subject).not_to receive(:finished_loading?) - subject.wait_for_requests(skip_finished_loading_check: true) + + expect(subject).not_to have_received(:finished_loading?) end end context 'when skip_resp_code_check is defaulted to false' do it 'call report' do - allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page) - subject.wait_for_requests + + expect(QA::Support::PageErrorChecker).to have_received(:check_page_for_error_code).with(Capybara.page) end end context 'when skip_resp_code_check is true' do it 'does not parse for an error code' do - expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code) - subject.wait_for_requests(skip_resp_code_check: true) + + expect(QA::Support::PageErrorChecker).not_to have_received(:check_page_for_error_code) end end end |