diff options
Diffstat (limited to 'qa/spec')
-rw-r--r-- | qa/spec/resource/base_spec.rb | 11 | ||||
-rw-r--r-- | qa/spec/runtime/env_spec.rb | 25 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/service_ping_disabled_spec.rb (renamed from qa/spec/scenario/test/integration/kubernetes_spec.rb) | 4 | ||||
-rw-r--r-- | qa/spec/spec_helper.rb | 7 | ||||
-rw-r--r-- | qa/spec/support/formatters/test_stats_formatter_spec.rb | 2 | ||||
-rw-r--r-- | qa/spec/support/shared_contexts/packages_registry_shared_context.rb | 10 | ||||
-rw-r--r-- | qa/spec/tools/reliable_report_spec.rb | 222 |
7 files changed, 163 insertions, 118 deletions
diff --git a/qa/spec/resource/base_spec.rb b/qa/spec/resource/base_spec.rb index b24ced9e310..2a26a479436 100644 --- a/qa/spec/resource/base_spec.rb +++ b/qa/spec/resource/base_spec.rb @@ -3,8 +3,9 @@ RSpec.describe QA::Resource::Base do include QA::Support::Helpers::StubEnv - let(:resource) { spy('resource') } + let(:resource) { spy('resource', username: 'qa') } let(:location) { 'http://location' } + let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} } shared_context 'with fabrication context' do subject do @@ -68,6 +69,8 @@ RSpec.describe QA::Resource::Base do end context "with debug log level" do + let(:method) { 'api' } + before do allow(QA::Runtime::Logger).to receive(:debug) end @@ -78,7 +81,7 @@ RSpec.describe QA::Resource::Base do subject.fabricate_via_api!('something', resource: resource, parents: []) expect(QA::Runtime::Logger).to have_received(:debug) do |&msg| - expect(msg.call).to match_regex(/==> Built a MyResource via api in [\d.\-e]+ seconds+/) + expect(msg.call).to match_regex(log_regex) end end end @@ -102,6 +105,8 @@ RSpec.describe QA::Resource::Base do end context "with debug log level" do + let(:method) { 'browser_ui' } + before do allow(QA::Runtime::Logger).to receive(:debug) end @@ -112,7 +117,7 @@ RSpec.describe QA::Resource::Base do subject.fabricate_via_browser_ui!('something', resource: resource, parents: []) expect(QA::Runtime::Logger).to have_received(:debug) do |&msg| - expect(msg.call).to match_regex(/==> Built a MyResource via browser_ui in [\d.\-e]+ seconds+/) + expect(msg.call).to match_regex(log_regex) end end end diff --git a/qa/spec/runtime/env_spec.rb b/qa/spec/runtime/env_spec.rb index fb18311bb52..80d8a9a1892 100644 --- a/qa/spec/runtime/env_spec.rb +++ b/qa/spec/runtime/env_spec.rb @@ -169,6 +169,10 @@ RSpec.describe QA::Runtime::Env do end describe '.knapsack?' do + before do + stub_env('CI_NODE_TOTAL', '2') + end + it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do stub_env('KNAPSACK_GENERATE_REPORT', 'true') @@ -190,28 +194,11 @@ RSpec.describe QA::Runtime::Env do it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do expect(described_class.knapsack?).to be_falsey end - end - describe '.knapsack?' do - it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do + it 'returns false if not running in parallel job' do + stub_env('CI_NODE_TOTAL', '1') stub_env('KNAPSACK_GENERATE_REPORT', 'true') - expect(described_class.knapsack?).to be_truthy - end - - it 'returns true if KNAPSACK_REPORT_PATH is defined' do - stub_env('KNAPSACK_REPORT_PATH', '/a/path') - - expect(described_class.knapsack?).to be_truthy - end - - it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do - stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern') - - expect(described_class.knapsack?).to be_truthy - end - - it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do expect(described_class.knapsack?).to be_falsey end end diff --git a/qa/spec/scenario/test/integration/kubernetes_spec.rb b/qa/spec/scenario/test/integration/service_ping_disabled_spec.rb index d5885b97343..2db254908f0 100644 --- a/qa/spec/scenario/test/integration/kubernetes_spec.rb +++ b/qa/spec/scenario/test/integration/service_ping_disabled_spec.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -RSpec.describe QA::Scenario::Test::Integration::Kubernetes do +RSpec.describe QA::Scenario::Test::Integration::ServicePingDisabled do describe '#perform' do it_behaves_like 'a QA scenario class' do - let(:tags) { [:kubernetes] } + let(:tags) { [:service_ping_disabled] } end end end diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 640f2de0ca2..4372d9d2728 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -64,6 +64,13 @@ RSpec.configure do |config| end end + config.after(:suite) do |suite| + # If any tests failed, leave the resources behind to help troubleshoot + next if suite.reporter.failed_examples.present? + + QA::Resource::ReusableProject.remove_all_via_api! + end + config.expect_with :rspec do |expectations| expectations.include_chain_clauses_in_custom_matcher_descriptions = true end diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index f9baf9bd9d9..71ab9c1d541 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -45,7 +45,7 @@ describe QA::Support::Formatters::TestStatsFormatter do job_name: "test-job", merge_request: "false", run_type: run_type, - stage: stage + stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first }, fields: { id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]', diff --git a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb index e686d254a44..348176d264b 100644 --- a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb +++ b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb @@ -42,13 +42,13 @@ module QA end let(:project_deploy_token) do - Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token| + Resource::ProjectDeployToken.fabricate_via_api! do |deploy_token| deploy_token.name = 'package-deploy-token' deploy_token.project = package_project - deploy_token.scopes = [ - :read_repository, - :read_package_registry, - :write_package_registry + deploy_token.scopes = %w[ + read_repository + read_package_registry + write_package_registry ] end end diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb index c7d4d28fb21..a048aa2e6ea 100644 --- a/qa/spec/tools/reliable_report_spec.rb +++ b/qa/spec/tools/reliable_report_spec.rb @@ -3,62 +3,94 @@ describe QA::Tools::ReliableReport do include QA::Support::Helpers::StubEnv - subject(:reporter) { described_class.new(run_type, range) } + subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) } + let(:gitlab_response) { instance_double("RestClient::Response", code: 200, body: { web_url: issue_url }.to_json) } let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } let(:query_api) { instance_double("InfluxDB2::QueryApi") } let(:slack_channel) { "#quality-reports" } - let(:run_type) { "package-and-qa" } - let(:range) { 30 } - let(:results) { 10 } - - let(:runs) { { 0 => stable_spec, 1 => unstable_spec } } - - let(:stable_spec) do - spec_values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb" } - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: spec_values), - instance_double("InfluxDB2::FluxRecord", values: spec_values), - instance_double("InfluxDB2::FluxRecord", values: spec_values) - ] - ) + let(:range) { 14 } + let(:issue_url) { "https://gitlab.com/issue/1" } + + let(:runs) do + values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb", "stage" => "manage" } + { + 0 => instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: values), + instance_double("InfluxDB2::FluxRecord", values: values), + instance_double("InfluxDB2::FluxRecord", values: values) + ] + ) + } end - let(:unstable_spec) do - spec_values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb" } - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: { **spec_values, "status" => "passed" }), - instance_double("InfluxDB2::FluxRecord", values: spec_values), - instance_double("InfluxDB2::FluxRecord", values: spec_values) - ] - ) + let(:reliable_runs) do + values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb", "stage" => "create" } + { + 0 => instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }), + instance_double("InfluxDB2::FluxRecord", values: values), + instance_double("InfluxDB2::FluxRecord", values: values) + ] + ) + } end - def flux_query(reliable) + def flux_query(reliable:) <<~QUERY - from(bucket: "e2e-test-stats") - |> range(start: -#{range}d) - |> filter(fn: (r) => r._measurement == "test-stats" and - r.run_type == "#{run_type}" and - r.status != "pending" and - r.merge_request == "false" and - r.quarantined == "false" and - r.reliable == "#{reliable}" and - r._field == "id" - ) - |> group(columns: ["name"]) + from(bucket: "e2e-test-stats") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats") + |> filter(fn: (r) => r.run_type == "staging-full" or + r.run_type == "staging-sanity" or + r.run_type == "staging-sanity-no-admin" or + r.run_type == "production-full" or + r.run_type == "production-sanity" or + r.run_type == "package-and-qa" or + r.run_type == "nightly" + ) + |> filter(fn: (r) => r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.reliable == "#{reliable}" and + r._field == "id" + ) + |> group(columns: ["name"]) QUERY end - def table(rows, title = nil) + def markdown_section(summary, result, stage, type) + <<~SECTION.strip + ``` + #{summary_table(summary, type)} + ``` + + ## #{stage} + + <details> + <summary>Executions table</summary> + + ``` + #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")} + ``` + + </details> + SECTION + end + + def summary_table(summary, type) + table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50)) + end + + def table(rows, headings, title) Terminal::Table.new( - headings: ["name", "runs", "failed", "failure rate"], + headings: headings, style: { all_separators: true }, title: title, rows: rows @@ -67,7 +99,7 @@ describe QA::Tools::ReliableReport do def name_column(spec_name) name = "name: '#{spec_name}'" - file = "file: 'spec.rb'".ljust(110) + file = "file: 'spec.rb'".ljust(160) "#{name}\n#{file}" end @@ -75,71 +107,85 @@ describe QA::Tools::ReliableReport do before do stub_env("QA_INFLUXDB_URL", "url") stub_env("QA_INFLUXDB_TOKEN", "token") - stub_env("CI_SLACK_WEBHOOK_URL", "slack_url") + stub_env("SLACK_WEBHOOK", "slack_url") + stub_env("CI_API_V4_URL", "gitlab_api_url") + stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token") + allow(RestClient::Request).to receive(:execute).and_return(gitlab_response) allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) - allow(query_api).to receive(:query).with(query: query).and_return(runs) - end - context "with stable spec report" do - let(:query) { flux_query(false) } - let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" } - let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" } - let(:title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" } - let(:rows) do - [ - [name_column("stable spec"), 3, 0, "0%"], - [name_column("unstable spec"), 3, 2, "66.67%"] - ] - end + allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs) + allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs) + end - it "prints top stable spec report to console" do - expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout - end + context "without report creation" do + let(:create_issue) { "false" } - it "sends top stable spec report to slack" do - slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" } + it "does not create report issue", :aggregate_failures do + expect { run }.to output.to_stdout - expect { reporter.notify_top_stable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout - expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args) - expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args) + expect(RestClient::Request).not_to have_received(:execute) + expect(slack_notifier).not_to have_received(:post) end end - context "with unstable spec report" do - let(:query) { flux_query(true) } - let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" } - let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" } - let(:title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" } - let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] } + context "with report creation" do + let(:create_issue) { "true" } + let(:issue_body) do + <<~TXT.strip + [[_TOC_]] - it "prints top unstable spec report to console" do - expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout - end + # Candidates for promotion to reliable - it "sends top unstable reliable spec report to slack" do - slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" } + #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')} - expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout - expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args) - expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args) + # Reliable specs with failures + + #{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')} + TXT + end + + it "creates report issue", :aggregate_failures do + expect { run }.to output.to_stdout + + expect(RestClient::Request).to have_received(:execute).with( + method: :post, + url: "gitlab_api_url/projects/278964/issues", + verify_ssl: false, + headers: { "PRIVATE-TOKEN" => "gitlab_token" }, + payload: { + title: "Reliable spec report", + description: issue_body, + labels: "Quality,test" + } + ) + expect(slack_notifier).to have_received(:post).with( + icon_emoji: ":tanuki-protect:", + text: <<~TEXT + ```#{summary_table([['manage', 1]], 'stable')}``` + ```#{summary_table([['create', 1]], 'unstable')}``` + + #{issue_url} + TEXT + ) end end - context "without unstable reliable specs" do - let(:query) { flux_query(true) } - let(:runs) { { 0 => stable_spec } } - let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" } - let(:no_result_message) { "No unstable tests present!" } + context "with failure" do + let(:create_issue) { "true" } - it "prints no result message to console" do - expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout + before do + allow(query_api).to receive(:query).and_raise("Connection error!") end - it "skips slack notification" do - expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout - expect(slack_notifier).not_to have_received(:post) + it "notifies failure", :aggregate_failures do + expect { expect { run }.to raise_error(SystemExit) }.to output.to_stdout + + expect(slack_notifier).to have_received(:post).with( + icon_emoji: ":sadpanda:", + text: "Reliable reporter failed to create report. Error: ```Connection error!```" + ) end end end |