summaryrefslogtreecommitdiff
path: root/qa/spec
diff options
context:
space:
mode:
Diffstat (limited to 'qa/spec')
-rw-r--r--qa/spec/resource/base_spec.rb100
-rw-r--r--qa/spec/resource/reusable_collection_spec.rb110
-rw-r--r--qa/spec/spec_helper.rb19
-rw-r--r--qa/spec/specs/helpers/quarantine_spec.rb14
-rw-r--r--qa/spec/specs/runner_spec.rb38
-rw-r--r--qa/spec/support/formatters/test_stats_formatter_spec.rb60
-rw-r--r--qa/spec/support/shared_contexts/packages_registry_shared_context.rb2
-rw-r--r--qa/spec/support/shared_examples/merge_with_code_owner_shared_examples.rb18
-rw-r--r--qa/spec/tools/reliable_report_spec.rb2
-rw-r--r--qa/spec/tools/test_resources_data_processor_spec.rb63
10 files changed, 350 insertions, 76 deletions
diff --git a/qa/spec/resource/base_spec.rb b/qa/spec/resource/base_spec.rb
index 2dd25f983bf..eab205ec5d1 100644
--- a/qa/spec/resource/base_spec.rb
+++ b/qa/spec/resource/base_spec.rb
@@ -3,10 +3,45 @@
RSpec.describe QA::Resource::Base do
include QA::Support::Helpers::StubEnv
- let(:resource) { spy('resource', username: 'qa') }
+ let(:resource) { spy('resource') }
let(:location) { 'http://location' }
let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} }
+ before do
+ allow(QA::Tools::TestResourceDataProcessor).to receive(:collect)
+ allow(QA::Tools::TestResourceDataProcessor).to receive(:write_to_file)
+ end
+
+ shared_context 'with simple resource' do
+ subject do
+ Class.new(QA::Resource::Base) do
+ def self.name
+ 'MyResource'
+ end
+
+ attribute :test do
+ 'block'
+ end
+
+ attribute :username do
+ 'qa'
+ end
+
+ attribute :no_block
+
+ def fabricate!(*args)
+ 'any'
+ end
+
+ def self.current_url
+ 'http://stub'
+ end
+ end
+ end
+
+ let(:resource) { subject.new }
+ end
+
shared_context 'with fabrication context' do
subject do
Class.new(described_class) do
@@ -56,23 +91,29 @@ RSpec.describe QA::Resource::Base do
end
describe '.fabricate_via_api!' do
- include_context 'with fabrication context'
+ context 'when fabricating' do
+ include_context 'with fabrication context'
- it_behaves_like 'fabrication method', :fabricate_via_api!
+ it_behaves_like 'fabrication method', :fabricate_via_api!
- it 'instantiates the resource, calls resource method returns the resource' do
- expect(resource).to receive(:fabricate_via_api!).and_return(location)
+ it 'instantiates the resource, calls resource method returns the resource' do
+ expect(resource).to receive(:fabricate_via_api!).and_return(location)
- result = subject.fabricate_via_api!(resource: resource, parents: [])
+ result = subject.fabricate_via_api!(resource: resource, parents: [])
- expect(result).to eq(resource)
+ expect(result).to eq(resource)
+ end
end
context "with debug log level" do
+ include_context 'with simple resource'
+
let(:method) { 'api' }
before do
allow(QA::Runtime::Logger).to receive(:debug)
+ allow(resource).to receive(:api_support?).and_return(true)
+ allow(resource).to receive(:fabricate_via_api!)
end
it 'logs the resource and build method' do
@@ -88,27 +129,32 @@ RSpec.describe QA::Resource::Base do
end
describe '.fabricate_via_browser_ui!' do
- include_context 'with fabrication context'
+ context 'when fabricating' do
+ include_context 'with fabrication context'
- it_behaves_like 'fabrication method', :fabricate_via_browser_ui!, :fabricate!
+ it_behaves_like 'fabrication method', :fabricate_via_browser_ui!, :fabricate!
- it 'instantiates the resource and calls resource method' do
- subject.fabricate_via_browser_ui!('something', resource: resource, parents: [])
+ it 'instantiates the resource and calls resource method' do
+ subject.fabricate_via_browser_ui!('something', resource: resource, parents: [])
- expect(resource).to have_received(:fabricate!).with('something')
- end
+ expect(resource).to have_received(:fabricate!).with('something')
+ end
- it 'returns fabrication resource' do
- result = subject.fabricate_via_browser_ui!('something', resource: resource, parents: [])
+ it 'returns fabrication resource' do
+ result = subject.fabricate_via_browser_ui!('something', resource: resource, parents: [])
- expect(result).to eq(resource)
+ expect(result).to eq(resource)
+ end
end
context "with debug log level" do
+ include_context 'with simple resource'
+
let(:method) { 'browser_ui' }
before do
allow(QA::Runtime::Logger).to receive(:debug)
+ # allow(resource).to receive(:fabricate!)
end
it 'logs the resource and build method' do
@@ -123,28 +169,6 @@ RSpec.describe QA::Resource::Base do
end
end
- shared_context 'with simple resource' do
- subject do
- Class.new(QA::Resource::Base) do
- attribute :test do
- 'block'
- end
-
- attribute :no_block
-
- def fabricate!
- 'any'
- end
-
- def self.current_url
- 'http://stub'
- end
- end
- end
-
- let(:resource) { subject.new }
- end
-
describe '.attribute' do
include_context 'with simple resource'
diff --git a/qa/spec/resource/reusable_collection_spec.rb b/qa/spec/resource/reusable_collection_spec.rb
new file mode 100644
index 00000000000..9116462b396
--- /dev/null
+++ b/qa/spec/resource/reusable_collection_spec.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+RSpec.describe QA::Resource::ReusableCollection do
+ let(:reusable_resource_class) do
+ Class.new do
+ prepend QA::Resource::Reusable
+
+ attr_reader :removed
+
+ def self.name
+ 'FooReusableResource'
+ end
+
+ def comparable
+ self.class.name
+ end
+
+ def remove_via_api!
+ @removed = true
+ end
+
+ def exists?() end
+ end
+ end
+
+ let(:another_reusable_resource_class) do
+ Class.new(reusable_resource_class) do
+ def self.name
+ 'BarReusableResource'
+ end
+ end
+ end
+
+ let(:a_resource_instance) { reusable_resource_class.new }
+ let(:another_resource_instance) { another_reusable_resource_class.new }
+
+ it 'is a singleton class' do
+ expect { described_class.new }.to raise_error(NoMethodError)
+ end
+
+ subject(:collection) do
+ described_class.instance
+ end
+
+ before do
+ described_class.register_resource_classes do |c|
+ reusable_resource_class.register(c)
+ another_reusable_resource_class.register(c)
+ end
+
+ collection.resource_classes = {
+ 'FooReusableResource' => {
+ reuse_as_identifier: {
+ resource: a_resource_instance
+ }
+ },
+ 'BarReusableResource' => {
+ another_reuse_as_identifier: {
+ resource: another_resource_instance
+ }
+ }
+ }
+
+ allow(a_resource_instance).to receive(:validate_reuse)
+ allow(another_resource_instance).to receive(:validate_reuse)
+ end
+
+ after do
+ collection.resource_classes = {}
+ end
+
+ describe '#each_resource' do
+ it 'yields each resource and reuse_as identifier in the collection' do
+ expect { |blk| collection.each_resource(&blk) }
+ .to yield_successive_args(
+ [:reuse_as_identifier, a_resource_instance],
+ [:another_reuse_as_identifier, another_resource_instance]
+ )
+ end
+ end
+
+ describe '.remove_all_via_api!' do
+ before do
+ allow(a_resource_instance).to receive(:exists?).and_return(true)
+ allow(another_resource_instance).to receive(:exists?).and_return(true)
+ end
+
+ it 'removes each instance of each resource class' do
+ described_class.remove_all_via_api!
+
+ expect(a_resource_instance.removed).to be true
+ expect(another_resource_instance.removed).to be true
+ end
+ end
+
+ describe '.validate_resource_reuse' do
+ it 'validates each instance of each resource class' do
+ expect(a_resource_instance).to receive(:validate_reuse)
+ expect(another_resource_instance).to receive(:validate_reuse)
+
+ described_class.validate_resource_reuse
+ end
+ end
+
+ describe '.register_resource_classes' do
+ it 'yields the hash of resource classes in the collection' do
+ expect { |blk| described_class.register_resource_classes(&blk) }.to yield_with_args(collection.resource_classes)
+ end
+ end
+end
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 47791f76970..2a6acd6d014 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -6,6 +6,7 @@ require 'securerandom'
require 'pathname'
require 'active_support/core_ext/hash'
require 'active_support/core_ext/object/blank'
+require 'rainbow/refinement'
require_relative 'qa_deprecation_toolkit_env'
QaDeprecationToolkitEnv.configure!
@@ -27,8 +28,16 @@ RSpec.configure do |config|
config.add_formatter QA::Support::Formatters::QuarantineFormatter
config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics?
+ config.before(:suite) do |suite|
+ QA::Resource::ReusableCollection.register_resource_classes do |collection|
+ QA::Resource::ReusableProject.register(collection)
+ QA::Resource::ReusableGroup.register(collection)
+ end
+ end
+
config.prepend_before do |example|
QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n")
+ QA::Runtime::Example.current = example
# Reset fabrication counters tracked in resource base
Thread.current[:api_fabrication] = 0
@@ -65,11 +74,15 @@ RSpec.configure do |config|
end
config.after(:suite) do |suite|
- # If any tests failed, leave the resources behind to help troubleshoot
- QA::Resource::ReusableProject.remove_all_via_api! unless suite.reporter.failed_examples.present?
-
# Write all test created resources to JSON file
QA::Tools::TestResourceDataProcessor.write_to_file
+
+ # If requested, confirm that resources were used appropriately (e.g., not left with changes that interfere with
+ # further reuse)
+ QA::Resource::ReusableCollection.validate_resource_reuse if QA::Runtime::Env.validate_resource_reuse?
+
+ # If any tests failed, leave the resources behind to help troubleshoot, otherwise remove them.
+ QA::Resource::ReusableCollection.remove_all_via_api! unless suite.reporter.failed_examples.present?
end
config.append_after(:suite) do
diff --git a/qa/spec/specs/helpers/quarantine_spec.rb b/qa/spec/specs/helpers/quarantine_spec.rb
index 8ea375cdb05..0f2d592d771 100644
--- a/qa/spec/specs/helpers/quarantine_spec.rb
+++ b/qa/spec/specs/helpers/quarantine_spec.rb
@@ -104,6 +104,20 @@ RSpec.describe QA::Specs::Helpers::Quarantine do
end
describe '.skip_or_run_quarantined_tests_or_contexts' do
+ context 'with explicitly disabled quarantine' do
+ before do
+ stub_env('DISABLE_QUARANTINE', 'true')
+ end
+
+ it 'runs quarantined test' do
+ group = describe_successfully do
+ it('is pending', :quarantine) {}
+ end
+
+ expect(group.examples.first.execution_result.status).to eq(:passed)
+ end
+ end
+
context 'with no tag focused' do
it 'skips quarantined tests' do
group = describe_successfully do
diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb
index 5cc9ff403cd..e52ca1fb17c 100644
--- a/qa/spec/specs/runner_spec.rb
+++ b/qa/spec/specs/runner_spec.rb
@@ -14,6 +14,7 @@ RSpec.describe QA::Specs::Runner do
allow(QA::Runtime::Browser).to receive(:configure!)
QA::Runtime::Scenario.define(:gitlab_address, "http://gitlab.test")
+ QA::Runtime::Scenario.define(:klass, "QA::Scenario::Test::Instance::All")
end
it_behaves_like 'excludes orchestrated, transient, and geo'
@@ -43,6 +44,43 @@ RSpec.describe QA::Specs::Runner do
subject.perform
end
+ it 'writes to file when examples are more than zero' do
+ allow(RSpec::Core::Runner).to receive(:run).and_return(0)
+
+ expect(File).to receive(:open).with('no_of_examples/test_instance_all.txt', 'w') { '22' }
+
+ subject.perform
+ end
+
+ it 'does not write to file when zero examples' do
+ out.string = '0 examples,'
+ allow(RSpec::Core::Runner).to receive(:run).and_return(0)
+
+ expect(File).not_to receive(:open)
+
+ subject.perform
+ end
+
+ it 'raises error when Rspec output does not match regex' do
+ out.string = '0'
+ allow(RSpec::Core::Runner).to receive(:run).and_return(0)
+
+ expect { subject.perform }
+ .to raise_error(QA::Specs::Runner::RegexMismatchError, 'Rspec output did not match regex')
+ end
+
+ context 'when --tag is specified as an option' do
+ subject { described_class.new.tap { |runner| runner.options = %w[--tag actioncable] } }
+
+ it 'includes the option value in the file name' do
+ expect_rspec_runner_arguments(['--dry-run', '--tag', '~geo', '--tag', 'actioncable', *described_class::DEFAULT_TEST_PATH_ARGS], [$stderr, anything])
+
+ expect(File).to receive(:open).with('no_of_examples/test_instance_all_actioncable.txt', 'w') { '22' }
+
+ subject.perform
+ end
+ end
+
after do
QA::Runtime::Scenario.attributes.delete(:count_examples_only)
end
diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb
index 2bfd7863653..84fc3b83185 100644
--- a/qa/spec/support/formatters/test_stats_formatter_spec.rb
+++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb
@@ -1,10 +1,12 @@
# frozen_string_literal: true
require 'rspec/core/sandbox'
+require 'active_support/testing/time_helpers'
describe QA::Support::Formatters::TestStatsFormatter do
include QA::Support::Helpers::StubEnv
include QA::Specs::Helpers::RSpec
+ include ActiveSupport::Testing::TimeHelpers
let(:url) { "http://influxdb.net" }
let(:token) { "token" }
@@ -22,6 +24,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
let(:file_path) { "./qa/specs/features/#{stage}/subfolder/some_spec.rb" }
let(:ui_fabrication) { 0 }
let(:api_fabrication) { 0 }
+ let(:fabrication_resources) { {} }
let(:influx_client_args) do
{
@@ -45,7 +48,8 @@ describe QA::Support::Formatters::TestStatsFormatter do
job_name: "test-job",
merge_request: "false",
run_type: run_type,
- stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first
+ stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first,
+ testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234'
},
fields: {
id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
@@ -56,8 +60,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
retry_attempts: 0,
job_url: ci_job_url,
pipeline_url: ci_pipeline_url,
- pipeline_id: ci_pipeline_id,
- testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234'
+ pipeline_id: ci_pipeline_id
}
}
end
@@ -88,6 +91,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
before do
allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client }
+ allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources }
end
context "without influxdb variables configured" do
@@ -135,6 +139,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
it('spec', :reliable, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
end
+ expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
end
end
@@ -147,6 +152,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
it('spec', :quarantine, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
end
+ expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
end
end
@@ -162,6 +168,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
it 'exports data to influxdb with correct run type' do
run_spec
+ expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
end
end
@@ -179,6 +186,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
it 'exports data to influxdb with correct run type' do
run_spec
+ expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
end
end
@@ -195,8 +203,54 @@ describe QA::Support::Formatters::TestStatsFormatter do
it 'exports data to influxdb with fabrication times' do
run_spec
+ expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
end
end
+
+ context 'with fabrication resources' do
+ let(:fabrication_resources) do
+ {
+ 'QA::Resource::Project' => [{
+ info: "with id '1'",
+ api_path: '/project',
+ fabrication_method: :api,
+ fabrication_time: 1,
+ http_method: :post,
+ timestamp: Time.now.to_s
+ }]
+ }
+ end
+
+ let(:fabrication_data) do
+ {
+ name: 'fabrication-stats',
+ time: DateTime.strptime(ci_timestamp).to_time,
+ tags: {
+ resource: 'QA::Resource::Project',
+ fabrication_method: :api,
+ http_method: :post,
+ run_type: run_type,
+ merge_request: "false"
+ },
+ fields: {
+ fabrication_time: 1,
+ info: "with id '1'",
+ job_url: ci_job_url,
+ timestamp: Time.now.to_s
+ }
+ }
+ end
+
+ around do |example|
+ freeze_time { example.run }
+ end
+
+ it 'exports fabrication stats data to influxdb' do
+ run_spec
+
+ expect(influx_write_api).to have_received(:write).with(data: [fabrication_data])
+ end
+ end
end
end
diff --git a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb
index 348176d264b..73a6c2bd99e 100644
--- a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb
+++ b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb
@@ -32,7 +32,7 @@ module QA
runner.name = "qa-runner-#{Time.now.to_i}"
runner.tags = ["runner-for-#{package_project.group.name}"]
runner.executor = :docker
- runner.token = package_project.group.runners_token
+ runner.token = package_project.group.reload!.runners_token
end
end
diff --git a/qa/spec/support/shared_examples/merge_with_code_owner_shared_examples.rb b/qa/spec/support/shared_examples/merge_with_code_owner_shared_examples.rb
index 40a8be8202a..4bbad9bf3e5 100644
--- a/qa/spec/support/shared_examples/merge_with_code_owner_shared_examples.rb
+++ b/qa/spec/support/shared_examples/merge_with_code_owner_shared_examples.rb
@@ -31,19 +31,13 @@ module QA
end
# Require approval from code owners on the default branch
- # The default branch is already protected, and we can't update a protected branch via the API (yet)
- # so we unprotect it first and then protect it again with the desired parameters
- Resource::ProtectedBranch.unprotect_via_api! do |protected_branch|
- protected_branch.project = project
- protected_branch.branch_name = project.default_branch
- end
-
- Resource::ProtectedBranch.fabricate_via_api! do |protected_branch|
- protected_branch.project = project
- protected_branch.branch_name = project.default_branch
- protected_branch.new_branch = false
- protected_branch.require_code_owner_approval = true
+ protected_branch = Resource::ProtectedBranch.fabricate_via_api! do |branch|
+ branch.project = project
+ branch.branch_name = project.default_branch
+ branch.new_branch = false
+ branch.require_code_owner_approval = true
end
+ protected_branch.set_require_code_owner_approval
# Push a change to the file with a CODEOWNERS rule
Resource::Repository::Push.fabricate! do |push|
diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb
index 1ff62df34e0..85b2590d3aa 100644
--- a/qa/spec/tools/reliable_report_spec.rb
+++ b/qa/spec/tools/reliable_report_spec.rb
@@ -167,7 +167,7 @@ describe QA::Tools::ReliableReport do
payload: {
title: "Reliable e2e test report",
description: issue_body,
- labels: "Quality,test,type::maintenance,reliable test report"
+ labels: "Quality,test,type::maintenance,reliable test report,automation:devops-mapping-disable"
}
)
expect(slack_notifier).to have_received(:post).with(
diff --git a/qa/spec/tools/test_resources_data_processor_spec.rb b/qa/spec/tools/test_resources_data_processor_spec.rb
index 6a8c0fd06a4..5117d1d367f 100644
--- a/qa/spec/tools/test_resources_data_processor_spec.rb
+++ b/qa/spec/tools/test_resources_data_processor_spec.rb
@@ -1,33 +1,60 @@
# frozen_string_literal: true
+require 'active_support/testing/time_helpers'
+
RSpec.describe QA::Tools::TestResourceDataProcessor do
+ include QA::Support::Helpers::StubEnv
+ include ActiveSupport::Testing::TimeHelpers
+
+ subject(:processor) { Class.new(described_class).instance }
+
let(:info) { 'information' }
- let(:api_path) { '/foo' }
- let(:result) { [{ info: info, api_path: api_path }] }
+ let(:api_response) { {} }
+ let(:method) { :api }
+ let(:time) { 2 }
+ let(:api_path) { resource.api_delete_path }
+ let(:resource) { QA::Resource::Project.init { |project| project.id = 1 } }
- describe '.collect' do
- context 'when resource is not restricted' do
- let(:resource) { instance_double(QA::Resource::Project, api_delete_path: '/foo', api_response: 'foo') }
+ let(:result) do
+ {
+ 'QA::Resource::Project' => [{
+ info: info,
+ api_path: api_path,
+ fabrication_method: method,
+ fabrication_time: time,
+ http_method: :post,
+ timestamp: Time.now.to_s
+ }]
+ }
+ end
+
+ before do
+ processor.collect(resource: resource, info: info, fabrication_method: method, fabrication_time: time)
+ end
+
+ around do |example|
+ freeze_time { example.run }
+ end
- it 'collects resource' do
- expect(described_class.collect(resource, info)).to eq(result)
- end
+ describe '.collect' do
+ it 'collects and stores resource' do
+ expect(processor.resources).to eq(result)
end
+ end
+
+ describe '.write_to_file' do
+ let(:resources_file) { Pathname.new(Faker::File.file_name(dir: 'tmp', ext: 'json')) }
- context 'when resource api response is nil' do
- let(:resource) { double(QA::Resource::Project, api_delete_path: '/foo', api_response: nil) }
+ before do
+ stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', resources_file)
- it 'does not collect resource' do
- expect(described_class.collect(resource, info)).to eq(nil)
- end
+ allow(File).to receive(:write)
end
- context 'when resource is restricted' do
- let(:resource) { double(QA::Resource::Sandbox, api_delete_path: '/foo', api_response: 'foo') }
+ it 'writes applicable resources to file' do
+ processor.write_to_file
- it 'does not collect resource' do
- expect(described_class.collect(resource, info)).to eq(nil)
- end
+ expect(File).to have_received(:write).with(resources_file, JSON.pretty_generate(result))
end
end
end
View Lorry Controller Status