Merge branch '50963-qa-orchestrated-tests-don-t-run-due-to-wrong-tag-filters-qa' into 'master'

Resolve "[QA] Orchestrated tests don't run due to wrong tag filters"

Closes #49654 and #50963

See merge request gitlab-org/gitlab-ce!21468

14 files changed, 174 insertions, 101 deletions

diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb
index a87d925ce32..cb1a1de6b9a 100644
--- a/qa/qa/scenario/template.rb
+++ b/qa/qa/scenario/template.rb
@@ -28,12 +28,8 @@ module QA
 
         Specs::Runner.perform do |specs|
           specs.tty = true
-          specs.options =
-            if rspec_options.any?
-              rspec_options
-            else
-              ['--tag', self.class.focus.join(','), '--', ::File.expand_path('../specs/features', __dir__)]
-            end
+          specs.tags = self.class.focus
+          specs.options = rspec_options if rspec_options.any?
         end
       end
     end
diff --git a/qa/qa/scenario/test/instance.rb b/qa/qa/scenario/test/instance.rb
index a2d503cc015..b4098619e4e 100644
--- a/qa/qa/scenario/test/instance.rb
+++ b/qa/qa/scenario/test/instance.rb
@@ -27,12 +27,7 @@ module QA
 
           Specs::Runner.perform do |specs|
             specs.tty = true
-            specs.options =
-              if rspec_options.any?
-                rspec_options
-              else
-                ['--', ::File.expand_path('../../specs/features', __dir__)]
-              end
+            specs.options = rspec_options if rspec_options.any?
           end
         end
       end
diff --git a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb
index dd24e8ffba5..e558049756d 100644
--- a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb
+++ b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb
@@ -30,7 +30,7 @@ module QA
           push.project = project
           push.directory = Pathname
             .new(__dir__)
-            .join('../../../fixtures/auto_devops_rack')
+            .join('../../../../../fixtures/auto_devops_rack')
           push.commit_message = 'Create Auto DevOps compatible rack application'
         end
 
diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb
index ccb9d5591de..5b5699d8a93 100644
--- a/qa/qa/specs/runner.rb
+++ b/qa/qa/specs/runner.rb
@@ -16,9 +16,9 @@ module QA
         args.push('--tty') if tty
 
         if tags.any?
-          tags.each { |tag| args.push(['-t', tag.to_s]) }
+          tags.each { |tag| args.push(['--tag', tag.to_s]) }
         else
-          args.push(%w[-t ~orchestrated])
+          args.push(%w[--tag ~orchestrated])
         end
 
         args.push(options)
diff --git a/qa/spec/git/repository_spec.rb b/qa/spec/git/repository_spec.rb
index 5c65128d10c..53bff3bf0b3 100644
--- a/qa/spec/git/repository_spec.rb
+++ b/qa/spec/git/repository_spec.rb
@@ -29,7 +29,7 @@ describe QA::Git::Repository do
 
   def cd_empty_temp_directory
     tmp_dir = 'tmp/git-repository-spec/'
-    FileUtils.rm_r(tmp_dir) if ::File.exist?(tmp_dir)
+    FileUtils.rm_rf(tmp_dir) if ::File.exist?(tmp_dir)
     FileUtils.mkdir_p tmp_dir
     FileUtils.cd tmp_dir
   end
diff --git a/qa/spec/scenario/test/instance/all_spec.rb b/qa/spec/scenario/test/instance/all_spec.rb
index 1d96352550b..9311d1d8199 100644
--- a/qa/spec/scenario/test/instance/all_spec.rb
+++ b/qa/spec/scenario/test/instance/all_spec.rb
@@ -1,45 +1,3 @@
 describe QA::Scenario::Test::Instance::All do
-  subject do
-    Class.new(described_class) do
-      tags :rspec, :foo
-    end
-  end
-
-  context '#perform' do
-    let(:arguments) { spy('Runtime::Scenario') }
-    let(:release) { spy('Runtime::Release') }
-    let(:runner) { spy('Specs::Runner') }
-
-    before do
-      stub_const('QA::Runtime::Release', release)
-      stub_const('QA::Runtime::Scenario', arguments)
-      stub_const('QA::Specs::Runner', runner)
-
-      allow(runner).to receive(:perform).and_yield(runner)
-    end
-
-    it 'sets an address of the subject' do
-      subject.perform("hello")
-
-      expect(arguments).to have_received(:define)
-        .with(:gitlab_address, "hello")
-    end
-
-    context 'no paths' do
-      it 'calls runner with default arguments' do
-        subject.perform("test")
-
-        expect(runner).to have_received(:options=)
-          .with(['--tag', 'rspec,foo', '--', ::File.expand_path('../../../../qa/specs/features', __dir__)])
-      end
-    end
-
-    context 'specifying paths' do
-      it 'calls runner with paths' do
-        subject.perform('test', 'path1', 'path2')
-
-        expect(runner).to have_received(:options=).with(%w[path1 path2])
-      end
-    end
-  end
+  it_behaves_like 'a QA scenario class'
 end
diff --git a/qa/spec/scenario/test/instance/smoke_spec.rb b/qa/spec/scenario/test/instance/smoke_spec.rb
index 386eefae930..b5db9783af3 100644
--- a/qa/spec/scenario/test/instance/smoke_spec.rb
+++ b/qa/spec/scenario/test/instance/smoke_spec.rb
@@ -1,45 +1,5 @@
 describe QA::Scenario::Test::Instance::Smoke do
-  subject { Class.new(described_class) { tags :smoke } }
-
-  context '#perform' do
-    let(:arguments) { spy('Runtime::Scenario') }
-    let(:release) { spy('Runtime::Release') }
-    let(:runner) { spy('Specs::Runner') }
-
-    before do
-      stub_const('QA::Runtime::Release', release)
-      stub_const('QA::Runtime::Scenario', arguments)
-      stub_const('QA::Specs::Runner', runner)
-
-      allow(runner).to receive(:perform).and_yield(runner)
-    end
-
-    it 'sets an address of the subject' do
-      subject.perform("hello")
-
-      expect(arguments).to have_received(:define)
-        .with(:gitlab_address, "hello")
-    end
-
-    it 'has a smoke tag' do
-      expect(subject.focus).to eq([:smoke]) # rubocop:disable Focus
-    end
-
-    context 'no paths' do
-      it 'calls runner with default arguments' do
-        subject.perform("test")
-
-        expect(runner).to have_received(:options=)
-          .with(['--tag', 'smoke', '--', ::File.expand_path('../../../../qa/specs/features', __dir__)])
-      end
-    end
-
-    context 'specifying paths' do
-      it 'calls runner with paths' do
-        subject.perform('test', 'path1', 'path2')
-
-        expect(runner).to have_received(:options=).with(%w[path1 path2])
-      end
-    end
+  it_behaves_like 'a QA scenario class' do
+    let(:tags) { [:smoke] }
   end
 end
diff --git a/qa/spec/scenario/test/integration/github_spec.rb b/qa/spec/scenario/test/integration/github_spec.rb
new file mode 100644
index 00000000000..c2aeb1ded1d
--- /dev/null
+++ b/qa/spec/scenario/test/integration/github_spec.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+describe QA::Scenario::Test::Integration::Github do
+  context '#perform' do
+    let(:env) { spy('Runtime::Env') }
+
+    before do
+      stub_const('QA::Runtime::Env', env)
+    end
+
+    it_behaves_like 'a QA scenario class' do
+      let(:tags) { [:github] }
+
+      it 'requires a GitHub access token' do
+        subject.perform('gitlab_address')
+
+        expect(env).to have_received(:require_github_access_token!)
+      end
+    end
+  end
+end
diff --git a/qa/spec/scenario/test/integration/kubernetes_spec.rb b/qa/spec/scenario/test/integration/kubernetes_spec.rb
new file mode 100644
index 00000000000..cb43994b229
--- /dev/null
+++ b/qa/spec/scenario/test/integration/kubernetes_spec.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+describe QA::Scenario::Test::Integration::Kubernetes do
+  context '#perform' do
+    it_behaves_like 'a QA scenario class' do
+      let(:tags) { [:kubernetes] }
+    end
+  end
+end
diff --git a/qa/spec/scenario/test/integration/ldap_spec.rb b/qa/spec/scenario/test/integration/ldap_spec.rb
new file mode 100644
index 00000000000..198856aec3f
--- /dev/null
+++ b/qa/spec/scenario/test/integration/ldap_spec.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+describe QA::Scenario::Test::Integration::LDAP do
+  context '#perform' do
+    it_behaves_like 'a QA scenario class' do
+      let(:tags) { [:ldap] }
+    end
+  end
+end
diff --git a/qa/spec/scenario/test/integration/mattermost_spec.rb b/qa/spec/scenario/test/integration/mattermost_spec.rb
new file mode 100644
index 00000000000..59caf2ba2cd
--- /dev/null
+++ b/qa/spec/scenario/test/integration/mattermost_spec.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+describe QA::Scenario::Test::Integration::Mattermost do
+  context '#perform' do
+    it_behaves_like 'a QA scenario class' do
+      let(:args) { %w[gitlab_address mattermost_address] }
+      let(:tags) { [:mattermost] }
+      let(:options) { ['path1']}
+
+      it 'requires a GitHub access token' do
+        subject.perform(*args)
+
+        expect(attributes).to have_received(:define)
+          .with(:mattermost_address, 'mattermost_address')
+      end
+    end
+  end
+end
diff --git a/qa/spec/scenario/test/integration/object_storage_spec.rb b/qa/spec/scenario/test/integration/object_storage_spec.rb
new file mode 100644
index 00000000000..2b7188223e0
--- /dev/null
+++ b/qa/spec/scenario/test/integration/object_storage_spec.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+describe QA::Scenario::Test::Integration::ObjectStorage do
+  context '#perform' do
+    it_behaves_like 'a QA scenario class' do
+      let(:tags) { [:object_storage] }
+    end
+  end
+end
diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb
new file mode 100644
index 00000000000..b237b954889
--- /dev/null
+++ b/qa/spec/specs/runner_spec.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+describe QA::Specs::Runner do
+  context '#perform' do
+    before do
+      allow(QA::Runtime::Browser).to receive(:configure!)
+    end
+
+    it 'excludes the orchestrated tag by default' do
+      expect(RSpec::Core::Runner).to receive(:run)
+        .with(['--tag', '~orchestrated', File.expand_path('../../qa/specs/features', __dir__)], $stderr, $stdout)
+        .and_return(0)
+
+      subject.perform
+    end
+
+    context 'when tty is set' do
+      subject do
+        described_class.new.tap do |runner|
+          runner.tty = true
+        end
+      end
+
+      it 'sets the `--tty` flag' do
+        expect(RSpec::Core::Runner).to receive(:run)
+          .with(['--tty', '--tag', '~orchestrated', File.expand_path('../../qa/specs/features', __dir__)], $stderr, $stdout)
+          .and_return(0)
+
+        subject.perform
+      end
+    end
+
+    context 'when tags are set' do
+      subject do
+        described_class.new.tap do |runner|
+          runner.tags = %i[orchestrated github]
+        end
+      end
+
+      it 'focuses on the given tags' do
+        expect(RSpec::Core::Runner).to receive(:run)
+          .with(['--tag', 'orchestrated', '--tag', 'github', File.expand_path('../../qa/specs/features', __dir__)], $stderr, $stdout)
+          .and_return(0)
+
+        subject.perform
+      end
+    end
+  end
+end
diff --git a/qa/spec/support/shared_examples/scenario_shared_examples.rb b/qa/spec/support/shared_examples/scenario_shared_examples.rb
new file mode 100644
index 00000000000..5fd55d7d96b
--- /dev/null
+++ b/qa/spec/support/shared_examples/scenario_shared_examples.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+shared_examples 'a QA scenario class' do
+  let(:attributes) { spy('Runtime::Scenario') }
+  let(:release) { spy('Runtime::Release') }
+  let(:runner) { spy('Specs::Runner') }
+
+  let(:args) { ['gitlab_address'] }
+  let(:tags) { [] }
+  let(:options) { %w[path1 path2] }
+
+  before do
+    stub_const('QA::Runtime::Release', release)
+    stub_const('QA::Runtime::Scenario', attributes)
+    stub_const('QA::Specs::Runner', runner)
+
+    allow(runner).to receive(:perform).and_yield(runner)
+  end
+
+  it 'responds to perform' do
+    expect(subject).to respond_to(:perform)
+  end
+
+  it 'sets an address of the subject' do
+    subject.perform(*args)
+
+    expect(attributes).to have_received(:define).with(:gitlab_address, 'gitlab_address')
+  end
+
+  it 'performs before hooks' do
+    subject.perform(*args)
+
+    expect(release).to have_received(:perform_before_hooks)
+  end
+
+  it 'sets tags on runner' do
+    subject.perform(*args)
+
+    expect(runner).to have_received(:tags=).with(tags)
+  end
+
+  context 'specifying RSpec options' do
+    it 'sets options on runner' do
+      subject.perform(*args, *options)
+
+      expect(runner).to have_received(:options=).with(options)
+    end
+  end
+end