Generate knapsack report for review-qa-all

Add knapsack qa report and use it to run tests in parallel

Use the RSpec runner with knapsack

The way the Knapsack runner uses exec to start rspec seems
incompatible with the way we expect it to work. Plus, it requires
specifying KNAPSACK_TEST_DIR.

Instead, we use knapsacks AllocatorBuilder to select the spec
files to run, and then start rspec as normal, via
RSpec::Core::Runner.run

This also means we can incorporate tags.

Let the job run automatically

Include KNAPSACK_TEST_FILE_PATTERN in vars

Check all defined knapsack env vars before requiring knapsack

8 files changed, 100 insertions, 1 deletions

diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 80356fa1dc2..f5ed3e1ad9a 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -165,7 +165,10 @@ review-qa-all:
   <<: *review-qa-base
   allow_failure: true
   when: manual
+  parallel: 5
   script:
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/review-qa-all_master_report.json
+    - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
     - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 
 .review-performance-base: &review-performance-base
diff --git a/qa/Gemfile b/qa/Gemfile
index 64215b24cf1..12994b85322 100644
--- a/qa/Gemfile
+++ b/qa/Gemfile
@@ -10,3 +10,4 @@ gem 'airborne', '~> 0.2.13'
 gem 'nokogiri', '~> 1.10.3'
 gem 'rspec-retry', '~> 0.6.1'
 gem 'faker', '~> 1.6', '>= 1.6.6'
+gem 'knapsack', '~> 1.17'
diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock
index a06c88b6f0a..6b0635ed0e2 100644
--- a/qa/Gemfile.lock
+++ b/qa/Gemfile.lock
@@ -39,6 +39,8 @@ GEM
       domain_name (~> 0.5)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
+    knapsack (1.17.1)
+      rake
     launchy (2.4.3)
       addressable (~> 2.3)
     method_source (0.9.0)
@@ -102,6 +104,7 @@ DEPENDENCIES
   capybara (~> 2.16.1)
   capybara-screenshot (~> 1.0.18)
   faker (~> 1.6, >= 1.6.6)
+  knapsack (~> 1.17)
   nokogiri (~> 1.10.3)
   pry-byebug (~> 3.5.1)
   rake (~> 12.3.0)
diff --git a/qa/knapsack/gitlab-ce/review-qa-all_master_report.json b/qa/knapsack/gitlab-ce/review-qa-all_master_report.json
new file mode 100644
index 00000000000..f147346ba0f
--- /dev/null
+++ b/qa/knapsack/gitlab-ce/review-qa-all_master_report.json
@@ -0,0 +1,42 @@
+{
+  "qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb": 9.697327613830566,
+  "qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb": 46.54227638244629,
+  "qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb": 10.214765310287476,
+  "qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb": 7.882027864456177,
+  "qa/specs/features/api/3_create/repository/files_spec.rb": 5.015859127044678,
+  "qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb": 12.772682905197144,
+  "qa/specs/features/browser_ui/3_create/repository/push_protected_branch_spec.rb": 29.76174831390381,
+  "qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb": 22.800872802734375,
+  "qa/specs/features/browser_ui/1_manage/login/register_spec.rb": 22.320587396621704,
+  "qa/specs/features/api/1_manage/users_spec.rb": 0.6089541912078857,
+  "qa/specs/features/browser_ui/3_create/repository/clone_spec.rb": 0.9618203639984131,
+  "qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb": 13.403101205825806,
+  "qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb": 8.810423135757446,
+  "qa/specs/features/browser_ui/3_create/repository/push_over_http_spec.rb": 7.730542182922363,
+  "qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb": 16.18057894706726,
+  "qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb": 8.31815505027771,
+  "qa/specs/features/browser_ui/3_create/repository/push_http_private_token_spec.rb": 9.48607873916626,
+  "qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb": 19.552733182907104,
+  "qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb": 17.273863554000854,
+  "qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb": 8.281434059143066,
+  "qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb": 18.047621726989746,
+  "qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb": 7.422840595245361,
+  "qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb": 3.438166856765747,
+  "qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb": 18.679633855819702,
+  "qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb": 27.943300485610962,
+  "qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb": 39.17585229873657,
+  "qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb": 40.09336972236633,
+  "qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb": 3.705310821533203,
+  "qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb": 5.812374591827393,
+  "qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb": 92.46774697303772,
+  "qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb": 100.28881478309631,
+  "qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb": 23.710937023162842,
+  "qa/specs/features/browser_ui/1_manage/login/login_via_oauth_spec.rb": 20.58603596687317,
+  "qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb": 25.460349321365356,
+  "qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb": 19.459370374679565,
+  "qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb": 6.731764793395996,
+  "qa/specs/features/browser_ui/6_release/deploy_key/add_deploy_key_spec.rb": 15.342933893203735,
+  "qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb": 11.280649185180664,
+  "qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb": 57.48992609977722,
+  "qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb": 32.5517954826355
+}
\ No newline at end of file
diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb
index 03cae3c1fe6..82510dfa03c 100644
--- a/qa/qa/runtime/env.rb
+++ b/qa/qa/runtime/env.rb
@@ -136,6 +136,10 @@ module QA
         ENV['GITLAB_QA_PASSWORD_2']
       end
 
+      def knapsack?
+        !!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN'])
+      end
+
       def ldap_username
         @ldap_username ||= ENV['GITLAB_LDAP_USERNAME']
       end
diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb
index 306913dafa6..f1cb9378de8 100644
--- a/qa/qa/specs/runner.rb
+++ b/qa/qa/specs/runner.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'rspec/core'
+require 'rspec/expectations'
+require 'knapsack'
 
 module QA
   module Specs
@@ -32,10 +34,25 @@ module QA
         end
 
         args.push(options)
-        args.push(DEFAULT_TEST_PATH_ARGS) unless options.any? { |opt| opt =~ %r{/features/} }
 
         Runtime::Browser.configure!
 
+        if Runtime::Env.knapsack?
+          allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
+
+          QA::Runtime::Logger.info ''
+          QA::Runtime::Logger.info 'Report specs:'
+          QA::Runtime::Logger.info allocator.report_node_tests.join(', ')
+          QA::Runtime::Logger.info ''
+          QA::Runtime::Logger.info 'Leftover specs:'
+          QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ')
+          QA::Runtime::Logger.info ''
+
+          args.push(['--', allocator.node_tests])
+        else
+          args.push(DEFAULT_TEST_PATH_ARGS) unless options.any? { |opt| opt =~ %r{/features/} }
+        end
+
         RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status|
           abort if status.nonzero?
         end
diff --git a/qa/spec/runtime/env_spec.rb b/qa/spec/runtime/env_spec.rb
index 04085efe2ce..2560695ef2e 100644
--- a/qa/spec/runtime/env_spec.rb
+++ b/qa/spec/runtime/env_spec.rb
@@ -168,6 +168,30 @@ describe QA::Runtime::Env do
     end
   end
 
+  describe '.knapsack?' do
+    it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do
+      stub_env('KNAPSACK_GENERATE_REPORT', 'true')
+
+      expect(described_class.knapsack?).to be_truthy
+    end
+
+    it 'returns true if KNAPSACK_REPORT_PATH is defined' do
+      stub_env('KNAPSACK_REPORT_PATH', '/a/path')
+
+      expect(described_class.knapsack?).to be_truthy
+    end
+
+    it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do
+      stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern')
+
+      expect(described_class.knapsack?).to be_truthy
+    end
+
+    it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do
+      expect(described_class.knapsack?).to be_falsey
+    end
+  end
+
   describe '.require_github_access_token!' do
     it 'raises ArgumentError if GITHUB_ACCESS_TOKEN is not defined' do
       stub_env('GITHUB_ACCESS_TOKEN', nil)
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index a368ffba711..f25dbf3a8ab 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -3,6 +3,11 @@
 require_relative '../qa'
 require 'rspec/retry'
 
+if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK']
+  require 'knapsack'
+  Knapsack::Adapters::RSpecAdapter.bind
+end
+
 %w[helpers shared_examples].each do |d|
   Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
 end