Generate parallel spec reportsqa-review-knapsack-reports

Creates reports for parallel specs and collates them into a single
HTML report and displays results on the merge request.

4 files changed, 120 insertions, 1 deletions

diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index ce019de213b..41d52c4e095 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -174,7 +174,38 @@ review-qa-all:
   script:
     - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/review-qa-all_master_report.json
     - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
-    - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
+    - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation
+
+parallel-spec-reports:
+  extends: .dedicated-runner
+  dependencies:
+    - review-qa-all
+  image: ruby:2.6-alpine
+  services: []
+  before_script: []
+  variables:
+    SETUP_DB: "false"
+    NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
+    BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
+  stage: post-test
+  allow_failure: true
+  when: manual
+  retry: 0
+  artifacts:
+    when: always
+    paths:
+      - qa/report-new.html
+      - qa/gitlab-qa-run-*
+    reports:
+      junit: qa/gitlab-qa-run-*/**/rspec-*.xml
+  script:
+    - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
+    - gem install nokogiri
+    - cd qa/gitlab-qa-run-*/gitlab-*
+    - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
+    - cd ../../..
+    - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
+    - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
 
 .review-performance-base: &review-performance-base
   <<: *review-qa-base
diff --git a/qa/Gemfile b/qa/Gemfile
index c46be8a0362..53e7cc497e2 100644
--- a/qa/Gemfile
+++ b/qa/Gemfile
@@ -10,6 +10,7 @@ gem 'selenium-webdriver', '~> 3.12'
 gem 'airborne', '~> 0.2.13'
 gem 'nokogiri', '~> 1.10.3'
 gem 'rspec-retry', '~> 0.6.1'
+gem 'rspec_junit_formatter', '~> 0.4.1'
 gem 'faker', '~> 1.6', '>= 1.6.6'
 gem 'knapsack', '~> 1.17'
 gem 'parallel_tests', '~> 2.29'
diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock
index 73aabf2c6ad..7d19366f83b 100644
--- a/qa/Gemfile.lock
+++ b/qa/Gemfile.lock
@@ -87,6 +87,8 @@ GEM
     rspec-retry (0.6.1)
       rspec-core (> 3.3)
     rspec-support (3.7.0)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rubyzip (1.2.2)
     selenium-webdriver (3.141.0)
       childprocess (~> 0.5)
@@ -116,6 +118,7 @@ DEPENDENCIES
   rake (~> 12.3.0)
   rspec (~> 3.7)
   rspec-retry (~> 0.6.1)
+  rspec_junit_formatter (~> 0.4.1) 
   selenium-webdriver (~> 3.12)
 
 BUNDLED WITH
diff --git a/scripts/merge-html-reports b/scripts/merge-html-reports
new file mode 100755
index 00000000000..7d1e15186c8
--- /dev/null
+++ b/scripts/merge-html-reports
@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+
+main_report_file = ARGV.shift
+unless main_report_file
+  puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+  exit 1
+end
+
+base_artifact_url = ARGV.shift
+unless base_artifact_url
+  puts 'usage: merge-html-reports <main-report> <base-artifact-url> [parallel reports...]'
+  exit 1
+end
+
+# Create the base report with empty body tag
+new_report = Nokogiri::HTML.parse(File.read(ARGV[0]))
+new_report.at_css('body').remove
+empty_body = Nokogiri::XML::Node.new('body', new_report)
+new_report.at_css('head').add_next_sibling(empty_body)
+
+ARGV.each do |report_file|
+  report = Nokogiri::HTML.parse(File.read(report_file))
+
+  report.css('a').each do |link|
+    link_suffix = link['href'].slice(19..-1)
+    link['href'] = base_artifact_url + link_suffix
+  end
+
+  header = report.css('div #rspec-header')
+  tests = report.css('dt[id^="example_group_"]')
+
+  tests.each do |test|
+    title = test.parent
+    group = title.parent
+    script = title.css('script')
+
+    if script.inner_html.include? 'makeYellow'
+      test.remove_class('passed')
+      test.add_class('not_implemented')
+
+      group.remove_class('passed')
+      group.add_class('not_implemented')
+      header.add_class('not_implemented')
+
+      script.remove
+      test.next_sibling.remove
+      test.next_sibling.remove
+
+    elsif script.inner_html.include? 'makeRed'
+      test.remove_class('passed')
+      test.add_class('failed')
+
+      group.remove_class('passed')
+      group.add_class('failed')
+      header.add_class('failed')
+
+      script.remove
+      test.next_sibling.remove
+      test.next_sibling.remove
+    end
+  end
+
+  duration = report.at_css('p#duration')
+  totals = report.at_css('p#totals')
+
+  duration_script = report.css('div.results script')[-2]
+  totals_script = report.css('div.results script')[-1]
+
+  duration_text = duration_script.text.slice(49..-3)
+  totals_text = totals_script.text.slice(47..-3)
+
+  duration.inner_html = duration_text
+  totals.inner_html = totals_text
+
+  duration_script.remove
+  totals_script.remove
+
+  # Add the new result after the last one to keep the test order
+  new_report.css('body')[-1].add_next_sibling(report.at_css('body'))
+end
+
+File.write(main_report_file, new_report)