Batch insert CI rspec_profiling datash-rspec-profile-batch-testing

Instead of inserting a row after each example to an external database,
we save the CI profiling reports into the `rspec_profiling` directory
and insert the data in one batch in the update-tests-metadata CI stage.
This should make each spec run faster and also reduce the number of
PostgreSQL connections needed by concurrent CI builds.

Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/10154

3 files changed, 57 insertions, 12 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0cc95ff8436..a62a2482b25 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,6 +66,7 @@ stages:
     paths:
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
 
 .use-pg: &use-pg
   services:
@@ -159,6 +160,7 @@ stages:
       - coverage/
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
       - tmp/capybara/
     reports:
       junit: junit_rspec.xml
@@ -336,6 +338,7 @@ retrieve-tests-metadata:
     - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
     - mkdir -p rspec_flaky/
+    - mkdir -p rspec_profiling/
     - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
 
@@ -350,12 +353,13 @@ update-tests-metadata:
       - rspec_flaky/
     policy: push
   script:
-    - retry gem install fog-aws mime-types activesupport --no-document
+    - retry gem install fog-aws mime-types activesupport rspec_profiling --no-document
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
     - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
     - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
+    - scripts/insert-rspec-profiling-data
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
     - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
 
diff --git a/config/initializers/rspec_profiling.rb b/config/initializers/rspec_profiling.rb
index 2de310753a9..d7ca760ff7b 100644
--- a/config/initializers/rspec_profiling.rb
+++ b/config/initializers/rspec_profiling.rb
@@ -1,10 +1,4 @@
 module RspecProfilingExt
-  module PSQL
-    def establish_connection
-      ::RspecProfiling::Collectors::PSQL::Result.establish_connection(ENV['RSPEC_PROFILING_POSTGRES_URL'])
-    end
-  end
-
   module Git
     def branch
       if ENV['CI_COMMIT_REF_NAME']
@@ -32,14 +26,11 @@ end
 
 if Rails.env.test?
   RspecProfiling.configure do |config|
-    if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?
-      RspecProfiling::Collectors::PSQL.prepend(RspecProfilingExt::PSQL)
-      config.collector = RspecProfiling::Collectors::PSQL
-    end
-
     if ENV.key?('CI')
       RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
       RspecProfiling::Run.prepend(RspecProfilingExt::Run)
+      config.collector = RspecProfiling::Collectors::CSV
+      config.csv_path = -> { "rspec_profiling/#{Time.now.to_i}-#{SecureRandom.hex(8)}-rspec-data.csv" }
     end
   end
 end
diff --git a/scripts/insert-rspec-profiling-data b/scripts/insert-rspec-profiling-data
new file mode 100755
index 00000000000..e1fb042d098
--- /dev/null
+++ b/scripts/insert-rspec-profiling-data
@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+require 'rspec_profiling'
+
+module RspecProfiling
+  module Collectors
+    class PSQL
+      def establish_connection
+        Result.establish_connection(results_url)
+      end
+
+      def prepared?
+        connection.data_source_exists?(table)
+      end
+
+      def results_url
+        ENV['RSPEC_PROFILING_POSTGRES_URL']
+      end
+    end
+  end
+end
+
+def insert_data(path)
+  puts "#{Time.now} Inserting CI stats..."
+
+  collector = RspecProfiling::Collectors::PSQL.new
+  collector.install
+
+  files = Dir[File.join(path, "/*.csv")]
+
+  puts "#{Time.now} Detected #{files.count} files in rspec_profiling"
+
+  files.each do |filename|
+    csv_data = CSV.open(filename, headers: :first_row)
+    count = 0
+
+    puts "#{Time.now}: Inserting #{filename}..."
+
+    csv_data.each do |entry|
+      entry = entry.to_h.each_value { |x| x&.strip! }
+      collector.insert(entry)
+      count += 1
+    end
+
+    puts "#{Time.now}: Done inserting #{count} lines in #{filename}"
+  end
+end
+
+insert_data('rspec_profiling') if ENV['RSPEC_PROFILING_POSTGRES_URL'].present?