From f11030173b909c64aa23ce7482af8fcee8554be9 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Tue, 12 Mar 2019 09:51:37 +0000
Subject: Batch insert CI rspec_profiling data

Instead of inserting a row after each example to an external database,
we save the CI profiling reports into the `rspec_profiling` directory
and insert the data in the update-tests-metadata CI stage.  This should
make each spec run faster and also reduce the number of PostgreSQL
connections needed by concurrent CI builds.

`scripts/insert-rspec-profiling-data` also inserts one file at a time
via the PostgreSQL COPY command for faster inserts. The one side effect
is that the `created_at` and `updated_at` timestamps aren't available
since they aren't generated in the CSV.

Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/10154
---
 .gitlab-ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to '.gitlab-ci.yml')

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ea1f082787..144063f208f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,6 +66,7 @@ stages:
     paths:
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
 
 .use-pg: &use-pg
   services:
@@ -159,6 +160,7 @@ stages:
       - coverage/
       - knapsack/
       - rspec_flaky/
+      - rspec_profiling/
       - tmp/capybara/
     reports:
       junit: junit_rspec.xml
@@ -336,6 +338,7 @@ retrieve-tests-metadata:
     - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
     - mkdir -p rspec_flaky/
+    - mkdir -p rspec_profiling/
     - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
 
@@ -350,7 +353,7 @@ update-tests-metadata:
       - rspec_flaky/
     policy: push
   script:
-    - retry gem install fog-aws mime-types activesupport --no-document
+    - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
     - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
     - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
@@ -358,6 +361,7 @@ update-tests-metadata:
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
     - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
+    - scripts/insert-rspec-profiling-data
 
 flaky-examples-check:
   <<: *dedicated-runner
-- 
cgit v1.2.1