Merge branch 'knapsack' into 'master'

Use Knapsack to evenly distribute specs

Try to distribute all tests evenly accross all nodes to reduce the testing time of GitLab CE.

This uses high amount of auto-scaled machines.


See merge request !4240

8 files changed, 212 insertions, 199 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e4d81ccf350..fc1e43fcd44 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,7 +2,7 @@ image: "ruby:2.1"
 
 services:
   - mysql:latest
-  - redis:latest
+  - redis:alpine
 
 cache:
   key: "ruby21"
@@ -13,234 +13,199 @@ variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   # retry tests only in CI environment
   RSPEC_RETRY_RETRY_COUNT: "3"
+  RAILS_ENV: "test"
+  SIMPLECOV: "true"
+  USE_DB: "true"
+  USE_BUNDLE_INSTALL: "true"
 
 before_script:
   - source ./scripts/prepare_build.sh
-  - ruby -v
-  - which ruby
-  - retry gem install bundler --no-ri --no-rdoc
   - cp config/gitlab.yml.example config/gitlab.yml
-  - touch log/application.log
-  - touch log/test.log
-  - retry bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"
-  - RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate
+  - bundle --version
+  - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"'
+  - retry gem install knapsack
+  - '[ "$USE_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate'
 
 stages:
+- prepare
 - test
-- notifications
+- post-test
 
-spec:feature:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:feature
-
-spec:api:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:api
-
-spec:models:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models
-
-spec:lib:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib
-
-spec:services:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:services
-
-spec:other:
-  stage: test
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
-
-spinach:project:half:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:half
-
-spinach:project:rest:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
-
-spinach:other:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:other
-
-teaspoon:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec teaspoon
-
-rubocop:
-  stage: test
-  script:
-    - bundle exec rubocop
-
-scss-lint:
-  stage: test
-  script:
-    - bundle exec rake scss_lint
-
-license-finder:
-  stage: test
-  script:
-    - bundle exec license_finder
-
-brakeman:
-  stage: test
-  script:
-    - bundle exec rake brakeman
-
-flog:
-  stage: test
-  script:
-    - bundle exec rake flog
-
-flay:
-  stage: test
-  script:
-    - bundle exec rake flay
-
-bundler:audit:
-  stage: test
-  only:
-    - master
-  script:
-    - "bundle exec bundle-audit check --update --ignore OSVDB-115941"
-
-db-migrate-reset:
-  stage: test
-  script:
-    - RAILS_ENV=test bundle exec rake db:migrate:reset
+# Prepare and merge knapsack tests
 
-# Ruby 2.2 jobs
-
-spec:feature:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-    - master
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:feature
+.knapsack-state: &knapsack-state
+  services: []
+  variables:
+    USE_DB: "false"
+    USE_BUNDLE_INSTALL: "false"
   cache:
-    key: "ruby22"
+    key: "knapsack"
     paths:
-    - vendor
-
-spec:api:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:api
-  cache:
-    key: "ruby22"
+    - knapsack/
+  artifacts:
     paths:
-    - vendor
+    - knapsack/
 
-spec:models:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
+knapsack:
+  <<: *knapsack-state
+  stage: prepare
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+    - mkdir -p knapsack/
+    - '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
+    - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
 
-spec:lib:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
+update-knapsack:
+  <<: *knapsack-state
+  stage: post-test
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+    - scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
+    - scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
+    - rm -f knapsack/*_node_*.json
 
-spec:services:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
-  script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:services
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+# Execute all testing suites
 
-spec:other:ruby22:
+.rspec-knapsack: &rspec-knapsack
   stage: test
-  image: ruby:2.2
-  only:
-  - master
   script:
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
-  cache:
-    key: "ruby22"
+    - bundle exec rake assets:precompile 2>/dev/null
+    - JOB_NAME=( $CI_BUILD_NAME )
+    - export CI_NODE_INDEX=${JOB_NAME[1]}
+    - export CI_NODE_TOTAL=${JOB_NAME[2]}
+    - export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_GENERATE_REPORT=true
+    - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
+    - knapsack rspec
+  artifacts:
     paths:
-    - vendor
-
-spinach:project:half:ruby22:
-  stage: test
-  image: ruby:2.2
-  only:
-  - master
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:half
-  cache:
-    key: "ruby22"
+    - knapsack/
+
+.spinach-knapsack: &spinach-knapsack
+  stage: test
+  script:
+    - bundle exec rake assets:precompile 2>/dev/null
+    - JOB_NAME=( $CI_BUILD_NAME )
+    - export CI_NODE_INDEX=${JOB_NAME[1]}
+    - export CI_NODE_TOTAL=${JOB_NAME[2]}
+    - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
+    - export KNAPSACK_GENERATE_REPORT=true
+    - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
+    - knapsack spinach "-r rerun"
+    # retry failed tests 3 times
+    - retry '[ ! -e tmp/spinach-rerun.txt ] || bin/spinach -r rerun $(cat tmp/spinach-rerun.txt)'
+  artifacts:
     paths:
-    - vendor
-
-spinach:project:rest:ruby22:
-  stage: test
-  image: ruby:2.2
+    - knapsack/
+
+rspec 0 20: *rspec-knapsack
+rspec 1 20: *rspec-knapsack
+rspec 2 20: *rspec-knapsack
+rspec 3 20: *rspec-knapsack
+rspec 4 20: *rspec-knapsack
+rspec 5 20: *rspec-knapsack
+rspec 6 20: *rspec-knapsack
+rspec 7 20: *rspec-knapsack
+rspec 8 20: *rspec-knapsack
+rspec 9 20: *rspec-knapsack
+rspec 10 20: *rspec-knapsack
+rspec 11 20: *rspec-knapsack
+rspec 12 20: *rspec-knapsack
+rspec 13 20: *rspec-knapsack
+rspec 14 20: *rspec-knapsack
+rspec 15 20: *rspec-knapsack
+rspec 16 20: *rspec-knapsack
+rspec 17 20: *rspec-knapsack
+rspec 18 20: *rspec-knapsack
+rspec 19 20: *rspec-knapsack
+
+spinach 0 10: *spinach-knapsack
+spinach 1 10: *spinach-knapsack
+spinach 2 10: *spinach-knapsack
+spinach 3 10: *spinach-knapsack
+spinach 4 10: *spinach-knapsack
+spinach 5 10: *spinach-knapsack
+spinach 6 10: *spinach-knapsack
+spinach 7 10: *spinach-knapsack
+spinach 8 10: *spinach-knapsack
+spinach 9 10: *spinach-knapsack
+
+# Execute all testing suites against Ruby 2.2
+
+.ruby-22: &ruby-22
+  image: "ruby:2.2"
   only:
-  - master
-  script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
+    - master
   cache:
     key: "ruby22"
     paths:
     - vendor
 
-spinach:other:ruby22:
+.rspec-knapsack-ruby22: &rspec-knapsack-ruby22
+  <<: *rspec-knapsack
+  <<: *ruby-22
+
+.spinach-knapsack-ruby22: &spinach-knapsack-ruby22
+  <<: *rspec-knapsack
+  <<: *ruby-22
+  
+rspec 0 20 ruby22: *rspec-knapsack-ruby22
+rspec 1 20 ruby22: *rspec-knapsack-ruby22
+rspec 2 20 ruby22: *rspec-knapsack-ruby22
+rspec 3 20 ruby22: *rspec-knapsack-ruby22
+rspec 4 20 ruby22: *rspec-knapsack-ruby22
+rspec 5 20 ruby22: *rspec-knapsack-ruby22
+rspec 6 20 ruby22: *rspec-knapsack-ruby22
+rspec 7 20 ruby22: *rspec-knapsack-ruby22
+rspec 8 20 ruby22: *rspec-knapsack-ruby22
+rspec 9 20 ruby22: *rspec-knapsack-ruby22
+rspec 10 20 ruby22: *rspec-knapsack-ruby22
+rspec 11 20 ruby22: *rspec-knapsack-ruby22
+rspec 12 20 ruby22: *rspec-knapsack-ruby22
+rspec 13 20 ruby22: *rspec-knapsack-ruby22
+rspec 14 20 ruby22: *rspec-knapsack-ruby22
+rspec 15 20 ruby22: *rspec-knapsack-ruby22
+rspec 16 20 ruby22: *rspec-knapsack-ruby22
+rspec 17 20 ruby22: *rspec-knapsack-ruby22
+rspec 18 20 ruby22: *rspec-knapsack-ruby22
+rspec 19 20 ruby22: *rspec-knapsack-ruby22
+
+spinach 0 10 ruby22: *spinach-knapsack-ruby22
+spinach 1 10 ruby22: *spinach-knapsack-ruby22
+spinach 2 10 ruby22: *spinach-knapsack-ruby22
+spinach 3 10 ruby22: *spinach-knapsack-ruby22
+spinach 4 10 ruby22: *spinach-knapsack-ruby22
+spinach 5 10 ruby22: *spinach-knapsack-ruby22
+spinach 6 10 ruby22: *spinach-knapsack-ruby22
+spinach 7 10 ruby22: *spinach-knapsack-ruby22
+spinach 8 10 ruby22: *spinach-knapsack-ruby22
+spinach 9 10 ruby22: *spinach-knapsack-ruby22
+
+# Other generic tests
+
+.exec: &exec
+  stage: test
+  script:
+    - bundle exec $CI_BUILD_NAME
+
+teaspoon: *exec
+rubocop: *exec
+rake scss_lint: *exec
+rake brakeman: *exec
+rake flog: *exec
+rake flay: *exec
+rake db:migrate:reset: *exec
+license_finder: *exec
+
+bundler:audit:
   stage: test
-  image: ruby:2.2
   only:
-  - master
+    - master
   script:
-    - RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
-    - RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:other
-  cache:
-    key: "ruby22"
-    paths:
-    - vendor
+    - "bundle exec bundle-audit check --update --ignore OSVDB-115941"
+
+# Notify slack in the end
 
 notify:slack:
-  stage: notifications
+  stage: post-test
   script:
     - ./scripts/notify_slack.sh "#builds" "Build on \`$CI_BUILD_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_BUILD_REF"/builds>"
   when: on_failure
diff --git a/Gemfile b/Gemfile
index 9af2ac8a9c1..482a6c18dd7 100644
--- a/Gemfile
+++ b/Gemfile
@@ -316,6 +316,7 @@ group :test do
   gem 'webmock', '~> 1.21.0'
   gem 'test_after_commit', '~> 0.4.2'
   gem 'sham_rack'
+  gem 'knapsack'
 end
 
 group :production do
diff --git a/Gemfile.lock b/Gemfile.lock
index 0eab33ec5ca..c85f9be7783 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -358,6 +358,9 @@ GEM
       actionpack (>= 3.0.0)
       activesupport (>= 3.0.0)
     kgio (2.10.0)
+    knapsack (1.11.0)
+      rake
+      timecop (>= 0.1.0)
     launchy (2.4.3)
       addressable (~> 2.3)
     letter_opener (1.4.1)
@@ -735,6 +738,7 @@ GEM
     thor (0.19.1)
     thread_safe (0.3.5)
     tilt (2.0.2)
+    timecop (0.8.1)
     timfel-krb5-auth (0.8.3)
     tinder (1.10.1)
       eventmachine (~> 1.0)
@@ -882,6 +886,7 @@ DEPENDENCIES
   jquery-ui-rails (~> 5.0.0)
   jwt
   kaminari (~> 0.17.0)
+  knapsack
   letter_opener_web (~> 1.3.0)
   license_finder
   licensee (~> 8.0.0)
diff --git a/Rakefile b/Rakefile
index 5dd389d5678..16261bf8ae2 100755
--- a/Rakefile
+++ b/Rakefile
@@ -3,8 +3,11 @@
 # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
 require File.expand_path('../config/application', __FILE__)
+require 'knapsack'
 
 relative_url_conf = File.expand_path('../config/initializers/relative_url', __FILE__)
 require relative_url_conf if File.exist?("#{relative_url_conf}.rb")
 
 Gitlab::Application.load_tasks
+
+Knapsack.load_tasks
diff --git a/features/support/env.rb b/features/support/env.rb
index 357d164d87f..4552db8ad77 100644
--- a/features/support/env.rb
+++ b/features/support/env.rb
@@ -11,11 +11,14 @@ ENV['RAILS_ENV'] = 'test'
 require './config/environment'
 require 'rspec/expectations'
 require 'sidekiq/testing/inline'
+require 'knapsack'
 
 require_relative 'capybara'
 require_relative 'db_cleaner'
 require_relative 'rerun'
 
+Knapsack::Adapters::SpinachAdapter.bind
+
 %w(select2_helper test_env repo_helpers).each do |f|
   require Rails.root.join('spec', 'support', f)
 end
diff --git a/scripts/merge-reports b/scripts/merge-reports
new file mode 100755
index 00000000000..f7b574001ac
--- /dev/null
+++ b/scripts/merge-reports
@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require 'yaml'
+
+main_report_file = ARGV.shift
+unless main_report_file
+  puts 'usage: merge_reports <main-report> [extra reports...]'
+  exit 1
+end
+
+puts "Loading #{main_report_file}..."
+main_report = JSON.parse(File.read(main_report_file))
+new_report = main_report.dup
+
+ARGV.each do |report_file|
+  report = JSON.parse(File.read(report_file))
+
+  # Remove existing values
+  updates = report.delete_if do |key, value|
+    main_report[key] && main_report[key] == value
+  end
+  new_report.merge!(updates)
+
+  puts "Merged #{report_file} adding #{updates.size} results."
+end
+
+File.write(main_report_file, JSON.pretty_generate(new_report))
+puts "Saved #{main_report_file}."
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index 247383aa46c..d6fb1a34e8c 100755
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -1,12 +1,16 @@
 #!/bin/bash
 
 retry() {
-    for i in $(seq 1 3); do
+    if eval "$@"; then
+        return 0
+    fi
+
+    for i in 2 1; do
+        sleep 3s
+        echo "Retrying $i..."
         if eval "$@"; then
             return 0
         fi
-        sleep 3s
-        echo "Retrying..."
     done
     return 1
 }
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 576d16e7ea3..a20f4c05971 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -15,6 +15,9 @@ require 'rspec/rails'
 require 'shoulda/matchers'
 require 'sidekiq/testing/inline'
 require 'rspec/retry'
+require 'knapsack'
+
+Knapsack::Adapters::RSpecAdapter.bind
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.