Merge pull request #9607 from chef/cache_gems

Cache gems in the verify pipeline to speed up tests

3 files changed, 70 insertions, 36 deletions

diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml
index 52bf06f8e5..69b65ad155 100644
--- a/.expeditor/verify.pipeline.yml
+++ b/.expeditor/verify.pipeline.yml
@@ -6,9 +6,6 @@ expeditor:
         automatic:
           limit: 1
       timeout_in_minutes: 30
-      retry:
-        automatic:
-          limit: 1
 
 steps:
 
@@ -19,7 +16,7 @@ steps:
 - label: "Integration Specs Ubuntu :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -32,7 +29,7 @@ steps:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - apt-get update -y
     - apt-get install -y cron locales # needed for functional tests to pass
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -43,10 +40,12 @@ steps:
 - label: "Unit Specs Ubuntu :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -54,7 +53,7 @@ steps:
 - label: "Integration Specs CentOS :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -66,7 +65,7 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - yum install -y crontabs e2fsprogs util-linux
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -77,10 +76,12 @@ steps:
 - label: "Unit Specs CentOS :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/centos-7
@@ -88,7 +89,7 @@ steps:
 - label: "Integration Specs openSUSE :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -100,7 +101,7 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - zypper install -y cronie
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -111,10 +112,12 @@ steps:
 - label: "Unit Specs openSUSE :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/opensuse-15
@@ -122,7 +125,7 @@ steps:
 - label: "Integration Specs Fedora :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -134,7 +137,7 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - yum install -y crontabs e2fsprogs util-linux
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -148,10 +151,12 @@ steps:
 - label: "Unit Specs Fedora :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/fedora-latest
@@ -160,6 +165,8 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_win_integration.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         host_os: windows
@@ -172,6 +179,8 @@ steps:
   commands:
     - scripts/bk_tests/bk_win_functional.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       windows:
         privileged: true
@@ -181,6 +190,8 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_run_choco.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         host_os: windows
@@ -190,6 +201,8 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_win_unit.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         host_os: windows
@@ -201,9 +214,11 @@ steps:
 - label: "Chefstyle :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake style
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -215,7 +230,7 @@ steps:
 - label: "Integration Specs :ruby: 2.7"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -228,7 +243,7 @@ steps:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - apt-get update -y
     - apt-get install -y cron locales net-tools # needed for functional tests to pass
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -239,10 +254,12 @@ steps:
 - label: "Unit Specs :ruby: 2.7"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: ruby:2.7-buster
@@ -254,9 +271,11 @@ steps:
 - label: "Test chef-sugar gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/chef-sugar master rake
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -264,9 +283,11 @@ steps:
 - label: "Test chef-zero gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/chef-zero master rake pedant
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -277,9 +298,11 @@ steps:
 - label: "Test cheffish gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/cheffish master rake spec
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -287,9 +310,11 @@ steps:
 - label: "Test chefspec gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chefspec/chefspec master rake
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -297,9 +322,11 @@ steps:
 - label: "Test knife-windows gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/knife-windows master rake spec
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -310,9 +337,11 @@ steps:
     - apt-get update -y
     - apt-get install -y graphviz
     - gem install bundler -v 1.17.3 # necessary for berks Gemfile.lock for now
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test berkshelf/berkshelf master rake
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
diff --git a/scripts/bk_tests/bk_container_prep.sh b/scripts/bk_tests/bk_container_prep.sh
index 8c3bda15e0..0e9c44b583 100755
--- a/scripts/bk_tests/bk_container_prep.sh
+++ b/scripts/bk_tests/bk_container_prep.sh
@@ -20,4 +20,4 @@ gem install bundler -v $(grep :bundler omnibus_overrides.rb | cut -d'"' -f2)
 bundle --version
 rm -f .bundle/config
 
-echo +++ testing
+echo "+++ Run tests"
diff --git a/scripts/bk_tests/bk_linux_exec.sh b/scripts/bk_tests/bk_linux_exec.sh
index b4bbce59f9..0325b9af44 100755
--- a/scripts/bk_tests/bk_linux_exec.sh
+++ b/scripts/bk_tests/bk_linux_exec.sh
@@ -3,15 +3,10 @@
 # Enable IPv6 in docker
 echo "--- Enabling ipv6 on docker"
 sudo systemctl stop docker
-echo "Enabling IPv6 in Docker config"
 dockerd_config="/etc/docker/daemon.json"
 sudo echo "$(jq '. + {"ipv6": true, "fixed-cidr-v6": "2001:2019:6002::/80", "ip-forward": false}' $dockerd_config)" > $dockerd_config
 sudo systemctl start docker
 
-# Verify Docker Is Running
-docker version
-sudo service docker status
-
 # Install C and C++
 echo "--- Installing package deps"
 sudo yum install -y gcc gcc-c++ openssl-devel readline-devel zlib-devel
@@ -32,17 +27,27 @@ sudo git clone https://github.com/asdf-vm/asdf.git /opt/asdf
 . /opt/asdf/asdf.sh
 . /opt/asdf/completions/asdf.bash
 
-echo "--- Installing Ruby ASDF plugin"
-/opt/asdf/bin/asdf plugin-add ruby https://github.com/asdf-vm/asdf-ruby.git
-
 echo "--- Installing Ruby 2.6.6"
+/opt/asdf/bin/asdf plugin-add ruby https://github.com/asdf-vm/asdf-ruby.git
 /opt/asdf/bin/asdf install ruby 2.6.6
 /opt/asdf/bin/asdf global ruby 2.6.6
 
 # Update Gems
+echo 'gem: --no-document' >> ~/.gemrc
 gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2)
-gem install bundler -v $(grep :bundler omnibus_overrides.rb | cut -d'"' -f2) --force --no-document
+gem install bundler -v $(grep :bundler omnibus_overrides.rb | cut -d'"' -f2) --force
 sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER )
+bundle install --jobs=3 --retry=3 --path=vendor/bundle
+
+echo "--- Config information"
+
+echo "!!!! RUBY VERSION !!!!"
 ruby --version
+echo "!!!! BUNDLE LOCATION !!!!"
 which bundle
-bundle install --jobs=3 --retry=3 --path=vendor/bundle
\ No newline at end of file
+echo "!!!! DOCKER VERSION !!!!"
+docker version
+echo "!!!! DOCKER STATUS !!!!"
+sudo service docker status
+
+echo "+++ Running tests"
\ No newline at end of file