Cache gems in Buildkite testing

Also improve the output. This is all work synced from master.

Signed-off-by: Tim Smith <tsmith@chef.io>

3 files changed, 74 insertions, 38 deletions

diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml
index 6e4b3e39c9..55c729eb39 100644
--- a/.expeditor/verify.pipeline.yml
+++ b/.expeditor/verify.pipeline.yml
@@ -6,9 +6,6 @@ expeditor:
         automatic:
           limit: 1
       timeout_in_minutes: 30
-      retry:
-        automatic:
-          limit: 1
 
 steps:
 
@@ -19,7 +16,7 @@ steps:
 - label: "Integration Specs Ubuntu :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -32,7 +29,7 @@ steps:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - apt-get update -y
     - apt-get install -y cron locales # needed for functional tests to pass
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -43,10 +40,12 @@ steps:
 - label: "Unit Specs Ubuntu :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -54,7 +53,7 @@ steps:
 - label: "Integration Specs CentOS :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -66,7 +65,7 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - yum install -y crontabs e2fsprogs util-linux
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -77,10 +76,12 @@ steps:
 - label: "Unit Specs CentOS :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/centos-7
@@ -88,7 +89,8 @@ steps:
 - label: "Integration Specs openSUSE :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - zypper install -y cron insserv-compat
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -99,8 +101,8 @@ steps:
 - label: "Functional Specs openSUSE :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - zypper install -y cronie
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - zypper install -y cronie insserv-compat
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -111,10 +113,12 @@ steps:
 - label: "Unit Specs openSUSE :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/opensuse-15
@@ -122,7 +126,7 @@ steps:
 - label: "Integration Specs Fedora :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -134,7 +138,7 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - yum install -y crontabs e2fsprogs util-linux
-    - cd /workdir; bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - cd /workdir; bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -148,10 +152,12 @@ steps:
 - label: "Unit Specs Fedora :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/fedora-latest
@@ -160,6 +166,8 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_win_integration.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         host_os: windows
@@ -172,6 +180,8 @@ steps:
   commands:
     - scripts/bk_tests/bk_win_functional.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       windows:
         privileged: true
@@ -181,6 +191,8 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_run_choco.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         host_os: windows
@@ -190,6 +202,8 @@ steps:
   commands:
     - /workdir/scripts/bk_tests/bk_win_unit.ps1
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         host_os: windows
@@ -201,9 +215,11 @@ steps:
 - label: "Chefstyle :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen ruby_prof
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen ruby_prof
     - bundle exec rake style
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -215,7 +231,7 @@ steps:
 - label: "Integration Specs :ruby: 2.5"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:integration
   expeditor:
     executor:
@@ -228,7 +244,7 @@ steps:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
     - apt-get update -y
     - apt-get install -y cron locales net-tools # needed for functional tests to pass
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:functional
   expeditor:
     executor:
@@ -239,10 +255,12 @@ steps:
 - label: "Unit Specs :ruby: 2.5"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec rake spec:unit
     - bundle exec rake component_specs
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: ruby:2.5-buster
@@ -254,9 +272,11 @@ steps:
 - label: "Test chef-sugar gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/chef-sugar master rake
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -264,9 +284,11 @@ steps:
 - label: "Test chef-zero gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/chef-zero master rake pedant
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -277,9 +299,11 @@ steps:
 - label: "Test cheffish gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/cheffish master rake spec
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -287,9 +311,11 @@ steps:
 - label: "Test chefspec gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chefspec/chefspec master rake
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -297,9 +323,11 @@ steps:
 - label: "Test knife-windows gem :ruby: 2.6"
   commands:
     - /workdir/scripts/bk_tests/bk_container_prep.sh
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test chef/knife-windows master rake spec
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
@@ -310,9 +338,11 @@ steps:
     - apt-get update -y
     - apt-get install -y graphviz
     - gem install bundler -v 1.17.3 # necessary for berks Gemfile.lock for now
-    - bundle install --jobs=3 --retry=3 --without omnibus_package docgen
+    - bundle install --jobs=3 --retry=3 --path=vendor/bundle --without omnibus_package docgen
     - bundle exec tasks/bin/run_external_test berkshelf/berkshelf master rake
   expeditor:
+    cached_folders:
+      - vendor
     executor:
       docker:
         image: rubydistros/ubuntu-18.04
diff --git a/scripts/bk_tests/bk_container_prep.sh b/scripts/bk_tests/bk_container_prep.sh
index 8c3bda15e0..0e9c44b583 100755
--- a/scripts/bk_tests/bk_container_prep.sh
+++ b/scripts/bk_tests/bk_container_prep.sh
@@ -20,4 +20,4 @@ gem install bundler -v $(grep :bundler omnibus_overrides.rb | cut -d'"' -f2)
 bundle --version
 rm -f .bundle/config
 
-echo +++ testing
+echo "+++ Run tests"
diff --git a/scripts/bk_tests/bk_linux_exec.sh b/scripts/bk_tests/bk_linux_exec.sh
index e74f598d40..743f1f9872 100755
--- a/scripts/bk_tests/bk_linux_exec.sh
+++ b/scripts/bk_tests/bk_linux_exec.sh
@@ -3,15 +3,10 @@
 # Enable IPv6 in docker
 echo "--- Enabling ipv6 on docker"
 sudo systemctl stop docker
-echo "Enabling IPv6 in Docker config"
 dockerd_config="/etc/docker/daemon.json"
 sudo echo "$(jq '. + {"ipv6": true, "fixed-cidr-v6": "2001:2019:6002::/80", "ip-forward": false}' $dockerd_config)" > $dockerd_config
 sudo systemctl start docker
 
-# Verify Docker Is Running
-docker version
-sudo service docker status
-
 # Install C and C++
 echo "--- Installing package deps"
 sudo yum install -y gcc gcc-c++ openssl-devel readline-devel zlib-devel
@@ -32,17 +27,28 @@ sudo git clone https://github.com/asdf-vm/asdf.git /opt/asdf
 . /opt/asdf/asdf.sh
 . /opt/asdf/completions/asdf.bash
 
-echo "--- Installing Ruby ASDF plugin"
+echo "--- Installing Ruby 2.6.6"
 /opt/asdf/bin/asdf plugin-add ruby https://github.com/asdf-vm/asdf-ruby.git
 
-echo "--- Installing Ruby 2.6.5"
-/opt/asdf/bin/asdf install ruby 2.6.5
-/opt/asdf/bin/asdf global ruby 2.6.5
+/opt/asdf/bin/asdf install ruby 2.6.6
+/opt/asdf/bin/asdf global ruby 2.6.6
 
 # Update Gems
+echo 'gem: --no-document' >> ~/.gemrc
 gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2)
-gem install bundler -v $(grep :bundler omnibus_overrides.rb | cut -d'"' -f2) --force --no-document
+gem install bundler -v $(grep :bundler omnibus_overrides.rb | cut -d'"' -f2) --force
 sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER )
+bundle install --jobs=3 --retry=3 --path=vendor/bundle
+
+echo "--- Config information"
+
+echo "!!!! RUBY VERSION !!!!"
 ruby --version
+echo "!!!! BUNDLE LOCATION !!!!"
 which bundle
-bundle install --jobs=3 --retry=3 --path=vendor/bundle
\ No newline at end of file
+echo "!!!! DOCKER VERSION !!!!"
+docker version
+echo "!!!! DOCKER STATUS !!!!"
+sudo service docker status
+
+echo "+++ Running tests"
\ No newline at end of file