Improvements to omnibus buildkite pipelinesnappj/improve-omnibus-buildkite-pipeline

Enable all platforms in omnibus buildkite pipeline.

Port appropriate functionality from ci/verify-chef.* scripts to
the new omnibus/omnibus-test.* scripts.

Signed-off-by: Jeremiah Snapp <jeremiah@chef.io>

3 files changed, 197 insertions, 56 deletions

diff --git a/.expeditor/release.omnibus.yml b/.expeditor/release.omnibus.yml
index 1fc0e4cd6d..edd03f7bf0 100644
--- a/.expeditor/release.omnibus.yml
+++ b/.expeditor/release.omnibus.yml
@@ -7,52 +7,51 @@ fips-platforms:
   - el-*-x86_64
   - windows-*
 builder-to-testers-map:
-  # aix-7.1-powerpc:
-  #   - aix-7.1-powerpc
+  aix-7.1-powerpc:
+    - aix-7.1-powerpc
   debian-8-x86_64:
     - debian-8-x86_64
     - debian-9-x86_64
   el-6-i386:
     - el-6-i386
-  # el-6-s390x:
-  #   - el-6-s390x
+  el-6-s390x:
+    - el-6-s390x
   el-6-x86_64:
     - el-6-x86_64
-  # el-7-aarch64:
-  #   - el-7-aarch64
-  # el-7-ppc64:
-  #   - el-7-ppc64
-  # el-7-ppc64le:
-  #   - el-7-ppc64le
-  # el-7-s390x:
-  #   - el-7-s390x
+  el-7-aarch64:
+    - el-7-aarch64
+  el-7-ppc64:
+    - el-7-ppc64
+  el-7-ppc64le:
+    - el-7-ppc64le
+  el-7-s390x:
+    - el-7-s390x
   el-7-x86_64:
     - el-7-x86_64
-  # freebsd-10-x86_64:
-  #   - freebsd-10-x86_64
-  #   - freebsd-11-x86_64
+  freebsd-10-x86_64:
+    - freebsd-10-x86_64
+    - freebsd-11-x86_64
   mac_os_x-10.12-x86_64:
     - mac_os_x-10.12-x86_64
     - mac_os_x-10.13-x86_64
     - mac_os_x-10.14-x86_64
-  # sles-11-s390x:
-  #   - sles-11-s390x
+  sles-11-s390x:
+    - sles-11-s390x
   sles-11-x86_64:
     - sles-11-x86_64
-  # sles-12-s390x:
-  #   - sles-12-s390x
-  #   - sles-15-s390x
+  sles-12-s390x:
+    - sles-12-s390x
   sles-12-x86_64:
     - sles-12-x86_64
     - sles-15-x86_64
-  # solaris-11-i86pc:
-  #   - solaris-11-i86pc
-  # solaris-11-sparc:
-  #   - solaris-11-sparc
+  solaris-11-i86pc:
+    - solaris-11-i86pc
+  solaris-11-sparc:
+    - solaris-11-sparc
   ubuntu-14.04-i386:
     - ubuntu-14.04-i386
-  # ubuntu-14.04-ppc64le:
-  #   - ubuntu-14.04-ppc64le
+  ubuntu-14.04-ppc64le:
+    - ubuntu-14.04-ppc64le
   ubuntu-14.04-x86_64:
     - ubuntu-14.04-x86_64
   ubuntu-16.04-x86_64:
diff --git a/omnibus/omnibus-test.ps1 b/omnibus/omnibus-test.ps1
index a0a0adadef..8e826cfb77 100644
--- a/omnibus/omnibus-test.ps1
+++ b/omnibus/omnibus-test.ps1
@@ -10,28 +10,72 @@ If ([string]::IsNullOrEmpty($product)) { $product = "chef" }
 $version = "$Env:VERSION"
 If ([string]::IsNullOrEmpty($version)) { $version = "latest" }
 
-Write-Output "--- Downloading $channel $product $version"
-$download_url = C:\opscode\omnibus-toolchain\embedded\bin\mixlib-install.bat download --url --channel "$channel" "$product" --version "$version"
-$package_file = "$Env:Temp\$(Split-Path -Path $download_url -Leaf)"
-Invoke-WebRequest -OutFile "$package_file" -Uri "$download_url"
-
-Write-Output "--- Checking that $package_file has been signed."
-If ((Get-AuthenticodeSignature "$package_file").Status -eq 'Valid') {
-  Write-Output "Verified $package_file has been signed."
-}
-Else {
-  Write-Output "Exiting with an error because $package_file has not been signed. Check your omnibus project config."
-  exit 1
+Write-Output "--- Installing $channel $product $version"
+$package_file = $(C:\opscode\omnibus-toolchain\bin\install-omnibus-product.ps1 -Product "$product" -Channel "$channel" -Version "$version" | Select-Object -Last 1)
+
+Write-Output "--- Verifying omnibus package is signed"
+C:\opscode\omnibus-toolchain\bin\check-omnibus-package-signed.ps1 "$package_file"
+
+Write-Output "--- Running verification for $channel $product $version"
+
+# We don't want to add the embedded bin dir to the main PATH as this
+# could mask issues in our binstub shebangs.
+$embedded_bin_dir = "C:\opscode\$product\embedded\bin"
+
+# Set TEMP and TMP environment variables to a short path because buildkite-agent user's default path is so long it causes tests to fail
+$Env:TEMP = "C:\cheftest"
+$Env:TMP = "C:\cheftest"
+Remove-Item -Recurse -Force $Env:TEMP -ErrorAction SilentlyContinue
+New-Item -ItemType directory -Path $Env:TEMP
+
+ForEach ($b in
+  "chef-client",
+  "knife",
+  "chef-solo",
+  "ohai"
+) {
+  Write-Output "Checking for existence of binfile $b..."
+
+  If (Test-Path -PathType Leaf -Path "C:\opscode\$product\bin\$b") {
+    Write-Output "Found $b!"
+  }
+  Else {
+    Write-Output "Error: Could not find $b"
+    exit 1
+  }
 }
 
-Write-Output "--- Installing $channel $product $version"
-Start-Process "$package_file" /quiet -Wait
+$Env:PATH = "C:\opscode\$product\bin;$Env:PATH"
+
+chef-client --version
+
+# Exercise various packaged tools to validate binstub shebangs
+& $embedded_bin_dir\ruby --version
+& $embedded_bin_dir\gem.bat --version
+& $embedded_bin_dir\bundle.bat --version
+& $embedded_bin_dir\rspec.bat --version
+
+$Env:PATH = "C:\opscode\$product\bin;C:\opscode\$product\embedded\bin;$Env:PATH"
+
+# Test against the vendored chef gem (cd into the output of "gem which chef")
+$chefdir = gem which chef
+$chefdir = Split-Path -Path "$chefdir" -Parent
+$chefdir = Split-Path -Path "$chefdir" -Parent
+Set-Location -Path $chefdir
+
+Get-Location
+
+# ffi-yajl must run in c-extension mode for perf, so force it so we don't accidentally fall back to ffi
+$Env:FORCE_FFI_YAJL = "ext"
 
-Write-Output "--- Testing $channel $product $version"
+# chocolatey functional tests fail so delete the chocolatey binary to avoid triggering them
+Remove-Item -Path C:\ProgramData\chocolatey\bin\choco.exe -ErrorAction SilentlyContinue
 
-$Env:PATH = "C:\opscode\chef\bin;${Env:PATH}"
-$Env:PROJECT_NAME = $product
+# some tests need winrm configured
+winrm quickconfig -quiet
 
-Write-Output "Running verification for $product"
+bundle
+If ($lastexitcode -ne 0) { Exit $lastexitcode }
 
-ci/verify-chef.bat
+bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o test.xml -f documentation spec/functional
+If ($lastexitcode -ne 0) { Exit $lastexitcode }
diff --git a/omnibus/omnibus-test.sh b/omnibus/omnibus-test.sh
index c0c1797be7..b6e740dee6 100644
--- a/omnibus/omnibus-test.sh
+++ b/omnibus/omnibus-test.sh
@@ -5,21 +5,19 @@ channel="${CHANNEL:-unstable}"
 product="${PRODUCT:-chef}"
 version="${VERSION:-latest}"
 
+export INSTALL_DIR="/opt/$product"
+
 echo "--- Installing $channel $product $version"
-package_file="$(install-omnibus-product -c "$channel" -P "$product" -v "$version" | tail -n 1)"
+package_file="$(/opt/omnibus-toolchain/bin/install-omnibus-product -c "$channel" -P "$product" -v "$version" | tail -n 1)"
 
 echo "--- Verifying omnibus package is signed"
-check-omnibus-package-signed "$package_file"
-
-echo "--- Testing $channel $product $version"
+/opt/omnibus-toolchain/bin/check-omnibus-package-signed "$package_file"
 
-export INSTALL_DIR=/opt/chef
-export PATH="/opt/chef/bin:$PATH"
-export PROJECT_NAME=$product
+sudo rm -f "$package_file"
 
-echo "Verifying ownership of package files"
+echo "--- Verifying ownership of package files"
 
-NONROOT_FILES="$(find "$INSTALL_DIR" ! -uid 0 -print)"
+NONROOT_FILES="$(find "$INSTALL_DIR" ! -user 0 -print)"
 if [[ "$NONROOT_FILES" == "" ]]; then
   echo "Packages files are owned by root.  Continuing verification."
 else
@@ -28,6 +26,106 @@ else
   exit 1
 fi
 
-echo "Running verification for $product"
+echo "--- Running verification for $channel $product $version"
+
+# Our tests hammer YUM pretty hard and the EL6 testers get corrupted
+# after some period of time. Rebuilding the RPM database clears
+# up the underlying corruption. We'll do this each test run just to
+# be safe.
+if [[ -f /etc/redhat-release ]]; then
+  major_version="$(sed 's/^.\+ release \([0-9]\+\).*/\1/' /etc/redhat-release)"
+  if [[ "$major_version" -lt "7" ]]; then
+    sudo rm -rf /var/lib/rpm/__db*
+    sudo db_verify /var/lib/rpm/Packages
+    sudo rpm --rebuilddb
+    sudo yum clean all
+  fi
+fi
+
+# Set up a custom tmpdir, and clean it up before and after the tests
+TMPDIR="${TMPDIR:-/tmp}/cheftest"
+export TMPDIR
+sudo rm -rf "$TMPDIR"
+mkdir -p "$TMPDIR"
+
+# Verify that we kill any orphaned test processes. Kill any orphaned rspec processes.
+ps ax | grep -E 'rspec' | grep -v grep | awk '{ print $1 }' | xargs sudo kill -s KILL || true
+
+PATH="/opt/$product/bin:$PATH"
+export PATH
+
+BIN_DIR="/opt/$product/bin"
+export BIN_DIR
+
+# We don't want to add the embedded bin dir to the main PATH as this
+# could mask issues in our binstub shebangs.
+EMBEDDED_BIN_DIR="/opt/$product/embedded/bin"
+export EMBEDDED_BIN_DIR
+
+# If we are on Mac our symlinks are located under /usr/local/bin
+# otherwise they are under /usr/bin
+if [[ -f /usr/bin/sw_vers ]]; then
+  USR_BIN_DIR="/usr/local/bin"
+else
+  USR_BIN_DIR="/usr/bin"
+fi
+export USR_BIN_DIR
+
+# sanity check that we're getting the correct symlinks from the pre-install script
+# solaris doesn't have readlink or test -e. ls -n is different on BSD. proceed with caution.
+if [[ ! -L $USR_BIN_DIR/chef-client ]] || [[ $(ls -l $USR_BIN_DIR/chef-client | awk '{print$NF}') != "$BIN_DIR/chef-client" ]]; then
+  echo "$USR_BIN_DIR/chef-client symlink to $BIN_DIR/chef-client was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+if [[ ! -L $USR_BIN_DIR/knife ]] || [[ $(ls -l $USR_BIN_DIR/knife | awk '{print$NF}') != "$BIN_DIR/knife" ]]; then
+  echo "$USR_BIN_DIR/knife symlink to $BIN_DIR/knife was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+if [[ ! -L $USR_BIN_DIR/chef-solo ]] || [[ $(ls -l $USR_BIN_DIR/chef-solo | awk '{print$NF}') != "$BIN_DIR/chef-solo" ]]; then
+  echo "$USR_BIN_DIR/chef-solo symlink to $BIN_DIR/chef-solo was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+if [[ ! -L $USR_BIN_DIR/ohai ]] || [[ $(ls -l $USR_BIN_DIR/ohai | awk '{print$NF}') != "$BIN_DIR/ohai" ]]; then
+  echo "$USR_BIN_DIR/ohai symlink to $BIN_DIR/ohai was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+chef-client --version
+
+# Exercise various packaged tools to validate binstub shebangs
+"$EMBEDDED_BIN_DIR/ruby" --version
+"$EMBEDDED_BIN_DIR/gem" --version
+"$EMBEDDED_BIN_DIR/bundle" --version
+"$EMBEDDED_BIN_DIR/rspec" --version
+
+# ffi-yajl must run in c-extension mode or we take perf hits, so we force it
+# before running rspec so that we don't wind up testing the ffi mode
+FORCE_FFI_YAJL=ext
+export FORCE_FFI_YAJL
+
+# chef-shell smoke tests require "rb-readline" which requires "infocmp"
+# most platforms provide "infocmp" by default via an "ncurses" package but SLES 11 and 12 provide it via "ncurses-devel" which
+# isn't typically installed. omnibus-toolchain has "infocmp" built-in so we add omnibus-toolchain to the PATH to ensure
+# tests will function properly.
+PATH="/opt/omnibus-toolchain/bin:/usr/local/bin:/opt/omnibus-toolchain/embedded/bin:$PATH"
+
+# add chef's bin paths to PATH to ensure tests function properly
+PATH="/opt/$product/bin:/opt/$product/embedded/bin:$PATH"
+
+gem_list="$(gem which chef)"
+lib_dir="$(dirname "$gem_list")"
+chef_gem="$(dirname "$lib_dir")"
+
+# ensure that PATH doesn't get reset by sudoers
+if [[ -d /etc/sudoers.d ]]; then
+  echo "Defaults:$(id -un) !secure_path, exempt_group += $(id -gn)" | sudo tee "/etc/sudoers.d/$(id -un)-preserve_path"
+elif [[ -d /usr/local/etc/sudoers.d ]]; then
+  echo "Defaults:$(id -un) !secure_path, exempt_group += $(id -gn)" | sudo tee "/usr/local/etc/sudoers.d/$(id -un)-preserve_path"
+fi
 
-sh ci/verify-chef.sh
+cd "$chef_gem"
+sudo -E bundle install
+sudo -E bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o test.xml -f documentation spec/functional