Create omnibus buildkite pipelines

Signed-off-by: Jeremiah Snapp <jeremiah@chef.io>

4 files changed, 286 insertions, 0 deletions

diff --git a/.expeditor/config.yml b/.expeditor/config.yml
index 3fadf6bfaa..fccf8d7b51 100644
--- a/.expeditor/config.yml
+++ b/.expeditor/config.yml
@@ -24,6 +24,11 @@ habitat_packages:
 pipelines:
   - verify:
       public: true
+  - omnibus/release
+  - omnibus/adhoc:
+      definition: .expeditor/release.omnibus.yml
+      env:
+        - ADHOC: true
 
 github:
   # The file where the MAJOR.MINOR.PATCH version is kept. The version in this file
diff --git a/.expeditor/release.omnibus.yml b/.expeditor/release.omnibus.yml
new file mode 100644
index 0000000000..edd03f7bf0
--- /dev/null
+++ b/.expeditor/release.omnibus.yml
@@ -0,0 +1,67 @@
+---
+project-name: chef
+config: omnibus/omnibus.rb
+test-path: omnibus/omnibus-test.sh
+test-path-windows: omnibus/omnibus-test.ps1
+fips-platforms:
+  - el-*-x86_64
+  - windows-*
+builder-to-testers-map:
+  aix-7.1-powerpc:
+    - aix-7.1-powerpc
+  debian-8-x86_64:
+    - debian-8-x86_64
+    - debian-9-x86_64
+  el-6-i386:
+    - el-6-i386
+  el-6-s390x:
+    - el-6-s390x
+  el-6-x86_64:
+    - el-6-x86_64
+  el-7-aarch64:
+    - el-7-aarch64
+  el-7-ppc64:
+    - el-7-ppc64
+  el-7-ppc64le:
+    - el-7-ppc64le
+  el-7-s390x:
+    - el-7-s390x
+  el-7-x86_64:
+    - el-7-x86_64
+  freebsd-10-x86_64:
+    - freebsd-10-x86_64
+    - freebsd-11-x86_64
+  mac_os_x-10.12-x86_64:
+    - mac_os_x-10.12-x86_64
+    - mac_os_x-10.13-x86_64
+    - mac_os_x-10.14-x86_64
+  sles-11-s390x:
+    - sles-11-s390x
+  sles-11-x86_64:
+    - sles-11-x86_64
+  sles-12-s390x:
+    - sles-12-s390x
+  sles-12-x86_64:
+    - sles-12-x86_64
+    - sles-15-x86_64
+  solaris-11-i86pc:
+    - solaris-11-i86pc
+  solaris-11-sparc:
+    - solaris-11-sparc
+  ubuntu-14.04-i386:
+    - ubuntu-14.04-i386
+  ubuntu-14.04-ppc64le:
+    - ubuntu-14.04-ppc64le
+  ubuntu-14.04-x86_64:
+    - ubuntu-14.04-x86_64
+  ubuntu-16.04-x86_64:
+    - ubuntu-16.04-x86_64
+    - ubuntu-18.04-x86_64
+  windows-2012r2-i386:
+    - windows-2012r2-i386
+  windows-2012r2-x86_64:
+    - windows-2008r2-x86_64
+    - windows-2012-x86_64
+    - windows-2012r2-x86_64
+    - windows-2016-x86_64
+    - windows-2019-x86_64
diff --git a/omnibus/omnibus-test.ps1 b/omnibus/omnibus-test.ps1
new file mode 100644
index 0000000000..8e826cfb77
--- /dev/null
+++ b/omnibus/omnibus-test.ps1
@@ -0,0 +1,81 @@
+# Stop script execution when a non-terminating error occurs
+$ErrorActionPreference = "Stop"
+
+$channel = "$Env:CHANNEL"
+If ([string]::IsNullOrEmpty($channel)) { $channel = "unstable" }
+
+$product = "$Env:PRODUCT"
+If ([string]::IsNullOrEmpty($product)) { $product = "chef" }
+
+$version = "$Env:VERSION"
+If ([string]::IsNullOrEmpty($version)) { $version = "latest" }
+
+Write-Output "--- Installing $channel $product $version"
+$package_file = $(C:\opscode\omnibus-toolchain\bin\install-omnibus-product.ps1 -Product "$product" -Channel "$channel" -Version "$version" | Select-Object -Last 1)
+
+Write-Output "--- Verifying omnibus package is signed"
+C:\opscode\omnibus-toolchain\bin\check-omnibus-package-signed.ps1 "$package_file"
+
+Write-Output "--- Running verification for $channel $product $version"
+
+# We don't want to add the embedded bin dir to the main PATH as this
+# could mask issues in our binstub shebangs.
+$embedded_bin_dir = "C:\opscode\$product\embedded\bin"
+
+# Set TEMP and TMP environment variables to a short path because buildkite-agent user's default path is so long it causes tests to fail
+$Env:TEMP = "C:\cheftest"
+$Env:TMP = "C:\cheftest"
+Remove-Item -Recurse -Force $Env:TEMP -ErrorAction SilentlyContinue
+New-Item -ItemType directory -Path $Env:TEMP
+
+ForEach ($b in
+  "chef-client",
+  "knife",
+  "chef-solo",
+  "ohai"
+) {
+  Write-Output "Checking for existence of binfile $b..."
+
+  If (Test-Path -PathType Leaf -Path "C:\opscode\$product\bin\$b") {
+    Write-Output "Found $b!"
+  }
+  Else {
+    Write-Output "Error: Could not find $b"
+    exit 1
+  }
+}
+
+$Env:PATH = "C:\opscode\$product\bin;$Env:PATH"
+
+chef-client --version
+
+# Exercise various packaged tools to validate binstub shebangs
+& $embedded_bin_dir\ruby --version
+& $embedded_bin_dir\gem.bat --version
+& $embedded_bin_dir\bundle.bat --version
+& $embedded_bin_dir\rspec.bat --version
+
+$Env:PATH = "C:\opscode\$product\bin;C:\opscode\$product\embedded\bin;$Env:PATH"
+
+# Test against the vendored chef gem (cd into the output of "gem which chef")
+$chefdir = gem which chef
+$chefdir = Split-Path -Path "$chefdir" -Parent
+$chefdir = Split-Path -Path "$chefdir" -Parent
+Set-Location -Path $chefdir
+
+Get-Location
+
+# ffi-yajl must run in c-extension mode for perf, so force it so we don't accidentally fall back to ffi
+$Env:FORCE_FFI_YAJL = "ext"
+
+# chocolatey functional tests fail so delete the chocolatey binary to avoid triggering them
+Remove-Item -Path C:\ProgramData\chocolatey\bin\choco.exe -ErrorAction SilentlyContinue
+
+# some tests need winrm configured
+winrm quickconfig -quiet
+
+bundle
+If ($lastexitcode -ne 0) { Exit $lastexitcode }
+
+bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o test.xml -f documentation spec/functional
+If ($lastexitcode -ne 0) { Exit $lastexitcode }
diff --git a/omnibus/omnibus-test.sh b/omnibus/omnibus-test.sh
new file mode 100644
index 0000000000..4b5ca82895
--- /dev/null
+++ b/omnibus/omnibus-test.sh
@@ -0,0 +1,133 @@
+#!/bin/bash
+set -ueo pipefail
+
+channel="${CHANNEL:-unstable}"
+product="${PRODUCT:-chef}"
+version="${VERSION:-latest}"
+
+export INSTALL_DIR="/opt/$product"
+
+echo "--- Installing $channel $product $version"
+package_file="$("/opt/$TOOLCHAIN/bin/install-omnibus-product" -c "$channel" -P "$product" -v "$version" | tail -1)"
+
+echo "--- Verifying omnibus package is signed"
+"/opt/$TOOLCHAIN/bin/check-omnibus-package-signed" "$package_file"
+
+sudo rm -f "$package_file"
+
+echo "--- Verifying ownership of package files"
+
+NONROOT_FILES="$(find "$INSTALL_DIR" ! -user 0 -print)"
+if [[ "$NONROOT_FILES" == "" ]]; then
+  echo "Packages files are owned by root.  Continuing verification."
+else
+  echo "Exiting with an error because the following files are not owned by root:"
+  echo "$NONROOT_FILES"
+  exit 1
+fi
+
+echo "--- Running verification for $channel $product $version"
+
+# Our tests hammer YUM pretty hard and the EL6 testers get corrupted
+# after some period of time. Rebuilding the RPM database clears
+# up the underlying corruption. We'll do this each test run just to
+# be safe.
+if [[ -f /etc/redhat-release ]]; then
+  major_version="$(sed 's/^.\+ release \([0-9]\+\).*/\1/' /etc/redhat-release)"
+  if [[ "$major_version" -lt "7" ]]; then
+    sudo rm -rf /var/lib/rpm/__db*
+    sudo db_verify /var/lib/rpm/Packages
+    sudo rpm --rebuilddb
+    sudo yum clean all
+  fi
+fi
+
+# Set up a custom tmpdir, and clean it up before and after the tests
+TMPDIR="${TMPDIR:-/tmp}/cheftest"
+export TMPDIR
+sudo rm -rf "$TMPDIR"
+mkdir -p "$TMPDIR"
+
+# Verify that we kill any orphaned test processes. Kill any orphaned rspec processes.
+ps ax | grep -E 'rspec' | grep -v grep | awk '{ print $1 }' | xargs sudo kill -s KILL || true
+
+PATH="/opt/$product/bin:$PATH"
+export PATH
+
+BIN_DIR="/opt/$product/bin"
+export BIN_DIR
+
+# We don't want to add the embedded bin dir to the main PATH as this
+# could mask issues in our binstub shebangs.
+EMBEDDED_BIN_DIR="/opt/$product/embedded/bin"
+export EMBEDDED_BIN_DIR
+
+# If we are on Mac our symlinks are located under /usr/local/bin
+# otherwise they are under /usr/bin
+if [[ -f /usr/bin/sw_vers ]]; then
+  USR_BIN_DIR="/usr/local/bin"
+else
+  USR_BIN_DIR="/usr/bin"
+fi
+export USR_BIN_DIR
+
+# sanity check that we're getting the correct symlinks from the pre-install script
+# solaris doesn't have readlink or test -e. ls -n is different on BSD. proceed with caution.
+if [[ ! -L $USR_BIN_DIR/chef-client ]] || [[ $(ls -l $USR_BIN_DIR/chef-client | awk '{print$NF}') != "$BIN_DIR/chef-client" ]]; then
+  echo "$USR_BIN_DIR/chef-client symlink to $BIN_DIR/chef-client was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+if [[ ! -L $USR_BIN_DIR/knife ]] || [[ $(ls -l $USR_BIN_DIR/knife | awk '{print$NF}') != "$BIN_DIR/knife" ]]; then
+  echo "$USR_BIN_DIR/knife symlink to $BIN_DIR/knife was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+if [[ ! -L $USR_BIN_DIR/chef-solo ]] || [[ $(ls -l $USR_BIN_DIR/chef-solo | awk '{print$NF}') != "$BIN_DIR/chef-solo" ]]; then
+  echo "$USR_BIN_DIR/chef-solo symlink to $BIN_DIR/chef-solo was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+if [[ ! -L $USR_BIN_DIR/ohai ]] || [[ $(ls -l $USR_BIN_DIR/ohai | awk '{print$NF}') != "$BIN_DIR/ohai" ]]; then
+  echo "$USR_BIN_DIR/ohai symlink to $BIN_DIR/ohai was not correctly created by the pre-install script!"
+  exit 1
+fi
+
+chef-client --version
+
+# Exercise various packaged tools to validate binstub shebangs
+"$EMBEDDED_BIN_DIR/ruby" --version
+"$EMBEDDED_BIN_DIR/gem" --version
+"$EMBEDDED_BIN_DIR/bundle" --version
+"$EMBEDDED_BIN_DIR/rspec" --version
+
+# ffi-yajl must run in c-extension mode or we take perf hits, so we force it
+# before running rspec so that we don't wind up testing the ffi mode
+FORCE_FFI_YAJL=ext
+export FORCE_FFI_YAJL
+
+# chef-shell smoke tests require "rb-readline" which requires "infocmp"
+# most platforms provide "infocmp" by default via an "ncurses" package but SLES 11 and 12 provide it via "ncurses-devel" which
+# isn't typically installed. omnibus-toolchain has "infocmp" built-in so we add omnibus-toolchain to the PATH to ensure
+# tests will function properly.
+PATH="/opt/$TOOLCHAIN/bin:/usr/local/bin:/opt/$TOOLCHAIN/embedded/bin:$PATH"
+
+# add chef's bin paths to PATH to ensure tests function properly
+PATH="/opt/$product/bin:/opt/$product/embedded/bin:$PATH"
+
+gem_list="$(gem which chef)"
+lib_dir="$(dirname "$gem_list")"
+chef_gem="$(dirname "$lib_dir")"
+
+# ensure that PATH doesn't get reset by sudoers
+if [[ -d /etc/sudoers.d ]]; then
+  echo "Defaults:$(id -un) !secure_path, exempt_group += $(id -gn)" | sudo tee "/etc/sudoers.d/$(id -un)-preserve_path"
+  sudo chmod 440 "/etc/sudoers.d/$(id -un)-preserve_path"
+elif [[ -d /usr/local/etc/sudoers.d ]]; then
+  echo "Defaults:$(id -un) !secure_path, exempt_group += $(id -gn)" | sudo tee "/usr/local/etc/sudoers.d/$(id -un)-preserve_path"
+  sudo chmod 440 "/usr/local/etc/sudoers.d/$(id -un)-preserve_path"
+fi
+
+cd "$chef_gem"
+sudo -E bundle install
+sudo -E bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o test.xml -f documentation spec/functional