update verify pipeline and omnibus build/test to use containers (#13489)

* update verify pipeline and omnibus build/test to use containers
* add execution permission on scripts
* add fips back in for opensuse but skip for windows
* jesseprieur/BS-159 - Add RPM Signing Key for RHEL/SLES/CentOS
* evanahlberg/BS-152 - Add MSI Signing to Windows Builds
* fix label on windows platforms
* fix role and add check for organization for aws credentials
* chmod omnibus test script and fix windows 2019 queue for omnibus test
* fix windows queue
* Remove dependencies that are part of chef-foundation.
* Disable s3 caching.
* Rename chef-gem as it conflicts with another omnibus-software.
* Speeding up debugging; Removing more-ruby-cleanup gem from omnibus chef package
* Adding more-ruby-cleanup back in
* Removing more-ruby-cleanup; Removing debugging steps
* Remove ruby dependency from more-ruby-cleanup.
* add retry and timeouts to all steps
* move omnibus test and build to own file and create ad hoc pipeline
* Adding in changes to use heredoc in verify pipeline
* Adding in retries/timeouts; Fixing missing agent
* allow for filtering of the omnibus build and test platforms
* only block on chef-oss org
* add canary adhoc pipeline and add back trigger for omnibus release
* Make omnibus s3 caching configurable via env.

Signed-off-by: Gregory Schofield <grschofi@progress.com>

Signed-off-by: Evan Ahlberg <evanahlberg@gmail.com>

Signed-off-by: Jesse Prieur <jesse.prieur@gmail.com>

23 files changed, 887 insertions, 401 deletions

diff --git a/.buildkite-platform.json b/.buildkite-platform.json
new file mode 100644
index 0000000000..4aa454c9f3
--- /dev/null
+++ b/.buildkite-platform.json
@@ -0,0 +1,4 @@
+{
+    "chef_foundation": "0.1.24",
+    "omnibus_toolchain": "3.0.0"
+}
\ No newline at end of file
diff --git a/.buildkite/build-test-omnibus.sh b/.buildkite/build-test-omnibus.sh
new file mode 100755
index 0000000000..79b18165e8
--- /dev/null
+++ b/.buildkite/build-test-omnibus.sh
@@ -0,0 +1,112 @@
+if [[ $BUILDKITE_ORGANIZATION_SLUG == "chef-oss" ]]; then
+  echo "- block: Build & Test Omnibus Packages"
+  echo "  prompt: Continue to run omnibus package build and tests for applicable platforms?"
+fi
+
+FILTER="${OMNIBUS_FILTER:=*}"
+
+platforms=("amazon-2:centos-7" "centos-6:centos-6" "centos-7:centos-7" "centos-8:centos-8" "rhel-9:rhel-9" "debian-9:debian-9" "debian-10:debian-9" "debian-11:debian-9" "ubuntu-1604:ubuntu-1604" "ubuntu-1804:ubuntu-1604" "ubuntu-2004:ubuntu-1604" "ubuntu-2204:ubuntu-1604" "sles-15:sles-15" "windows-2019:windows-2019")
+
+omnibus_build_platforms=()
+omnibus_test_platforms=()
+
+# build build array and test array based on filter
+for platform in ${platforms[@]}; do
+    case ${platform%:*} in
+        $FILTER)
+            omnibus_build_platforms[${#omnibus_build_platforms[@]}]=${platform#*:}
+            omnibus_test_platforms[${#omnibus_test_platforms[@]}]=$platform
+            ;;
+    esac
+done
+
+# remove duplicates from build array
+omnibus_build_platforms=($(printf "%s\n" "${omnibus_build_platforms[@]}" | sort -u | tr '\n' ' '))
+
+for platform in ${omnibus_build_platforms[@]}; do
+  if [[ $platform != *"windows"* ]]; then
+    echo "- label: \":hammer_and_wrench::docker: $platform\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  key: build-$platform"
+    echo "  agents:"
+    echo "    queue: default-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      privileged: true"
+    echo "      propagate-environment: true"
+    echo "      environment:"
+    echo "        - RPM_SIGNING_KEY"
+    echo "        - CHEF_FOUNDATION_VERSION"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/omnibus_chef_build.sh"
+    echo "  timeout_in_minutes: 60"
+  else 
+    echo "- label: \":hammer_and_wrench::windows: $platform\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  key: build-$platform"
+    echo "  agents:"
+    echo "    queue: default-$platform-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      shell:"
+    echo "      - powershell"
+    echo "      - \"-Command\""
+    echo "      propagate-environment: true"
+    echo "      environment:"
+    echo "        - CHEF_FOUNDATION_VERSION"
+    echo "        - BUILDKITE_AGENT_ACCESS_TOKEN"
+    echo "        - AWS_ACCESS_KEY_ID"
+    echo "        - AWS_SECRET_ACCESS_KEY"
+    echo "        - AWS_SESSION_TOKEN"
+    echo "      volumes:"
+    echo '        - "c:\\buildkite-agent:c:\\buildkite-agent"'
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/omnibus_chef_build.ps1"
+    echo "  timeout_in_minutes: 60"
+  fi
+done
+
+echo "- wait: ~"
+
+for platform in ${omnibus_test_platforms[@]}; do
+  if [[ $platform != *"windows"* ]]; then
+    echo "- env:"
+    echo "    OMNIBUS_BUILDER_KEY: build-${platform#*:}"
+    echo "  label: \":mag::docker: ${platform%:*}\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  agents:"
+    echo "    queue: default-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-${platform%:*}:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      privileged: true"
+    echo "      propagate-environment: true"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.sh"
+    echo "    - omnibus/omnibus-test.sh"
+    echo "  timeout_in_minutes: 60"
+  else
+    echo "- env:"
+    echo "    OMNIBUS_BUILDER_KEY: build-windows-2019"
+    echo "  key: test-windows-2019"
+    echo '  label: ":mag::windows: windows-2019"'
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  agents:"
+    echo "    queue: default-windows-2019-privileged"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.ps1"
+    echo "    - ./omnibus/omnibus-test.ps1"
+    echo "  timeout_in_minutes: 60"
+  fi
+done
+
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 8789433886..c942fda34a 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -7,6 +7,27 @@ set -eu
 
 docker ps || true
 
+# Get chef foundation version from the json file
+CHEF_FOUNDATION_VERSION=$(cat .buildkite-platform.json | jq -r '.chef_foundation')
+export CHEF_FOUNDATION_VERSION
+echo $CHEF_FOUNDATION_VERSION
+
+OMNIBUS_TOOLCHAIN_VERSION=$(cat .buildkite-platform.json | jq -r '.omnibus_toolchain')
+export OMNIBUS_TOOLCHAIN_VERSION
+echo $OMNIBUS_TOOLCHAIN_VERSION
+
+if [ $BUILDKITE_STEP_KEY == "build-windows-2019" ] && [ $BUILDKITE_ORGANIZATION_SLUG == "chef" ]
+then
+  TOKEN=$(curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+  RESPONSE=$(curl -sH "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/default-windows-2019-privileged-chef-Role)
+  AWS_ACCESS_KEY_ID=$(echo $RESPONSE | jq -r '.AccessKeyId')
+  export AWS_ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY=$(echo $RESPONSE | jq -r '.SecretAccessKey')
+  export AWS_SECRET_ACCESS_KEY
+  AWS_SESSION_TOKEN=$(echo $RESPONSE | jq -r '.Token')
+  export AWS_SESSION_TOKEN
+fi
+
 # We've now seen cases where origin/main on the build hosts can get
 # out of date. This causes us to build components unnecessarily.
 # Fetching it here hopefully will prevent this situation.
@@ -35,3 +56,10 @@ if [[ "$BUILDKITE_BRANCH" != "main" ]]; then
       "Couldn't rebase onto main ([${main}](${github}${main})), building PR HEAD ([${pr_head}](${github}${pr_head}))."
   fi
 fi
+
+# Only execute if on RHEL/CentOS/SLES
+if [[ "$BUILDKITE_LABEL" =~ rhel|sles|centos ]] && [[ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]]; then
+  export VAULT_ADDR="https://vault.ps.chef.co"
+  export VAULT_TOKEN="$(vault login -method=aws -path=aws/private-cd -token-only header_value=vault.ps.chef.co role=ci)"
+  export RPM_SIGNING_KEY="$(vault kv get -field packages_at_chef_io account/static/packages/signing_certs)"
+fi
\ No newline at end of file
diff --git a/.buildkite/verify.adhoc.pipeline.sh b/.buildkite/verify.adhoc.pipeline.sh
new file mode 100755
index 0000000000..51ad3eed4a
--- /dev/null
+++ b/.buildkite/verify.adhoc.pipeline.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# exit immediately on failure, or if an undefined variable is used
+set -eu
+
+echo "---"
+echo "steps:"
+echo ""
+
+# include build and test omnibus pipeline
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+source "$DIR/build-test-omnibus.sh"
\ No newline at end of file
diff --git a/.buildkite/verify.pipeline.sh b/.buildkite/verify.pipeline.sh
new file mode 100755
index 0000000000..2f47e0da57
--- /dev/null
+++ b/.buildkite/verify.pipeline.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+
+# exit immediately on failure, or if an undefined variable is used
+set -eu
+
+echo "---"
+echo "steps:"
+echo ""
+
+test_platforms=("centos-6" "centos-7" "centos-8" "rhel-9" "debian-9" "ubuntu-1604" "sles-15")
+
+for platform in ${test_platforms[@]}; do
+  echo "- label: \"{{matrix}} $platform :ruby:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  echo "    queue: default-privileged"
+  echo "  matrix:"
+  echo "    - \"Unit\""
+  echo "    - \"Integration\""
+  echo "    - \"Functional\""
+  echo "  plugins:"
+  echo "  - docker#v3.5.0:"
+  echo "      image: chefes/omnibus-toolchain-${platform#*:}:$OMNIBUS_TOOLCHAIN_VERSION"
+  echo "      privileged: true"
+  echo "      environment:"
+  echo "        - CHEF_FOUNDATION_VERSION"
+  echo "      propagate-environment: true"
+  echo "  commands:"
+  echo "    - .expeditor/scripts/prep_and_run_tests.sh {{matrix}}"
+  echo "  timeout_in_minutes: 60"
+done
+
+win_test_platforms=("windows-2019:windows-2019")
+
+for platform in ${win_test_platforms[@]}; do
+  echo "- label: \"{{matrix}} ${platform#*:} :windows:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  echo "    queue: default-${platform%:*}-privileged"
+  echo "  matrix:"
+  echo "    - \"Unit\""
+  echo "    - \"Integration\""
+  echo "  plugins:"
+  echo "  - docker#v3.5.0:"
+  echo "      image: chefes/omnibus-toolchain-${platform#*:}:$OMNIBUS_TOOLCHAIN_VERSION"
+  echo "      shell:"
+  echo "      - powershell"
+  echo "      - \"-Command\""
+  echo "      environment:"
+  echo "        - CHEF_FOUNDATION_VERSION"
+  echo "      propagate-environment: true"
+  echo "  commands:"
+  echo "    - .\.expeditor\scripts\prep_and_run_tests.ps1 {{matrix}}"
+  echo "  timeout_in_minutes: 60"
+
+done
+
+for platform in ${win_test_platforms[@]}; do
+  echo "- label: \"Functional ${platform#*:} :windows:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  commands:"
+  echo "    - .\.expeditor\scripts\prep_and_run_tests.ps1 Functional"
+  echo "  agents:"
+  echo "    queue: single-use-windows-2019-privileged"
+  echo "  env:"
+  echo "  - CHEF_FOUNDATION_VERSION"
+  echo "    - .\.expeditor\scripts\prep_and_run_tests.ps1 {{matrix}}"
+  echo "  timeout_in_minutes: 60"
+done
+
+external_gems=("chef-zero" "cheffish" "chefspec" "knife-windows" "berkshelf")
+
+for gem in ${external_gems[@]}; do
+  echo "- label: \"$gem gem :ruby:\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  echo "    queue: default"
+  echo "  plugins:"
+  echo "  - docker#v3.5.0:"
+  echo "      image: chefes/omnibus-toolchain-ubuntu-1804:$OMNIBUS_TOOLCHAIN_VERSION"
+  echo "      environment:"
+  echo "        - CHEF_FOUNDATION_VERSION"
+  if [ $gem == "chef-zero" ] 
+  then
+    echo "        - PEDANT_OPTS=--skip-oc_id"
+    echo "        - CHEF_FS=true"
+  fi
+  echo "      propagate-environment: true"
+  echo "  - chef/cache#v1.5.0:"
+  echo "      s3_bucket: core-buildkite-cache-chef-oss-prod"
+  echo "      cached_folders:"
+  echo "      - vendor"
+  echo "  timeout_in_minutes: 60"
+  echo "  commands:"
+  echo "    - .expeditor/scripts/bk_container_prep.sh"
+  if [ $gem == "berkshelf" ]
+  then
+    echo "    - export PATH=\"/opt/chef/bin:/usr/local/sbin:/usr/sbin:/sbin:${PATH}\""
+    echo "    - apt-get update -y"
+    # cspell:disable-next-line
+    echo "    - apt-get install -y graphviz"
+    echo "    - bundle config set --local without omnibus_package"
+  else
+    echo "    - export PATH=\"/opt/chef/bin:${PATH}\""
+    echo "    - bundle config set --local without omnibus_package"
+    echo "    - bundle config set --local path 'vendor/bundle'"
+  fi
+  echo "    - bundle install --jobs=3 --retry=3"
+  case $gem in 
+    "chef-zero")
+      echo "    - bundle exec tasks/bin/run_external_test chef/chef-zero main rake pedant"
+      ;;
+    "cheffish")
+      echo "    - bundle exec tasks/bin/run_external_test chef/cheffish main rake spec"
+      ;;
+    "chefspec")
+      echo "    - bundle exec tasks/bin/run_external_test chefspec/chefspec main rake"
+      ;;
+    "knife-windows")
+      echo "    - bundle exec tasks/bin/run_external_test chef/knife-windows main rake spec"
+      ;;
+    "berkshelf")
+      echo "    - bundle exec tasks/bin/run_external_test chef/berkshelf main rake"
+      ;;
+    *)
+      echo -e "\n Gem $gem is not valid\n" >&2
+      exit 1
+      ;;
+  esac
+done
+
+habitat_plans=("linux" "linux-kernel2" "windows")
+
+for plan in ${habitat_plans[@]}; do
+  echo "- label: \":habicat: $plan plan\""
+  echo "  retry:"
+  echo "    automatic:"
+  echo "      limit: 1"
+  echo "  agents:"
+  if [ $plan == "windows" ]
+  then
+    echo "    queue: single-use-windows-2019-privileged"
+  else
+    echo "    queue: single-use-privileged"
+  fi
+  echo "  plugins:"
+  echo "  - chef/cache#v1.5.0:"
+  echo "      s3_bucket: core-buildkite-cache-chef-oss-prod"
+  echo "      cached_folders:"
+  echo "      - vendor"
+  echo "  timeout_in_minutes: 60"
+  echo "  commands:"
+  if [ $plan == "windows" ]
+  then
+    echo "    - ./.expeditor/scripts/verify-plan.ps1"
+  else
+    echo "    - sudo ./.expeditor/scripts/install-hab.sh 'x86_64-$plan'"
+    echo "    - sudo ./.expeditor/scripts/verify-plan.sh"
+  fi
+done
+
+# include build and test omnibus pipeline
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+source "$DIR/build-test-omnibus.sh"
\ No newline at end of file
diff --git a/.expeditor/config.yml b/.expeditor/config.yml
index 6d0f4c7721..82347262fb 100644
--- a/.expeditor/config.yml
+++ b/.expeditor/config.yml
@@ -30,6 +30,24 @@ pipelines:
       public: true
       env:
         - IGNORE_ARTIFACTORY_RUBY_PROXY: true # Artifactory is throwing 500's when downloading some gems.
+  - verify/release:
+      definition: .expeditor/verify.pipeline.yml
+      env:
+        - IGNORE_CACHE: true # caching causes constant build failures
+        - IGNORE_ARTIFACTORY_RUBY_PROXY: true
+  - verify/adhoc:
+      definition: .expeditor/verify.adhoc.pipeline.yml
+      env:
+        - ADHOC: true
+        - IGNORE_CACHE: true # caching causes constant build failures
+        - IGNORE_ARTIFACTORY_RUBY_PROXY: true # Artifactory is throwing 500's when downloading some gems.
+  - verify/adhoc-canary:
+      canary: true
+      definition: .expeditor/verify.adhoc.pipeline.yml
+      env:
+        - ADHOC: true
+        - IGNORE_CACHE: true # caching causes constant build failures
+        - IGNORE_ARTIFACTORY_RUBY_PROXY: true # Artifactory is throwing 500's when downloading some gems.
   - docker/build:
       definition: .expeditor/docker-build.pipeline.yml
       trigger: default
@@ -130,6 +148,11 @@ subscriptions:
             - "Expeditor: Skip Omnibus"
             - "Expeditor: Skip All"
           only_if: built_in:bump_version
+      - trigger_pipeline:verify/release:
+          ignore_labels:
+            - "Expeditor: Skip Omnibus"
+            - "Expeditor: Skip All"
+          only_if: built_in:bump_version
 
   # the habitat chain
   - workload: buildkite_hab_build_group_published:{{agent_id}}:*
diff --git a/.expeditor/scripts/bk_container_prep.sh b/.expeditor/scripts/bk_container_prep.sh
index e065f20579..a54f808df6 100755
--- a/.expeditor/scripts/bk_container_prep.sh
+++ b/.expeditor/scripts/bk_container_prep.sh
@@ -1,10 +1,14 @@
 # This script gets a container ready to run our various tests in BuildKite
 
-echo "--- Container Config..."
+# source /etc/os-release
+# echo $PRETTY_NAME
 
-source /etc/os-release
-echo $PRETTY_NAME
+# Install Chef Foundation
+echo "--- Installing Chef Foundation"
+curl -fsSL https://omnitruck.chef.io/chef/install.sh | bash -s -- -c "current" -P "chef-foundation" -v "$CHEF_FOUNDATION_VERSION"
+export PATH="/opt/chef/bin:${PATH}"
 
+echo "--- Container Config..."
 echo "ruby version:"
 ruby -v
 echo "bundler version:"
diff --git a/.expeditor/scripts/download_built_omnibus_pkgs.ps1 b/.expeditor/scripts/download_built_omnibus_pkgs.ps1
new file mode 100644
index 0000000000..7f337857ea
--- /dev/null
+++ b/.expeditor/scripts/download_built_omnibus_pkgs.ps1
@@ -0,0 +1,11 @@
+$ErrorActionPreference = "Stop"
+
+Write-Host "--- Installing package from BuildKite"
+buildkite-agent artifact download "pkg\*.msi" . --step "${Env:OMNIBUS_BUILDER_KEY}"
+$package_file = (Get-ChildItem pkg -Filter "*.msi").FullName 
+
+Write-Output "--- Installing $package_file"
+Start-Process "$package_file" /quiet -Wait
+
+Write-Output "--- Deleting $package_file"
+Remove-Item -Force "$package_file" -ErrorAction SilentlyContinue
\ No newline at end of file
diff --git a/.expeditor/scripts/download_built_omnibus_pkgs.sh b/.expeditor/scripts/download_built_omnibus_pkgs.sh
new file mode 100755
index 0000000000..d8d7311ceb
--- /dev/null
+++ b/.expeditor/scripts/download_built_omnibus_pkgs.sh
@@ -0,0 +1,59 @@
+#! /bin/bash
+set -eu -o pipefail
+
+echo "--- Installing package from BuildKite"
+
+if [[ $OSTYPE == "msys" ]]; then
+  buildkite-agent artifact download "pkg\*.msi" . --step "$OMNIBUS_BUILDER_KEY"
+  package_file=$(find pkg/*)
+else
+  extensions=( deb rpm amd64.sh )
+  for ext in "${extensions[@]}"
+  do
+    buildkite-agent artifact download "pkg/*.${ext}" . --step "$OMNIBUS_BUILDER_KEY" || true
+  done
+  package_file=$(find pkg/*)
+fi
+
+if [[ -z $package_file ]]; then
+  buildkite-agent annotate "Failed to download packages from the $OMNIBUS_BUILDER_KEY builder." --style "warning" --context "ctx-warn" || true
+  exit 1
+fi
+
+# if [[ -v $OMNIBUS_RPM_SIGNING_PASSPHRASE ]]; then
+#   case "$package_file" in
+#     *.rpm)
+#       echo "--- Checking that $package_file has been signed."
+#       if [[ $(rpm -qpi "$package_file" 2>&1 | grep -c "Signature.*Key ID") -eq 1 ]]; then
+#         echo "Verified $package_file has been signed."
+#       else
+#         echo "Exiting with an error because $package_file has not been signed. Check your omnibus project config."
+#         exit 1
+#       fi
+#       ;;
+#   esac
+# fi
+
+echo "--- Installing ${package_file}"
+FILE_TYPE="${package_file##*.}"
+case "$FILE_TYPE" in
+  "rpm")
+    if [[ "${IGNORE_INSTALL_DEPENDENCIES:-false}" == true ]]; then
+      IGNORE_DEPENDS_OPTION="--nodeps"
+    fi
+    sudo rpm -Uvh ${IGNORE_DEPENDS_OPTION:-} --oldpackage --replacepkgs "$package_file"
+    ;;
+  "deb")
+    if [[ "${IGNORE_INSTALL_DEPENDENCIES:-false}" == true ]]; then
+      IGNORE_DEPENDS_OPTION="--force-depends"
+    fi
+    sudo dpkg ${IGNORE_DEPENDS_OPTION:-} -i "$package_file"
+    ;;
+  "sh" )
+    sudo sh "$package_file"
+    ;;
+  *)
+    echo "Unknown filetype: $FILE_TYPE"
+    exit 1
+    ;;
+esac
\ No newline at end of file
diff --git a/.expeditor/scripts/omnibus_chef_build.ps1 b/.expeditor/scripts/omnibus_chef_build.ps1
new file mode 100644
index 0000000000..c3ac569ff4
--- /dev/null
+++ b/.expeditor/scripts/omnibus_chef_build.ps1
@@ -0,0 +1,66 @@
+$ScriptDir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent
+
+if ($env:BUILDKITE_ORGANIZATION_SLUG -eq "chef-oss" )
+{
+  Write-Output "--- Generating self-signed Windows package signing certificate"
+  $thumb = (New-SelfSignedCertificate -Type Custom -Subject "CN=Chef Software, O=Progress, C=US" -KeyUsage DigitalSignature -FriendlyName "Chef Software Inc." -CertStoreLocation "Cert:\LocalMachine\My" -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.3", "2.5.29.19={text}")).Thumbprint 
+}
+else
+{
+  Write-Output "--- Installing Windows package signing certificate"
+  $windows_certificate_json = "windows-package-signing-certificate.json"
+  $windows_certificate_pfx = "windows-package-signing-certificate.pfx"
+  
+  aws ssm get-parameter --name "windows-package-signing-cert" --with-decryption --region "us-west-1" --query Parameter.Value --output text | Set-Content -Path $windows_certificate_json
+  If ($lastexitcode -ne 0) { Throw $lastexitcode }
+  
+  $cert_passphrase = Get-Content $windows_certificate_json | ConvertFrom-Json | Select-Object -ExpandProperty cert_passphrase | ConvertTo-SecureString -asplaintext -force
+  Get-Content $windows_certificate_json | ConvertFrom-Json | Select-Object -ExpandProperty cert_content_base64 | Set-Content -Path $windows_certificate_pfx
+  Remove-Item -Force $windows_certificate_json
+  Import-PfxCertificate $windows_certificate_pfx -CertStoreLocation Cert:\LocalMachine\My -Password $cert_passphrase
+  Remove-Item -Force $windows_certificate_pfx
+  $thumb = "13B510D1CF1B3467856A064F1BEA12D0884D2528"
+}
+
+Write-Output "THUMB=$thumb"
+
+$env:ARTIFACTORY_BASE_PATH="com/getchef"
+$env:ARTIFACTORY_ENDPOINT="https://artifactory-internal.ps.chef.co/artifactory"
+$env:ARTIFACTORY_USERNAME="buildkite"
+
+Write-Output "--- Install Chef Foundation"
+. { Invoke-WebRequest -useb https://omnitruck.chef.io/chef/install.ps1 } | Invoke-Expression; install -channel "current" -project "chef-foundation" -v $CHEF_FOUNDATION_VERSION
+
+$env:OMNIBUS_SIGNING_IDENTITY="${thumb}"
+$env:HOMEDRIVE = "C:"
+$env:HOMEPATH = "\buildkite-agent"
+$env:OMNIBUS_TOOLCHAIN_INSTALL_DIR = "C:\opscode\omnibus-toolchain"
+$env:SSL_CERT_FILE = "${env:OMNIBUS_TOOLCHAIN_INSTALL_DIR}\embedded\ssl\certs\cacert.pem"
+$env:MSYS2_INSTALL_DIR = "C:\msys64"
+$env:BASH_ENV = "${env:MSYS2_INSTALL_DIR}\etc\bash.bashrc"
+$env:OMNIBUS_WINDOWS_ARCH = "x64"
+$env:MSYSTEM = "MINGW64"
+$omnibus_toolchain_msystem = & "${env:OMNIBUS_TOOLCHAIN_INSTALL_DIR}\embedded\bin\ruby" -e "puts RUBY_PLATFORM"
+If ($omnibus_toolchain_msystem -eq "x64-mingw-ucrt") {
+  $env:MSYSTEM = "UCRT64"
+}
+$original_path = $env:PATH
+$env:PATH = "${env:MSYS2_INSTALL_DIR}\$env:MSYSTEM\bin;${env:MSYS2_INSTALL_DIR}\usr\bin;${env:OMNIBUS_TOOLCHAIN_INSTALL_DIR}\embedded\bin;C:\wix;C:\Program Files (x86)\Windows Kits\8.1\bin\x64;${original_path}"
+Write-Output "env:PATH = $env:PATH"
+
+Write-Output "--- Running bundle install for Omnibus"
+Set-Location "$($ScriptDir)/../../omnibus"
+bundle config set --local without development
+bundle install
+
+Write-Output "--- Building Chef"
+bundle exec omnibus build chef -l internal --override append_timestamp:false
+
+Write-Output "--- Uploading package to BuildKite"
+C:\buildkite-agent\bin\buildkite-agent.exe artifact upload "pkg/*.msi*"
+
+# if ($env:BUILDKITE_ORGANIZATION_SLUG -ne "chef-oss" )
+# {
+#   Write-Output "--- Publishing package to Artifactory"
+#   bundle exec ruby "${SCRIPT_DIR}/omnibus_chef_publish.rb"
+# }
diff --git a/.expeditor/scripts/omnibus_chef_build.sh b/.expeditor/scripts/omnibus_chef_build.sh
new file mode 100755
index 0000000000..f2ede50d08
--- /dev/null
+++ b/.expeditor/scripts/omnibus_chef_build.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+set -ueo pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+export ARTIFACTORY_BASE_PATH="com/getchef"
+export ARTIFACTORY_ENDPOINT="https://artifactory-internal.ps.chef.co/artifactory"
+export ARTIFACTORY_USERNAME="buildkite"
+
+export PROJECT_NAME="chef"
+export PATH="/opt/omnibus-toolchain/bin:${PATH}"
+export OMNIBUS_FIPS_MODE="true"
+export OMNIBUS_PIPELINE_DEFINITION_PATH="${SCRIPT_DIR}/../release.omnibus.yaml"
+
+echo "--- Installing Chef Foundation"
+curl -fsSL https://omnitruck.chef.io/chef/install.sh | bash -s -- -c "current" -P "chef-foundation" -v "$CHEF_FOUNDATION_VERSION"
+
+if [[ -f "/opt/omnibus-toolchain/embedded/ssl/certs/cacert.pem" ]]; then
+  export SSL_CERT_FILE="/opt/omnibus-toolchain/embedded/ssl/certs/cacert.pem"
+fi
+
+if [[ "$BUILDKITE_LABEL" =~ rhel|sles|centos ]] && [[ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]]; then
+  export OMNIBUS_RPM_SIGNING_PASSPHRASE=''
+
+  echo "$RPM_SIGNING_KEY" | gpg --import
+
+  cat <<-EOF > ~/.rpmmacros
+    %_signature gpg
+    %_gpg_name  Opscode Packages
+EOF
+fi
+
+echo "--- Running bundle install for Omnibus"
+cd "${SCRIPT_DIR}/../../omnibus"
+bundle config set --local without development
+bundle install
+
+echo "--- Building Chef"
+bundle exec omnibus build chef -l internal --override append_timestamp:false
+
+echo "--- Uploading package to BuildKite"
+extensions=( bff deb dmg msi p5p rpm solaris amd64.sh i386.sh )
+for ext in "${extensions[@]}"
+do
+  buildkite-agent artifact upload "pkg/*.${ext}*"
+done
+
+# echo "--- Publishing package to Artifactory"
+# bundle exec ruby "${SCRIPT_DIR}/omnibus_chef_publish.rb"
\ No newline at end of file
diff --git a/.expeditor/scripts/omnibus_chef_publish.rb b/.expeditor/scripts/omnibus_chef_publish.rb
new file mode 100644
index 0000000000..a818c67799
--- /dev/null
+++ b/.expeditor/scripts/omnibus_chef_publish.rb
@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+
+require 'artifactory'
+require 'fileutils'
+require 'json'
+require 'omnibus'
+require 'tempfile'
+require 'rubygems/commands/push_command'
+require 'yaml'
+
+OMNIBUS_PACKAGE_PATTERN = '**/{pkg,notarized}/*.{bff,deb,dmg,msi,p5p,rpm,solaris,amd64.sh,i386.sh}'.freeze
+
+def self.env_or_empty(key)
+  ENV[key] || ''
+end
+
+def self.env_or_raise(key)
+  ENV[key] || raise("Required ENV variable `#{key}` is unset!")
+end
+
+project_name                     = env_or_raise('PROJECT_NAME')
+omnibus_pipeline_definition_path = env_or_raise('OMNIBUS_PIPELINE_DEFINITION_PATH')
+artifactory_endpoint             = env_or_raise('ARTIFACTORY_ENDPOINT')
+artifactory_base_path            = env_or_raise('ARTIFACTORY_BASE_PATH')
+artifactory_username             = env_or_raise('ARTIFACTORY_USERNAME')
+artifactory_password             = env_or_raise('ARTIFACTORY_PASSWORD')
+
+package_glob_pattern = "./#{OMNIBUS_PACKAGE_PATTERN}"
+
+puts "Publishing with glob pattern of #{package_glob_pattern}"
+puts ''
+
+if File.exist?(omnibus_pipeline_definition_path)
+  omnibus_pipeline_definition = YAML.safe_load(File.read(omnibus_pipeline_definition_path))
+  skip_artifactory_platforms = omnibus_pipeline_definition["skip-artifactory-platforms"] || []
+  builder_to_testers_map = omnibus_pipeline_definition['builder-to-testers-map']
+
+  skip_artifactory_platforms.each do |skip_platform|
+    builder_to_testers_map.each { |builder, tester| tester.reject! { |tester| File.fnmatch(skip_platform, tester) } }.reject! { |builder, tester| tester.empty? }
+  end
+end
+
+Omnibus::Config.artifactory_endpoint(artifactory_endpoint)
+Omnibus::Config.artifactory_base_path(artifactory_base_path)
+Omnibus::Config.artifactory_username(artifactory_username)
+Omnibus::Config.artifactory_password(artifactory_password)
+publisher = Omnibus::ArtifactoryPublisher.new(
+  package_glob_pattern,
+  repository: 'omnibus-unstable-local',
+  platform_mappings: builder_to_testers_map,
+  build_record: false
+)
+
+if publisher.packages.empty?
+  puts "Could not locate any #{project_name} artifacts to publish."
+  return
+else
+  publisher.publish do |package|
+    puts "Published '#{package.name}' for #{package.metadata[:platform]}-#{package.metadata[:platform_version]}-#{package.metadata[:arch]}"
+  end
+
+  puts <<-EOH
+
+DONE! \\m/
+
+  EOH
+end
+
+# This publishes the chef gem to artifactory
+if (project_name == "chef") && (ENV['ADHOC'] != 'true')
+  GEM_PACKAGE_PATTERN = '**/[^/]*\.gem'.freeze
+  gem_base_name = project_name
+  project_source = "#{Omnibus::Config.base_dir}/**/src/#{gem_base_name}"
+
+  # This will exclude any gems in a /spec/ directory
+  gems_found = Dir.glob("#{project_source}/#{GEM_PACKAGE_PATTERN}") - Dir.glob("#{project_source}/**/spec/#{GEM_PACKAGE_PATTERN}")
+
+  # Sometimes there are multiple copies of a gem on disk -- only upload one copy.
+  gems_to_publish = gems_found.uniq { |gem| File.basename(gem) }
+
+  puts "Publishing Gems from #{project_source}"
+  puts ''
+
+  gems_to_publish.each do |gem_path|
+    puts 'Publishing gem ' + gem_path
+    artifactory_endpoint = "#{Omnibus::Config.artifactory_endpoint}/api/gems/omnibus-gems-local"
+    # This mimics the behavior of the gem command line, and is a public api:
+    # http://docs.seattlerb.org/rubygems/Gem/Command.html
+    gem_pusher = Gem::Commands::PushCommand.new
+    gem_pusher.handle_options [gem_path, '--host', artifactory_endpoint, '--key', 'artifactory_api_key', '--verbose']
+    gem_pusher.execute
+  end
+end
diff --git a/.expeditor/scripts/prep_and_run_tests.ps1 b/.expeditor/scripts/prep_and_run_tests.ps1
new file mode 100644
index 0000000000..76e475cd8a
--- /dev/null
+++ b/.expeditor/scripts/prep_and_run_tests.ps1
@@ -0,0 +1,27 @@
+param(
+    # The test type ot be run (unit, integration or functional)
+    [Parameter(Position=0)][String]$TestType
+)
+
+. { Invoke-WebRequest -useb https://omnitruck.chef.io/chef/install.ps1 } | Invoke-Expression; install -channel "current" -project "chef-foundation" -v $CHEF_FOUNDATION_VERSION
+$env:Path = 'C:\Program Files\Git\mingw64\bin;C:\Program Files\Git\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Windows\System32\OpenSSH\;C:\ProgramData\chocolatey\bin;C:\Program Files (x86)\Windows Kits\8.1\Windows Performance Toolkit\;C:\Program Files\Git\cmd;C:\Users\ContainerAdministrator\AppData\Local\Microsoft\WindowsApps;C:\opscode\chef\bin;C:\opscode\chef\embedded\bin'
+
+if ($TestType -eq 'Functional') {
+    winrm quickconfig -q
+}
+
+Write-Output "--- Running Chef bundle install"
+bundle install --jobs=3 --retry=3 
+
+switch ($TestType) {
+    "Unit"          {[string[]]$RakeTest = 'spec:unit','component_specs'; break}
+    "Integration"   {[string[]]$RakeTest = "spec:integration"; break}
+    "Functional"    {[string[]]$RakeTest = "spec:functional"; break}
+    default         {throw "TestType $TestType not valid"}
+}
+
+foreach($test in $RakeTest) {
+    Write-Output "--- Chef $test run"
+    bundle exec rake $test
+    if (-not $?) { throw "Chef $test tests failed" }
+}
\ No newline at end of file
diff --git a/.expeditor/scripts/prep_and_run_tests.sh b/.expeditor/scripts/prep_and_run_tests.sh
new file mode 100755
index 0000000000..221f6e13cc
--- /dev/null
+++ b/.expeditor/scripts/prep_and_run_tests.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ -z "$1" ]
+  then
+    echo "No TestType supplied"
+fi
+
+TestType=$1
+
+curl -fsSL https://omnitruck.chef.io/chef/install.sh | bash -s -- -c "current" -P "chef-foundation" -v "$CHEF_FOUNDATION_VERSION"
+export PATH="/opt/chef/bin:${PATH}"
+
+if [ "$TestType" == "Unit" ]
+then
+    mkdir spec/data/nodes && touch spec/data/nodes/test.rb && touch spec/data/nodes/default.rb && touch spec/data/nodes/test.example.com.rb
+fi
+
+echo "--- Running Chef bundle install"
+bundle install --jobs=3 --retry=3 
+
+case $TestType in
+
+    Unit)
+        RakeTest=("spec:unit" "component_specs")
+        ;;
+
+    Integration)
+        RakeTest=("spec:integration")
+        ;;
+
+    Functional)
+        RakeTest=("spec:functional")
+        ;;
+
+    *)
+        echo -e "\nTestType $TestType not valid\n" >&2
+        exit 1
+        ;;
+esac
+
+for test in "${RakeTest[@]}"
+do
+    echo "--- Chef $test run"
+    bundle exec rake "$test"
+done
\ No newline at end of file
diff --git a/.expeditor/verify.adhoc.pipeline.yml b/.expeditor/verify.adhoc.pipeline.yml
new file mode 100644
index 0000000000..12f140bdb4
--- /dev/null
+++ b/.expeditor/verify.adhoc.pipeline.yml
@@ -0,0 +1,12 @@
+---
+expeditor:
+  defaults:
+    buildkite:
+      retry:
+        automatic:
+          limit: 1
+      timeout_in_minutes: 60
+
+steps:
+  - command: .buildkite/verify.adhoc.pipeline.sh | buildkite-agent pipeline upload
+    label: ":pipeline: Upload"
diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml
index ed321683e8..579fd42f5f 100644
--- a/.expeditor/verify.pipeline.yml
+++ b/.expeditor/verify.pipeline.yml
@@ -1,7 +1,5 @@
 ---
 expeditor:
-  cached_folders:
-    - vendor
   defaults:
     buildkite:
       retry:
@@ -10,366 +8,5 @@ expeditor:
       timeout_in_minutes: 60
 
 steps:
-
-#########################################################################
-  # Tests Ruby 3.1
-#########################################################################
-
-- label: "Integration Ubuntu 18.04 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:integration
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-        privileged: true
-
-- label: "Functional Ubuntu 18.04 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - apt-get update -y
-    - apt-get install -y cron locales libarchive-dev # needed for functional tests to pass
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:functional
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-        privileged: true
-
-- label: "Unit Ubuntu 18.04 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - apt-get update -y
-    - apt-get install -y libarchive-dev
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:unit
-    - bundle exec rake component_specs
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-
-- label: "Integration Ubuntu 20.04 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:integration
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-20.04:3.1
-        privileged: true
-
-- label: "Functional Ubuntu 20.04 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - apt-get update -y
-    - apt-get install -y cron locales libarchive-dev # needed for functional tests to pass
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:functional
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-20.04:3.1
-        privileged: true
-
-- label: "Unit Ubuntu 20.04 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - apt-get update -y
-    - apt-get install -y libarchive-dev
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:unit
-    - bundle exec rake component_specs
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-20.04:3.1
-
-- label: "Integration CentOS 7 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:integration
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/centos-7:3.1
-        privileged: true
-
-- label: "Functional CentOS 7 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - yum install -y crontabs e2fsprogs
-    - yum install -y libarchive-devel
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:functional
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/centos-7:3.1
-        privileged: true
-
-- label: "Unit CentOS 7 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - yum install -y libarchive-devel
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:unit
-    - bundle exec rake component_specs
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/centos-7:3.1
-
-- label: "Integration openSUSE 15 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - /workdir/.expeditor/scripts/zypper_prep.sh
-    - zypper install -y cron insserv-compat
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:integration
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/opensuse-15:3.1
-        privileged: true
-
-- label: "Functional openSUSE 15 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - /workdir/.expeditor/scripts/zypper_prep.sh
-    - zypper install -y cronie insserv-compat
-    - zypper install -y libarchive-devel
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:functional
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/opensuse-15:3.1
-        privileged: true
-
-- label: "Unit openSUSE 15 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - /workdir/.expeditor/scripts/zypper_prep.sh
-    - zypper install -y cron insserv-compat libarchive-devel
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:unit
-    - bundle exec rake component_specs
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/opensuse-15:3.1
-
-- label: "Integration AlmaLinux 8 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:integration
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/almalinux-8:3.1
-        privileged: true
-
-- label: "Functional AlmaLinux 8 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - dnf install -y crontabs e2fsprogs
-    - cd /workdir; bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:functional
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/almalinux-8:3.1
-        privileged: true
-        environment:
-          - FORCE_FFI_YAJL=ext
-          - CHEF_LICENSE=accept-no-persist
-
-- label: "Unit AlmaLinux 8 :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - dnf install dnf-plugins-core -y
-    - dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -y
-    - dnf config-manager --enable epel
-    - dnf config-manager --set-enabled powertools
-    - dnf install -y libarchive-devel
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec rake spec:unit
-    - bundle exec rake component_specs
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/almalinux-8:3.1
-
-- label: "Functional Windows :ruby: 3.1"
-  commands:
-    - .expeditor/scripts/bk_win_functional.ps1
-  expeditor:
-    executor:
-      windows:
-        privileged: true
-        single-use: true
-        shell: ["powershell", "-Command"]
-
-- label: "Integration Windows :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_win_integration.ps1
-  expeditor:
-    executor:
-      docker:
-        host_os: windows
-        image: rubydistros/windows-2019:3.1
-        environment:
-          - FORCE_FFI_YAJL=ext
-          - CHEF_LICENSE=accept-no-persist
-        shell: ["powershell", "-Command"]
-
-- label: "Unit Windows :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_win_unit.ps1
-  expeditor:
-    executor:
-      docker:
-        host_os: windows
-        image: rubydistros/windows-2019:3.1
-        environment:
-          - FORCE_FFI_YAJL=ext
-          - CHEF_LICENSE=accept-no-persist
-        shell: ["powershell", "-Command"]
-
-#########################################################################
-  # EXTERNAL GEM TESTING
-#########################################################################
-
-- label: "chef-zero gem :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec tasks/bin/run_external_test chef/chef-zero main rake pedant
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-        environment:
-          - PEDANT_OPTS=--skip-oc_id
-          - CHEF_FS=true
-
-- label: "cheffish gem :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec tasks/bin/run_external_test chef/cheffish main rake spec
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-
-- label: "chefspec gem :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec tasks/bin/run_external_test chefspec/chefspec main rake
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-
-- label: "knife-windows gem :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - bundle config set --local without omnibus_package
-    - bundle config set --local path 'vendor/bundle'
-    - bundle install --jobs=3 --retry=3
-    - bundle exec tasks/bin/run_external_test chef/knife-windows main rake spec
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-
-- label: "berkshelf gem :ruby: 3.1"
-  commands:
-    - /workdir/.expeditor/scripts/bk_container_prep.sh
-    - apt-get update -y
-    - apt-get install -y graphviz
-    - bundle config set --local without omnibus_package
-    - bundle install --jobs=3 --retry=3
-    - bundle exec tasks/bin/run_external_test chef/berkshelf main rake
-  expeditor:
-    executor:
-      docker:
-        image: rubydistros/ubuntu-18.04:3.1
-
-- label: ":habicat: Linux plan"
-  commands:
-    - sudo ./.expeditor/scripts/install-hab.sh 'x86_64-linux'
-    - sudo ./.expeditor/scripts/verify-plan.sh
-  timeout_in_minutes: 60
-  expeditor:
-    executor:
-      linux:
-        privileged: true
-        single-use: true
-
-- label: ":habicat: Linux plan (kernel2)"
-  commands:
-    - sudo ./.expeditor/scripts/install-hab.sh 'x86_64-linux-kernel2'
-    - sudo ./.expeditor/scripts/verify-plan.sh
-  timeout_in_minutes: 60
-  expeditor:
-    executor:
-      linux:
-        privileged: true
-        single-use: true
-
-- label: ":habicat: Windows plan"
-  commands:
-    - ./.expeditor/scripts/verify-plan.ps1
-  timeout_in_minutes: 0
-  expeditor:
-    executor:
-      windows:
-        privileged: true
-        single-use: true
-        shell: ["powershell", "-Command"]
+  - command: .buildkite/verify.pipeline.sh | buildkite-agent pipeline upload
+    label: ":pipeline: Upload"
\ No newline at end of file
diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb
index 217170ffef..2cb807f887 100644
--- a/omnibus/config/projects/chef.rb
+++ b/omnibus/config/projects/chef.rb
@@ -45,28 +45,8 @@ override :chef, version: "local_source"
 overrides_path = File.expand_path("../../../../omnibus_overrides.rb", current_file)
 instance_eval(IO.read(overrides_path), overrides_path)
 
-dependency "preparation"
+dependency "chef-local-source"
 
-dependency "chef"
-
-#
-# addons which require omnibus software defns (not direct deps of chef itself - RFC-063)
-#
-dependency "nokogiri" # (nokogiri cannot go in the Gemfile, see wall of text in the software defn)
-
-# FIXME?: might make sense to move dependencies below into the omnibus-software chef
-#  definition or into a chef-complete definition added to omnibus-software.
-dependency "gem-permissions"
-dependency "shebang-cleanup"
-dependency "version-manifest"
-dependency "openssl-customization"
-
-# devkit needs to come dead last these days so we do not use it to compile any gems
-dependency "ruby-msys2-devkit" if windows?
-
-dependency "ruby-cleanup"
-
-# further gem cleanup other projects might not yet want to use
 dependency "more-ruby-cleanup"
 
 package :rpm do
@@ -94,7 +74,7 @@ package :msi do
   upgrade_code msi_upgrade_code
   wix_candle_extension "WixUtilExtension"
   wix_light_extension "WixUtilExtension"
-  signing_identity "13B510D1CF1B3467856A064F1BEA12D0884D2528", machine_store: true
+  signing_identity ENV.fetch("OMNIBUS_SIGNING_IDENTITY", "13B510D1CF1B3467856A064F1BEA12D0884D2528"), machine_store: true
   parameters ChefLogDllPath: windows_safe_path(gem_path("chef-[0-9]*-x64-mingw-ucrt/ext/win32-eventlog/chef-log.dll")),
              ProjectLocationDir: project_location_dir
 end
@@ -104,4 +84,4 @@ package :appx do
   skip_packager true
 end
 
-runtime_dependency "coreutils" if rhel?
+runtime_dependency "coreutils" if rhel?
\ No newline at end of file
diff --git a/omnibus/config/software/chef-foundation.rb b/omnibus/config/software/chef-foundation.rb
new file mode 100644
index 0000000000..7352d6c9e3
--- /dev/null
+++ b/omnibus/config/software/chef-foundation.rb
@@ -0,0 +1,21 @@
+name "chef-foundation"
+license "Apache-2.0"
+license_file "LICENSE"
+
+# Grab accompanying notice file.
+# So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree.
+license_file "NOTICE"
+
+skip_transitive_dependency_licensing true
+
+if windows?
+  source path: "c:/opscode/chef"
+else
+  source path: "/opt/chef"
+end
+
+relative_path "chef-foundation"
+
+build do
+  sync "#{project_dir}", "#{install_dir}"
+end
\ No newline at end of file
diff --git a/omnibus/config/software/chef-local-source.rb b/omnibus/config/software/chef-local-source.rb
new file mode 100644
index 0000000000..528354422d
--- /dev/null
+++ b/omnibus/config/software/chef-local-source.rb
@@ -0,0 +1,115 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# expeditor/ignore: no version pinning
+
+name "chef"
+default_version "main"
+
+license "Apache-2.0"
+license_file "LICENSE"
+
+# Grab accompanying notice file.
+# So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree.
+license_file "NOTICE"
+
+# For the specific super-special version "local_source", build the source from
+# the local git checkout. This is what you'd want to occur by default if you
+# just ran omnibus build locally.
+version("local_source") do
+  source path: "#{project.files_path}/../..",
+         # Since we are using the local repo, we try to not copy any files
+         # that are generated in the process of bundle installing omnibus.
+         # If the install steps are well-behaved, this should not matter
+         # since we only perform bundle and gem installs from the
+         # omnibus cache source directory, but we do this regardless
+         # to maintain consistency between what a local build sees and
+         # what a github based build will see.
+         options: { exclude: [ "omnibus/vendor" ] }
+end
+
+# For any version other than "local_source", fetch from github.
+# This is the behavior the transitive omnibus software deps such as chef-dk
+# expect.
+if version != "local_source"
+  source git: "https://github.com/chef/chef.git"
+end
+
+dependency "chef-foundation"
+
+relative_path "chef"
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+
+  # The --without groups here MUST match groups in https://github.com/chef/chef/blob/main/Gemfile
+  excluded_groups = %w{docgen chefstyle}
+  excluded_groups << "ruby_prof" if aix?
+  excluded_groups << "ruby_shadow" if aix?
+  excluded_groups << "ed25519" if solaris2?
+
+  # these are gems which are not shipped but which must be installed in the testers
+  bundle_excludes = excluded_groups + %w{development test}
+
+  bundle "install --without #{bundle_excludes.join(" ")}", env: env
+
+  ruby "post-bundle-install.rb", env: env
+
+  # use the rake install task to build/install chef-config/chef-utils
+  command "rake install:local", env: env
+
+  gemspec_name = if windows?
+                   # Chef18 is built with ruby3.1 so platform name is changed.
+                   RUBY_PLATFORM == "x64-mingw-ucrt" ? "chef-universal-mingw-ucrt.gemspec" : "chef-universal-mingw32.gemspec"
+                 else
+                   "chef.gemspec"
+                 end
+
+  # This step will build native components as needed - the event log dll is
+  # generated as part of this step.  This is why we need devkit.
+  gem "build #{gemspec_name}", env: env
+
+  # ensure we put the gems in the right place to get picked up by the publish scripts
+  delete "pkg"
+  mkdir "pkg"
+  copy "chef*.gem", "pkg"
+
+  # Always deploy the powershell modules in the correct place.
+  if windows?
+    mkdir "#{install_dir}/modules/chef"
+    copy "distro/powershell/chef/*", "#{install_dir}/modules/chef"
+  end
+
+  block do
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "inspec-core-bin", without: excluded_groups, env: env
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "chef-bin", without: excluded_groups, env: env
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "chef", without: excluded_groups, env: env
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "ohai", without: excluded_groups, env: env
+  end
+
+  # The rubyzip gem ships with some test fixture data compressed in a format Apple's notarization service
+  # cannot understand. We need to delete that archive to pass notarization.
+  block "Delete test folder of rubyzip gem so downstream projects pass notarization" do
+    env["VISUAL"] = "echo"
+    %w{rubyzip}.each do |gem|
+      gem_install_dir = shellout!("#{install_dir}/embedded/bin/gem open #{gem}", env: env).stdout.chomp
+      remove_directory "#{gem_install_dir}/test"
+    end
+  end
+end
\ No newline at end of file
diff --git a/omnibus/config/software/more-ruby-cleanup.rb b/omnibus/config/software/more-ruby-cleanup.rb
index ba3c0d3fd4..7201da3860 100644
--- a/omnibus/config/software/more-ruby-cleanup.rb
+++ b/omnibus/config/software/more-ruby-cleanup.rb
@@ -24,8 +24,6 @@ license :project_license
 
 source path: "#{project.files_path}/#{name}"
 
-dependency "ruby"
-
 build do
   block "Removing console and setup binaries" do
     Dir.glob("#{install_dir}/embedded/lib/ruby/gems/*/gems/*/bin/{console,setup}").each do |f|
@@ -141,4 +139,4 @@ build do
       end
     end
   end
-end
+end
\ No newline at end of file
diff --git a/omnibus/omnibus-test.sh b/omnibus/omnibus-test.sh
index 47b21ae431..83c64ff2a6 100644..100755
--- a/omnibus/omnibus-test.sh
+++ b/omnibus/omnibus-test.sh
@@ -21,7 +21,9 @@ sudo rm -rf "$TMPDIR"
 mkdir -p "$TMPDIR"
 
 # Verify that we kill any orphaned test processes. Kill any orphaned rspec processes.
-sudo kill -9 $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') || true
+if [[ $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') ]]; then
+  sudo kill -9 $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') || true
+fi
 
 export PATH="/opt/chef/bin:$PATH"
 export BIN_DIR="/opt/chef/bin"
@@ -117,5 +119,17 @@ fi
 export CHEF_LICENSE=accept-no-persist
 
 cd "$chef_gem"
-sudo -E bundle install --jobs=3 --retry=3
-sudo -E bundle exec rspec --profile -f progress
+
+# only add -E if not on centos 6
+sudo_path="$(command -v sudo)"
+# cspell:disable-next-line
+rhel_sudo="/opt/rh/devtoolset-7/root/usr/bin/sudo"
+sudo_args=""
+if [[ "$sudo_path" != "$rhel_sudo" ]]; then
+  echo "HERE"
+  sudo -E bundle install --jobs=3 --retry=3
+  sudo -E bundle exec rspec --profile -f progress
+else
+  sudo bundle install --jobs=3 --retry=3
+  sudo bundle exec rspec --profile -f progress
+fi
diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb
index 99817f7caf..b29aea5a70 100644
--- a/omnibus/omnibus.rb
+++ b/omnibus/omnibus.rb
@@ -35,7 +35,7 @@ use_git_caching true
 
 # Enable S3 asset caching
 # ------------------------------
-use_s3_caching true
+use_s3_caching ENV.fetch("OMNIBUS_USE_S3_CACHING", false)
 s3_access_key  ENV["AWS_ACCESS_KEY_ID"]
 s3_secret_key  ENV["AWS_SECRET_ACCESS_KEY"]
 s3_bucket      "opscode-omnibus-cache"
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 1d040b0f1f..669385e321 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -138,9 +138,9 @@ RSpec.configure do |config|
 
   config.filter_run_excluding skip_buildkite: true if ENV["BUILDKITE"]
 
-  config.filter_run_excluding fips_mode: !fips_mode_build? unless opensuse?
-  # RubyDistros OpenSUSE docker images have a broken fips
-  config.filter_run_excluding :fips_mode if opensuse?
+  config.filter_run_excluding fips_mode: !fips_mode_build? unless windows?
+  # Skip fips on windows
+  config.filter_run_excluding :fips_mode if windows?
 
   config.filter_run_excluding windows_only: true unless windows?
   config.filter_run_excluding not_supported_on_windows: true if windows?