add esoteric platforms to validate adhoc/release pipelines (#13546)

* run pre-command on all verify pipelines
* update pre-command to pull aws credentials on both chef and chef canary
* update pre-command pipeline names
* add esoteric builds
* add build_timestamp environment variable
* update sha on omnibus plugin
* skip transitive depenceny licensing for chef-foundation
* use chef-foundation
* remove notarize
* add solaris tests
* add aix tests
* add all esoteric platforms
* Adding in changes to run publish of omnibus packages to Artifactory, as well as related gems for Linux/Windows
* set lib path in chef omnibus
* move lib dirs to software
* Adding in if statement around artifactory api key, as its not needed for macOS (only Artifactory password needed)
* Log every macho?
* Fixing regex check
* Extend the bin dirs and lib dirs.
* remove mac osx 10
* Remove lib dirs.
* Update the version of omnibus-buildkite-plugin.
* Fixing unbound variable error on filter; Fixing notarize macOS so it doesn't run when macOS is filtered out of build
* Adding in changes to retry/timeout on esoteric builds; Adding in dependency check on macOS Notarize
* add build record step on validate/release pipeline
* fix comparison for pipeline slug
* Adding in code to promote the packages when runnong validate/release
* remove comments and fix omnibus branch
* Fixing potential issue with BUILDKITE_BUILD_CREATOR_TEAMS being unset
* get role attached to ec2 server instead of hardcoding
* Update omnibus bundle with license_scout
* Dynamically determine the version of ruby.
* replace . with _ in notraize step
* Move lib_dirs logic to chef-foundation
* remove skip_transitive_depenency_licensing and add comments to build/test arrays
* increase timeout on esoteric to 120 mins
* fix mac osx depends on and update pipeline name to regex
* align omnibus plugin version to 0.2.83
* add check for centos6 in omnibus-test.sh
* add comments for empty array check

---------

Signed-off-by: Evan Ahlberg <evanahlberg@gmail.com>
Signed-off-by: Jesse Prieur <jesse.prieur@gmail.com>
Signed-off-by: Gregory Schofield <grschofi@progress.com>
Co-authored-by: Jesse Prieur <jesse.prieur@gmail.com>
Co-authored-by: Gregory Schofield <grschofi@progress.com>

14 files changed, 337 insertions, 154 deletions

diff --git a/.buildkite-platform.json b/.buildkite-platform.json
index 4aa454c9f3..7597215975 100644
--- a/.buildkite-platform.json
+++ b/.buildkite-platform.json
@@ -1,4 +1,4 @@
 {
-    "chef_foundation": "0.1.24",
+    "chef_foundation": "3.0.3",
     "omnibus_toolchain": "3.0.0"
 }
\ No newline at end of file
diff --git a/.buildkite/build-test-omnibus.sh b/.buildkite/build-test-omnibus.sh
index a77b1ba4bd..ddf4c9533f 100755
--- a/.buildkite/build-test-omnibus.sh
+++ b/.buildkite/build-test-omnibus.sh
@@ -1,17 +1,24 @@
-if [[ $BUILDKITE_ORGANIZATION_SLUG == "chef-oss" ]]; then
+if [[ -z "${BUILDKITE_BUILD_CREATOR_TEAMS:-}" ]]
+then
   echo "- block: Build & Test Omnibus Packages"
   echo "  prompt: Continue to run omnibus package build and tests for applicable platforms?"
+else
+  echo "- wait: ~"
 fi
 
 FILTER="${OMNIBUS_FILTER:=*}"
 
-platforms=("amazon-2:centos-7" "centos-6:centos-6" "centos-7:centos-7" "centos-8:centos-8" "rhel-9:rhel-9" "debian-9:debian-9" "debian-10:debian-9" "debian-11:debian-9" "ubuntu-1604:ubuntu-1604" "ubuntu-1804:ubuntu-1604" "ubuntu-2004:ubuntu-1604" "ubuntu-2204:ubuntu-1604" "sles-15:sles-15" "windows-2019:windows-2019")
+# array of all container platforms in the format test-platform:build-platform
+container_platforms=("amazon-2:centos-7" "centos-6:centos-6" "centos-7:centos-7" "centos-8:centos-8" "rhel-9:rhel-9" "debian-9:debian-9" "debian-10:debian-9" "debian-11:debian-9" "ubuntu-1604:ubuntu-1604" "ubuntu-1804:ubuntu-1604" "ubuntu-2004:ubuntu-1604" "ubuntu-2204:ubuntu-1604" "sles-15:sles-15" "windows-2019:windows-2019")
+
+# array of all esoteric platforms in the format test-platform:build-platform
+esoteric_platforms=("aix-7.1-powerpc:aix-7.1-powerpc" "aix-7.2-powerpc:aix-7.1-powerpc" "aix-7.3-powerpc:aix-7.1-powerpc" "el-7-ppc64:el-7-ppc64" "el-7-ppc64le:el-7-ppc64le" "el-7-s390x:el-7-s390x" "el-8-s390x:el-7-s390x" "freebsd-12-amd64:freebsd-12-amd64" "freebsd-13-amd64:freebsd-12-amd64" "mac_os_x-10.15-x86_64:mac_os_x-10.15-x86_64" "mac_os_x-11-x86_64:mac_os_x-10.15-x86_64" "mac_os_x-12-x86_64:mac_os_x-10.15-x86_64" "mac_os_x-11-arm64:mac_os_x-11-arm64" "mac_os_x-12-arm64:mac_os_x-11-arm64" "solaris2-5.11-i386:solaris2-5.11-i386" "solaris2-5.11-sparc:solaris2-5.11-sparc" "sles-12-s390x:sles-12-s390x" "sles-15-s390x:sles-12-s390x")
 
 omnibus_build_platforms=()
 omnibus_test_platforms=()
 
 # build build array and test array based on filter
-for platform in ${platforms[@]}; do
+for platform in ${container_platforms[@]}; do
     case ${platform%:*} in
         $FILTER)
             omnibus_build_platforms[${#omnibus_build_platforms[@]}]=${platform#*:}
@@ -21,92 +28,270 @@ for platform in ${platforms[@]}; do
 done
 
 # remove duplicates from build array
-omnibus_build_platforms=($(printf "%s\n" "${omnibus_build_platforms[@]}" | sort -u | tr '\n' ' '))
+if [[ ! -z "${omnibus_build_platforms:-}" ]]
+then
+  omnibus_build_platforms=($(printf "%s\n" "${omnibus_build_platforms[@]}" | sort -u | tr '\n' ' '))
+fi
+
+## add esoteric platforms in chef/chef-canary
+if [ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]
+then
+  esoteric_build_platforms=()
+  esoteric_test_platforms=()
+
+  # build build array and test array based on filter
+  for platform in ${esoteric_platforms[@]}; do
+    case ${platform%:*} in
+        $FILTER)
+            esoteric_build_platforms[${#esoteric_build_platforms[@]}]=${platform#*:}
+            esoteric_test_platforms[${#esoteric_test_platforms[@]}]=$platform
+            ;;
+    esac
+  done
+
+  # remove duplicates from build array
+  # using shell parameter expansion this checks to make sure the esoteric_build_platforms array isn't empty if OMNIBUS_FILTER is only container platforms 
+  # prevents esoteric_build_platforms unbound variable error 
+  if [[ ! -z "${esoteric_build_platforms:-}" ]]
+  then
+    esoteric_build_platforms=($(printf "%s\n" "${esoteric_build_platforms[@]}" | sort -u | tr '\n' ' '))
+  fi
+fi
+
+# using shell parameter expansion this checks to make sure the omnibus_build_platforms array isn't empty if OMNIBUS_FILTER is only esoteric platforms 
+# prevents omnibus_build_platforms unbound variable error 
+if [[ ! -z "${omnibus_build_platforms:-}" ]]
+then
+  for platform in ${omnibus_build_platforms[@]}; do
+    if [[ $platform != *"windows"* ]]; then
+      echo "- label: \":hammer_and_wrench::docker: $platform\""
+      echo "  retry:"
+      echo "    automatic:"
+      echo "      limit: 1"
+      echo "  key: build-$platform"
+      echo "  agents:"
+      echo "    queue: default-privileged"
+      echo "  plugins:"
+      echo "  - docker#v3.5.0:"
+      echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+      echo "      privileged: true"
+      echo "      propagate-environment: true"
+      echo "      environment:"
+      echo "        - ARTIFACTORY_PASSWORD"
+      echo "        - ARTIFACTORY_API_KEY"
+      echo "        - RPM_SIGNING_KEY"
+      echo "        - CHEF_FOUNDATION_VERSION"
+      echo "  commands:"
+      echo "    - ./.expeditor/scripts/omnibus_chef_build.sh"
+      echo "  timeout_in_minutes: 60"
+    else 
+      echo "- label: \":hammer_and_wrench::windows: $platform\""
+      echo "  retry:"
+      echo "    automatic:"
+      echo "      limit: 1"
+      echo "  key: build-$platform"
+      echo "  agents:"
+      echo "    queue: default-$platform-privileged"
+      echo "  plugins:"
+      echo "  - docker#v3.5.0:"
+      echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+      echo "      shell:"
+      echo "      - powershell"
+      echo "      - \"-Command\""
+      echo "      propagate-environment: true"
+      echo "      environment:"
+      echo "        - CHEF_FOUNDATION_VERSION"
+      echo "        - BUILDKITE_AGENT_ACCESS_TOKEN"
+      echo "        - ARTIFACTORY_PASSWORD"
+      echo "        - ARTIFACTORY_API_KEY"
+      echo "        - AWS_ACCESS_KEY_ID"
+      echo "        - AWS_SECRET_ACCESS_KEY"
+      echo "        - AWS_SESSION_TOKEN"
+      echo "      volumes:"
+      echo '        - "c:\\buildkite-agent:c:\\buildkite-agent"'
+      echo "  commands:"
+      echo "    - ./.expeditor/scripts/omnibus_chef_build.ps1"
+      echo "  timeout_in_minutes: 120"
+    fi
+  done
+fi
+
+if [ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ] && [[ ! -z "${esoteric_build_platforms:-}" ]]
+then
 
-for platform in ${omnibus_build_platforms[@]}; do
-  if [[ $platform != *"windows"* ]]; then
-    echo "- label: \":hammer_and_wrench::docker: $platform\""
+  for platform in ${esoteric_build_platforms[@]}; do
+    # replace . with _ in build key
+    build_key=$(echo $platform | tr . _)
+    echo "- env:"
+    if [ $platform == "el-7-ppc64" ] || [ $platform == "el-7-ppc64le" ]
+    then
+      echo "    OMNIBUS_FIPS_MODE: true"
+    else
+      echo "    OMNIBUS_FIPS_MODE: false"
+    fi
+    echo "    IGNORE_CACHE: true"
+    echo "  key: build-$build_key"
+    echo "  label: \":hammer_and_wrench: $platform\""
     echo "  retry:"
     echo "    automatic:"
     echo "      limit: 1"
-    echo "  key: build-$platform"
+    echo "  timeout_in_minutes: 120"
     echo "  agents:"
-    echo "    queue: default-privileged"
+    echo "    queue: omnibus-$platform"
+    if [[ $platform == mac_os_x* ]]
+    then
+      echo "    omnibus: builder"
+      echo "    omnibus-toolchain: \"*\""
+    fi
     echo "  plugins:"
-    echo "  - docker#v3.5.0:"
-    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
-    echo "      privileged: true"
-    echo "      propagate-environment: true"
-    echo "      environment:"
-    echo "        - RPM_SIGNING_KEY"
-    echo "        - CHEF_FOUNDATION_VERSION"
-    echo "  commands:"
-    echo "    - ./.expeditor/scripts/omnibus_chef_build.sh"
-    echo "  timeout_in_minutes: 60"
-  else 
-    echo "- label: \":hammer_and_wrench::windows: $platform\""
-    echo "  retry:"
-    echo "    automatic:"
-    echo "      limit: 1"
-    echo "  key: build-$platform"
+    echo "  - chef/omnibus#852c8f81fb6dd12ff3471a8d825ec20a1168c4c4:"
+    echo "      build: chef"
+    echo "      chef-foundation-version: $CHEF_FOUNDATION_VERSION"
+    echo "      config: omnibus/omnibus.rb"
+    echo "      install-dir: \"/opt/chef\""
+    if [ $build_key == "mac_os_x-10_15-x86_64" ]
+    then
+      echo "      remote-host: buildkite-omnibus-$platform"
+    fi
+    echo "      omnibus-pipeline-definition-path: \".expeditor/release.omnibus.yml\""
+    if [ $build_key == "mac_os_x-11-arm64" ]
+    then
+      echo "  concurrency: 1"
+      echo "  concurrency_group: omnibus-$build_key/build/chef"
+    fi
+  done
+
+  if  [[ " ${esoteric_build_platforms[*]} " =~ "mac_os_x" ]]
+  then
+    echo "- key: notarize-macos"
+    echo "  label: \":lock_with_ink_pen: Notarize macOS Packages\""
     echo "  agents:"
-    echo "    queue: default-$platform-privileged"
+    echo "    queue: omnibus-mac_os_x-12-x86_64"
     echo "  plugins:"
-    echo "  - docker#v3.5.0:"
-    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
-    echo "      shell:"
-    echo "      - powershell"
-    echo "      - \"-Command\""
-    echo "      propagate-environment: true"
-    echo "      environment:"
-    echo "        - CHEF_FOUNDATION_VERSION"
-    echo "        - BUILDKITE_AGENT_ACCESS_TOKEN"
-    echo "        - AWS_ACCESS_KEY_ID"
-    echo "        - AWS_SECRET_ACCESS_KEY"
-    echo "        - AWS_SESSION_TOKEN"
-    echo "      volumes:"
-    echo '        - "c:\\buildkite-agent:c:\\buildkite-agent"'
-    echo "  commands:"
-    echo "    - ./.expeditor/scripts/omnibus_chef_build.ps1"
-    echo "  timeout_in_minutes: 120"
+    echo "  - chef/omnibus#v0.2.83:"
+    echo "      config: omnibus/omnibus.rb"
+    echo "      remote-host: buildkite-omnibus-mac_os_x-12-x86_64"
+    echo "      notarize-macos-package: chef"
+    echo "      omnibus-pipeline-definition-path: \".expeditor/release.omnibus.yml\""
+    echo "  depends_on:"
+    for platform in ${esoteric_build_platforms[@]}; do
+      if [[ $platform =~ mac_os_x ]]
+      then
+        echo "  - build-$(echo $platform | tr . _)"
+      fi
+    done
   fi
-done
+fi
+
+if [ $BUILDKITE_PIPELINE_SLUG == "chef-chef-main-validate-release" ]
+then
+  echo "- wait: ~"
+  echo "- key: create-build-record"
+  echo "  label: \":artifactory: Create Build Record\""
+  echo "  plugins:"
+  echo "  - chef/omnibus#v0.2.83:"
+  echo "    create-build-record: chef"
+fi
 
 echo "- wait: ~"
 
-for platform in ${omnibus_test_platforms[@]}; do
-  if [[ $platform != *"windows"* ]]; then
+# using shell parameter expansion this checks to make sure the omnibus_test_platforms array isn't empty if OMNIBUS_FILTER is only esoteric platforms 
+# prevents omnibus_test_platforms unbound variable error 
+if [[ ! -z "${omnibus_test_platforms:-}" ]]
+then
+  for platform in ${omnibus_test_platforms[@]}; do
+    if [[ $platform != *"windows"* ]]; then
+      echo "- env:"
+      echo "    OMNIBUS_BUILDER_KEY: build-${platform#*:}"
+      echo "  label: \":mag::docker: ${platform%:*}\""
+      echo "  retry:"
+      echo "    automatic:"
+      echo "      limit: 1"
+      echo "  agents:"
+      echo "    queue: default-privileged"
+      echo "  plugins:"
+      echo "  - docker#v3.5.0:"
+      echo "      image: chefes/omnibus-toolchain-${platform%:*}:$OMNIBUS_TOOLCHAIN_VERSION"
+      echo "      privileged: true"
+      echo "      propagate-environment: true"
+      echo "  commands:"
+      echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.sh"
+      echo "    - omnibus/omnibus-test.sh"
+      echo "  timeout_in_minutes: 60"
+    else
+      echo "- env:"
+      echo "    OMNIBUS_BUILDER_KEY: build-windows-2019"
+      echo "  key: test-windows-2019"
+      echo '  label: ":mag::windows: windows-2019"'
+      echo "  retry:"
+      echo "    automatic:"
+      echo "      limit: 1"
+      echo "  agents:"
+      echo "    queue: default-windows-2019-privileged"
+      echo "  commands:"
+      echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.ps1"
+      echo "    - ./omnibus/omnibus-test.ps1"
+      echo "  timeout_in_minutes: 120"
+    fi
+  done
+fi
+
+# using shell parameter expansion this checks to make sure the esoteric_test_platforms array isn't empty if OMNIBUS_FILTER is only container platforms
+# prevents esoteric_test_platforms unbound variable error 
+if [ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ] && [[ ! -z "${esoteric_test_platforms:-}" ]]
+then
+  
+  for platform in ${esoteric_test_platforms[@]}; do
+    build_key=$(echo ${platform#*:} | tr . _)
+    test_key=$(echo ${platform%:*} | tr . _)
     echo "- env:"
-    echo "    OMNIBUS_BUILDER_KEY: build-${platform#*:}"
-    echo "  label: \":mag::docker: ${platform%:*}\""
+    if [ $build_key == "el-7-ppc64" ] || [ $build_key == "el-7-ppc64le" ]
+    then
+      echo "    OMNIBUS_FIPS_MODE: true"
+    else
+      echo "    OMNIBUS_FIPS_MODE: false"
+    fi
+    echo "    OMNIBUS_BUILDER_KEY: build-${build_key}"
+    echo "  key: test-${test_key}"
+    echo "  label: \":mag: ${platform%:*}\""
     echo "  retry:"
     echo "    automatic:"
     echo "      limit: 1"
+    echo "  timeout_in_minutes: 90"
     echo "  agents:"
-    echo "    queue: default-privileged"
+    echo "    queue: omnibus-${platform%:*}"
+    if [ $build_key == "mac_os_x-10_15-x86_64" ] || [ $build_key == "mac_os_x-11-arm64" ]
+    then
+      echo "    omnibus: tester"
+      echo "    omnibus-toolchain: \"*\""
+    fi
     echo "  plugins:"
-    echo "  - docker#v3.5.0:"
-    echo "      image: chefes/omnibus-toolchain-${platform%:*}:$OMNIBUS_TOOLCHAIN_VERSION"
-    echo "      privileged: true"
-    echo "      propagate-environment: true"
-    echo "  commands:"
-    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.sh"
-    echo "    - omnibus/omnibus-test.sh"
-    echo "  timeout_in_minutes: 60"
-  else
-    echo "- env:"
-    echo "    OMNIBUS_BUILDER_KEY: build-windows-2019"
-    echo "  key: test-windows-2019"
-    echo '  label: ":mag::windows: windows-2019"'
-    echo "  retry:"
-    echo "    automatic:"
-    echo "      limit: 1"
-    echo "  agents:"
-    echo "    queue: default-windows-2019-privileged"
-    echo "  commands:"
-    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.ps1"
-    echo "    - ./omnibus/omnibus-test.ps1"
-    echo "  timeout_in_minutes: 120"
-  fi
-done
+    echo "  - chef/omnibus#v0.2.83:"
+    echo "      test: chef"
+    echo "      test-path: omnibus/omnibus-test.sh"
+    echo "      install-dir: \"/opt/chef\""
+    if [[ ${platform%:*} == mac_os_x*x86_64 ]]
+    then
+      echo "      remote-host: buildkite-omnibus-${platform%:*}"
+    fi
+    if [ $test_key == "mac_os_x-11-arm64" ] || [ $test_key == "mac_os_x-12-arm64" ]
+    then
+      echo "  concurrency: 1"
+      echo "  concurrency_group: omnibus-$test_key/test/chef"
+    fi
+    if [ $test_key == "freebsd-13-amd64" ]
+    then
+      echo "  soft_fail: true"
+    fi
+  done
+fi
 
+if [ $BUILDKITE_PIPELINE_SLUG == "chef-chef-main-validate-release" ]
+then
+  echo "- wait: ~"
+  echo "- key: promote"
+  echo "  label: \":artifactory: Promote to Current\""
+  echo "  plugins:"
+  echo "  - chef/omnibus#v0.2.83:"
+  echo "    promote: chef"
+fi
\ No newline at end of file
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 874b91ebdd..f4c1f76d42 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -3,23 +3,24 @@
 set -eu
 
 # Only execute in the verify pipeline
-[[ "$BUILDKITE_PIPELINE_NAME" =~ verify$ ]] || [[ "$BUILDKITE_PIPELINE_NAME" =~ validate/.* ]] || exit 0
+[[ "$BUILDKITE_PIPELINE_NAME" =~ (verify|validate/(release|adhoc|canary))$ ]]
 
 docker ps || true
 
 # Get chef foundation version from the json file
 CHEF_FOUNDATION_VERSION=$(cat .buildkite-platform.json | jq -r '.chef_foundation')
 export CHEF_FOUNDATION_VERSION
-echo $CHEF_FOUNDATION_VERSION
+echo "Chef Foundation Version: $CHEF_FOUNDATION_VERSION" 
 
 OMNIBUS_TOOLCHAIN_VERSION=$(cat .buildkite-platform.json | jq -r '.omnibus_toolchain')
 export OMNIBUS_TOOLCHAIN_VERSION
-echo $OMNIBUS_TOOLCHAIN_VERSION
+echo "Omnibus Toolchain Version: $OMNIBUS_TOOLCHAIN_VERSION"
 
 if [ $BUILDKITE_STEP_KEY == "build-windows-2019" ] && [[ "$BUILDKITE_ORGANIZATION_SLUG" =~ chef(-canary)?$ ]]
 then
   TOKEN=$(curl -sX PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
-  RESPONSE=$(curl -sH "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/default-windows-2019-privileged-$BUILDKITE_ORGANIZATION_SLUG-Role)
+  ROLE=$(curl -sH "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/)
+  RESPONSE=$(curl -sH "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/$ROLE)
   AWS_ACCESS_KEY_ID=$(echo $RESPONSE | jq -r '.AccessKeyId')
   export AWS_ACCESS_KEY_ID
   AWS_SECRET_ACCESS_KEY=$(echo $RESPONSE | jq -r '.SecretAccessKey')
@@ -57,9 +58,20 @@ if [[ "$BUILDKITE_BRANCH" != "main" ]]; then
   fi
 fi
 
-# Only execute if on RHEL/CentOS/SLES
-if [[ "$BUILDKITE_LABEL" =~ rhel|sles|centos ]] && [[ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]]; then
+# Only if on chef-canary or chef org
+if [[ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]]; then
+
   export VAULT_ADDR="https://vault.ps.chef.co"
-  export VAULT_TOKEN="$(vault login -method=aws -path=aws/private-cd -token-only header_value=vault.ps.chef.co role=ci)"
-  export RPM_SIGNING_KEY="$(vault kv get -field packages_at_chef_io account/static/packages/signing_certs)"
+  export VAULT_TOKEN=$(vault login -method=aws -path=aws/private-cd -token-only header_value=vault.ps.chef.co role=ci)
+
+  if [[ ! $BUILDKITE_LABEL =~ macOS|mac_os_x ]]; then
+    lita_password=$(aws ssm get-parameter --name "artifactory-lita-password" --with-decryption --query Parameter.Value --output text)
+    export ARTIFACTORY_API_KEY=$(echo -n "lita:${lita_password}" | base64)
+  fi
+  export ARTIFACTORY_PASSWORD=$(vault kv get -field password account/static/artifactory/buildkite)
+
+  # Only if on RPM-based Linux distros
+  if [[ "$BUILDKITE_LABEL" =~ rhel|sles|centos ]]; then
+    export RPM_SIGNING_KEY=$(vault kv get -field packages_at_chef_io account/static/packages/signing_certs)
+  fi
 fi
\ No newline at end of file
diff --git a/.buildkite/verify.adhoc.pipeline.sh b/.buildkite/verify.adhoc.pipeline.sh
index 51ad3eed4a..e72ded85dc 100755
--- a/.buildkite/verify.adhoc.pipeline.sh
+++ b/.buildkite/verify.adhoc.pipeline.sh
@@ -4,6 +4,8 @@
 set -eu
 
 echo "---"
+echo "env:"
+echo "  BUILD_TIMESTAMP: $(date +%Y-%m-%d_%H-%M-%S)"
 echo "steps:"
 echo ""
 
diff --git a/.buildkite/verify.pipeline.sh b/.buildkite/verify.pipeline.sh
index b3ced7ef17..c675ab42f6 100755
--- a/.buildkite/verify.pipeline.sh
+++ b/.buildkite/verify.pipeline.sh
@@ -4,6 +4,8 @@
 set -eu
 
 echo "---"
+echo "env:"
+echo "  BUILD_TIMESTAMP: $(date +%Y-%m-%d_%H-%M-%S)"
 echo "steps:"
 echo ""
 
@@ -168,6 +170,6 @@ for plan in ${habitat_plans[@]}; do
 done
 
 # include build and test omnibus pipeline
-# DIR="${BASH_SOURCE%/*}"
-# if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
-# source "$DIR/build-test-omnibus.sh"
\ No newline at end of file
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+source "$DIR/build-test-omnibus.sh"
\ No newline at end of file
diff --git a/.expeditor/config.yml b/.expeditor/config.yml
index 6a338ec4e1..29d0cf96f5 100644
--- a/.expeditor/config.yml
+++ b/.expeditor/config.yml
@@ -143,11 +143,11 @@ subscriptions:
             - "Expeditor: Skip Habitat"
             - "Expeditor: Skip All"
           only_if: built_in:bump_version
-      - trigger_pipeline:omnibus/release:
-          ignore_labels:
-            - "Expeditor: Skip Omnibus"
-            - "Expeditor: Skip All"
-          only_if: built_in:bump_version
+      # - trigger_pipeline:omnibus/release:
+      #     ignore_labels:
+      #       - "Expeditor: Skip Omnibus"
+      #       - "Expeditor: Skip All"
+      #     only_if: built_in:bump_version
       - trigger_pipeline:validate/release:
           ignore_labels:
             - "Expeditor: Skip Omnibus"
diff --git a/.expeditor/scripts/omnibus_chef_build.ps1 b/.expeditor/scripts/omnibus_chef_build.ps1
index c3ac569ff4..bcedaa1b60 100644
--- a/.expeditor/scripts/omnibus_chef_build.ps1
+++ b/.expeditor/scripts/omnibus_chef_build.ps1
@@ -31,9 +31,11 @@ $env:ARTIFACTORY_USERNAME="buildkite"
 Write-Output "--- Install Chef Foundation"
 . { Invoke-WebRequest -useb https://omnitruck.chef.io/chef/install.ps1 } | Invoke-Expression; install -channel "current" -project "chef-foundation" -v $CHEF_FOUNDATION_VERSION
 
+$env:PROJECT_NAME="chef"
+$env:OMNIBUS_PIPELINE_DEFINITION_PATH="${ScriptDir}/../release.omnibus.yaml"
 $env:OMNIBUS_SIGNING_IDENTITY="${thumb}"
 $env:HOMEDRIVE = "C:"
-$env:HOMEPATH = "\buildkite-agent"
+$env:HOMEPATH = "\Users\ContainerAdministrator"
 $env:OMNIBUS_TOOLCHAIN_INSTALL_DIR = "C:\opscode\omnibus-toolchain"
 $env:SSL_CERT_FILE = "${env:OMNIBUS_TOOLCHAIN_INSTALL_DIR}\embedded\ssl\certs\cacert.pem"
 $env:MSYS2_INSTALL_DIR = "C:\msys64"
@@ -59,8 +61,11 @@ bundle exec omnibus build chef -l internal --override append_timestamp:false
 Write-Output "--- Uploading package to BuildKite"
 C:\buildkite-agent\bin\buildkite-agent.exe artifact upload "pkg/*.msi*"
 
-# if ($env:BUILDKITE_ORGANIZATION_SLUG -ne "chef-oss" )
-# {
-#   Write-Output "--- Publishing package to Artifactory"
-#   bundle exec ruby "${SCRIPT_DIR}/omnibus_chef_publish.rb"
-# }
+if ($env:BUILDKITE_ORGANIZATION_SLUG -ne "chef-oss" )
+{
+  Write-Output "--- Setting up Gem API Key"
+  $env:GEM_HOST_API_KEY = "Basic ${env:ARTIFACTORY_API_KEY}"
+
+  Write-Output "--- Publishing package to Artifactory"
+  bundle exec ruby "${ScriptDir}/omnibus_chef_publish.rb"
+}
diff --git a/.expeditor/scripts/omnibus_chef_build.sh b/.expeditor/scripts/omnibus_chef_build.sh
index f2ede50d08..3c4130a8c3 100755
--- a/.expeditor/scripts/omnibus_chef_build.sh
+++ b/.expeditor/scripts/omnibus_chef_build.sh
@@ -45,5 +45,10 @@ do
   buildkite-agent artifact upload "pkg/*.${ext}*"
 done
 
-# echo "--- Publishing package to Artifactory"
-# bundle exec ruby "${SCRIPT_DIR}/omnibus_chef_publish.rb"
\ No newline at end of file
+if [[ $BUILDKITE_ORGANIZATION_SLUG != "chef-oss" ]]; then
+  echo "--- Setting up Gem credentials"
+  export GEM_HOST_API_KEY="Basic ${ARTIFACTORY_API_KEY}"
+
+  echo "--- Publishing package to Artifactory"
+  bundle exec ruby "${SCRIPT_DIR}/omnibus_chef_publish.rb"
+fi
\ No newline at end of file
diff --git a/.expeditor/scripts/omnibus_chef_publish.rb b/.expeditor/scripts/omnibus_chef_publish.rb
index a818c67799..9413e9740d 100644
--- a/.expeditor/scripts/omnibus_chef_publish.rb
+++ b/.expeditor/scripts/omnibus_chef_publish.rb
@@ -87,7 +87,7 @@ if (project_name == "chef") && (ENV['ADHOC'] != 'true')
     # This mimics the behavior of the gem command line, and is a public api:
     # http://docs.seattlerb.org/rubygems/Gem/Command.html
     gem_pusher = Gem::Commands::PushCommand.new
-    gem_pusher.handle_options [gem_path, '--host', artifactory_endpoint, '--key', 'artifactory_api_key', '--verbose']
+    gem_pusher.handle_options [gem_path, '--host', artifactory_endpoint, '--verbose']
     gem_pusher.execute
   end
 end
diff --git a/cspell.json b/cspell.json
index 0a504ef1cc..f1c875dace 100644
--- a/cspell.json
+++ b/cspell.json
@@ -629,6 +629,7 @@
     "linuxmint",
     "LISTBOX",
     "listprop",
+    "lita",
     "ljust",
     "lltstype",
     "losetup",
diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb
index 10f7f25b15..27e420e92a 100644
--- a/omnibus/config/projects/chef.rb
+++ b/omnibus/config/projects/chef.rb
@@ -41,34 +41,10 @@ end
 
 override :chef, version: "local_source"
 
-# Load dynamically updated overrides
-overrides_path = File.expand_path("../../../../omnibus_overrides.rb", current_file)
-instance_eval(IO.read(overrides_path), overrides_path)
-
-dependency "preparation"
-# dependency "chef-local-source"
-
-dependency "chef"
-
-#
-# addons which require omnibus software defns (not direct deps of chef itself - RFC-063)
-#
-dependency "nokogiri" # (nokogiri cannot go in the Gemfile, see wall of text in the software defn)
-
-# FIXME?: might make sense to move dependencies below into the omnibus-software chef
-#  definition or into a chef-complete definition added to omnibus-software.
-dependency "gem-permissions"
+dependency "chef-local-source"
 dependency "shebang-cleanup"
-dependency "version-manifest"
-dependency "openssl-customization"
-
-# devkit needs to come dead last these days so we do not use it to compile any gems
-dependency "ruby-msys2-devkit" if windows?
-
-dependency "ruby-cleanup"
 
 # further gem cleanup other projects might not yet want to use
-
 dependency "more-ruby-cleanup"
 
 package :rpm do
diff --git a/omnibus/config/software/chef-local-source.rb b/omnibus/config/software/chef-local-source.rb
index 528354422d..49a2bfda59 100644
--- a/omnibus/config/software/chef-local-source.rb
+++ b/omnibus/config/software/chef-local-source.rb
@@ -25,6 +25,8 @@ license_file "LICENSE"
 # So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree.
 license_file "NOTICE"
 
+skip_transitive_dependency_licensing false
+
 # For the specific super-special version "local_source", build the source from
 # the local git checkout. This is what you'd want to occur by default if you
 # just ran omnibus build locally.
@@ -47,6 +49,16 @@ if version != "local_source"
   source git: "https://github.com/chef/chef.git"
 end
 
+# In order to pass notarization we need to sign any binaries and libraries included in the package.
+# This makes sure we include and bins and libs that are brought in by gems.
+ruby_version = "3.1.2"
+ruby_version = ruby_version.split(".")[0..1].join(".")
+ruby_mmv = "#{ruby_version}.0"
+ruby_dir = "#{install_dir}/embedded/lib/ruby/#{ruby_mmv}"
+gem_dir = "#{install_dir}/embedded/lib/ruby/gems/#{ruby_mmv}"
+bin_dirs bin_dirs.concat ["#{gem_dir}/gems/*/bin/**"]
+lib_dirs ["#{ruby_dir}/**", "#{gem_dir}/extensions/**", "#{gem_dir}/bundler/gems/extensions/**", "#{gem_dir}/bundler/gems/*", "#{gem_dir}/bundler/gems/*/lib/**", "#{gem_dir}/gems/*", "#{gem_dir}/gems/*/lib/**", "#{gem_dir}/gems/*/ext/**"]
+
 dependency "chef-foundation"
 
 relative_path "chef"
diff --git a/omnibus/omnibus-test.sh b/omnibus/omnibus-test.sh
index 9bdaa7cdc5..83c64ff2a6 100755
--- a/omnibus/omnibus-test.sh
+++ b/omnibus/omnibus-test.sh
@@ -120,5 +120,16 @@ export CHEF_LICENSE=accept-no-persist
 
 cd "$chef_gem"
 
-sudo -E bundle install --jobs=3 --retry=3
-sudo -E bundle exec rspec --profile -f progress
\ No newline at end of file
+# only add -E if not on centos 6
+sudo_path="$(command -v sudo)"
+# cspell:disable-next-line
+rhel_sudo="/opt/rh/devtoolset-7/root/usr/bin/sudo"
+sudo_args=""
+if [[ "$sudo_path" != "$rhel_sudo" ]]; then
+  echo "HERE"
+  sudo -E bundle install --jobs=3 --retry=3
+  sudo -E bundle exec rspec --profile -f progress
+else
+  sudo bundle install --jobs=3 --retry=3
+  sudo bundle exec rspec --profile -f progress
+fi
diff --git a/omnibus_overrides.rb b/omnibus_overrides.rb
deleted file mode 100644
index c94511af66..0000000000
--- a/omnibus_overrides.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# THIS IS NOW HAND MANAGED, JUST EDIT THE THING
-# keep it machine-parsable since CI uses it
-#
-# NOTE: You MUST update omnibus-software when adding new versions of
-# software here: bundle exec rake dependencies:update_omnibus_gemfile_lock
-override "libffi", version: "3.4.2"
-override "libiconv", version: "1.16"
-override "liblzma", version: "5.2.5"
-override "libtool", version: "2.4.2"
-
-# libxslt 1.1.35 does not build successfully with libxml2 2.9.13 on Windows so we will pin
-# windows builds to libxslt 1.1.34 and libxml2 2.9.10 for now and followup later with the
-# work to fix that issue in IPACK-145.
-override "libxml2", version: windows? ? "2.9.10" : "2.9.13"
-override "libxslt", version: windows? ? "1.1.34" : "1.1.35"
-
-override "libyaml", version: "0.1.7"
-override "makedepend", version: "1.0.5"
-override "ncurses", version: "6.3"
-override "nokogiri", version: "1.13.6"
-override "openssl", version: mac_os_x? ? "1.1.1m" : "1.0.2zb"
-override "pkg-config-lite", version: "0.28-1"
-override :ruby, version: aix? ? "3.0.3" : "3.1.2"
-override "ruby-windows-devkit-bash", version: "3.1.23-4-msys-1.0.18"
-override "ruby-msys2-devkit", version: "3.1.2-1"
-override "util-macros", version: "1.19.0"
-override "xproto", version: "7.0.28"
-override "zlib", version: "1.2.11"