update verify pipeline and omnibus build/test to use containers (#13489)

* update verify pipeline and omnibus build/test to use containers
* add execution permission on scripts
* add fips back in for opensuse but skip for windows
* jesseprieur/BS-159 - Add RPM Signing Key for RHEL/SLES/CentOS
* evanahlberg/BS-152 - Add MSI Signing to Windows Builds
* fix label on windows platforms
* fix role and add check for organization for aws credentials
* chmod omnibus test script and fix windows 2019 queue for omnibus test
* fix windows queue
* Remove dependencies that are part of chef-foundation.
* Disable s3 caching.
* Rename chef-gem as it conflicts with another omnibus-software.
* Speeding up debugging; Removing more-ruby-cleanup gem from omnibus chef package
* Adding more-ruby-cleanup back in
* Removing more-ruby-cleanup; Removing debugging steps
* Remove ruby dependency from more-ruby-cleanup.
* add retry and timeouts to all steps
* move omnibus test and build to own file and create ad hoc pipeline
* Adding in changes to use heredoc in verify pipeline
* Adding in retries/timeouts; Fixing missing agent
* allow for filtering of the omnibus build and test platforms
* only block on chef-oss org
* add canary adhoc pipeline and add back trigger for omnibus release
* Make omnibus s3 caching configurable via env.

Signed-off-by: Gregory Schofield <grschofi@progress.com>

Signed-off-by: Evan Ahlberg <evanahlberg@gmail.com>

Signed-off-by: Jesse Prieur <jesse.prieur@gmail.com>

1 files changed, 112 insertions, 0 deletions

diff --git a/.buildkite/build-test-omnibus.sh b/.buildkite/build-test-omnibus.sh
new file mode 100755
index 0000000000..79b18165e8
--- /dev/null
+++ b/.buildkite/build-test-omnibus.sh
@@ -0,0 +1,112 @@
+if [[ $BUILDKITE_ORGANIZATION_SLUG == "chef-oss" ]]; then
+  echo "- block: Build & Test Omnibus Packages"
+  echo "  prompt: Continue to run omnibus package build and tests for applicable platforms?"
+fi
+
+FILTER="${OMNIBUS_FILTER:=*}"
+
+platforms=("amazon-2:centos-7" "centos-6:centos-6" "centos-7:centos-7" "centos-8:centos-8" "rhel-9:rhel-9" "debian-9:debian-9" "debian-10:debian-9" "debian-11:debian-9" "ubuntu-1604:ubuntu-1604" "ubuntu-1804:ubuntu-1604" "ubuntu-2004:ubuntu-1604" "ubuntu-2204:ubuntu-1604" "sles-15:sles-15" "windows-2019:windows-2019")
+
+omnibus_build_platforms=()
+omnibus_test_platforms=()
+
+# build build array and test array based on filter
+for platform in ${platforms[@]}; do
+    case ${platform%:*} in
+        $FILTER)
+            omnibus_build_platforms[${#omnibus_build_platforms[@]}]=${platform#*:}
+            omnibus_test_platforms[${#omnibus_test_platforms[@]}]=$platform
+            ;;
+    esac
+done
+
+# remove duplicates from build array
+omnibus_build_platforms=($(printf "%s\n" "${omnibus_build_platforms[@]}" | sort -u | tr '\n' ' '))
+
+for platform in ${omnibus_build_platforms[@]}; do
+  if [[ $platform != *"windows"* ]]; then
+    echo "- label: \":hammer_and_wrench::docker: $platform\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  key: build-$platform"
+    echo "  agents:"
+    echo "    queue: default-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      privileged: true"
+    echo "      propagate-environment: true"
+    echo "      environment:"
+    echo "        - RPM_SIGNING_KEY"
+    echo "        - CHEF_FOUNDATION_VERSION"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/omnibus_chef_build.sh"
+    echo "  timeout_in_minutes: 60"
+  else 
+    echo "- label: \":hammer_and_wrench::windows: $platform\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  key: build-$platform"
+    echo "  agents:"
+    echo "    queue: default-$platform-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-$platform:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      shell:"
+    echo "      - powershell"
+    echo "      - \"-Command\""
+    echo "      propagate-environment: true"
+    echo "      environment:"
+    echo "        - CHEF_FOUNDATION_VERSION"
+    echo "        - BUILDKITE_AGENT_ACCESS_TOKEN"
+    echo "        - AWS_ACCESS_KEY_ID"
+    echo "        - AWS_SECRET_ACCESS_KEY"
+    echo "        - AWS_SESSION_TOKEN"
+    echo "      volumes:"
+    echo '        - "c:\\buildkite-agent:c:\\buildkite-agent"'
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/omnibus_chef_build.ps1"
+    echo "  timeout_in_minutes: 60"
+  fi
+done
+
+echo "- wait: ~"
+
+for platform in ${omnibus_test_platforms[@]}; do
+  if [[ $platform != *"windows"* ]]; then
+    echo "- env:"
+    echo "    OMNIBUS_BUILDER_KEY: build-${platform#*:}"
+    echo "  label: \":mag::docker: ${platform%:*}\""
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  agents:"
+    echo "    queue: default-privileged"
+    echo "  plugins:"
+    echo "  - docker#v3.5.0:"
+    echo "      image: chefes/omnibus-toolchain-${platform%:*}:$OMNIBUS_TOOLCHAIN_VERSION"
+    echo "      privileged: true"
+    echo "      propagate-environment: true"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.sh"
+    echo "    - omnibus/omnibus-test.sh"
+    echo "  timeout_in_minutes: 60"
+  else
+    echo "- env:"
+    echo "    OMNIBUS_BUILDER_KEY: build-windows-2019"
+    echo "  key: test-windows-2019"
+    echo '  label: ":mag::windows: windows-2019"'
+    echo "  retry:"
+    echo "    automatic:"
+    echo "      limit: 1"
+    echo "  agents:"
+    echo "    queue: default-windows-2019-privileged"
+    echo "  commands:"
+    echo "    - ./.expeditor/scripts/download_built_omnibus_pkgs.ps1"
+    echo "    - ./omnibus/omnibus-test.ps1"
+    echo "  timeout_in_minutes: 60"
+  fi
+done
+