update verify pipeline and omnibus build/test to use containers (#13489)

* update verify pipeline and omnibus build/test to use containers
* add execution permission on scripts
* add fips back in for opensuse but skip for windows
* jesseprieur/BS-159 - Add RPM Signing Key for RHEL/SLES/CentOS
* evanahlberg/BS-152 - Add MSI Signing to Windows Builds
* fix label on windows platforms
* fix role and add check for organization for aws credentials
* chmod omnibus test script and fix windows 2019 queue for omnibus test
* fix windows queue
* Remove dependencies that are part of chef-foundation.
* Disable s3 caching.
* Rename chef-gem as it conflicts with another omnibus-software.
* Speeding up debugging; Removing more-ruby-cleanup gem from omnibus chef package
* Adding more-ruby-cleanup back in
* Removing more-ruby-cleanup; Removing debugging steps
* Remove ruby dependency from more-ruby-cleanup.
* add retry and timeouts to all steps
* move omnibus test and build to own file and create ad hoc pipeline
* Adding in changes to use heredoc in verify pipeline
* Adding in retries/timeouts; Fixing missing agent
* allow for filtering of the omnibus build and test platforms
* only block on chef-oss org
* add canary adhoc pipeline and add back trigger for omnibus release
* Make omnibus s3 caching configurable via env.

Signed-off-by: Gregory Schofield <grschofi@progress.com>

Signed-off-by: Evan Ahlberg <evanahlberg@gmail.com>

Signed-off-by: Jesse Prieur <jesse.prieur@gmail.com>

6 files changed, 158 insertions, 30 deletions

diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb
index 217170ffef..2cb807f887 100644
--- a/omnibus/config/projects/chef.rb
+++ b/omnibus/config/projects/chef.rb
@@ -45,28 +45,8 @@ override :chef, version: "local_source"
 overrides_path = File.expand_path("../../../../omnibus_overrides.rb", current_file)
 instance_eval(IO.read(overrides_path), overrides_path)
 
-dependency "preparation"
+dependency "chef-local-source"
 
-dependency "chef"
-
-#
-# addons which require omnibus software defns (not direct deps of chef itself - RFC-063)
-#
-dependency "nokogiri" # (nokogiri cannot go in the Gemfile, see wall of text in the software defn)
-
-# FIXME?: might make sense to move dependencies below into the omnibus-software chef
-#  definition or into a chef-complete definition added to omnibus-software.
-dependency "gem-permissions"
-dependency "shebang-cleanup"
-dependency "version-manifest"
-dependency "openssl-customization"
-
-# devkit needs to come dead last these days so we do not use it to compile any gems
-dependency "ruby-msys2-devkit" if windows?
-
-dependency "ruby-cleanup"
-
-# further gem cleanup other projects might not yet want to use
 dependency "more-ruby-cleanup"
 
 package :rpm do
@@ -94,7 +74,7 @@ package :msi do
   upgrade_code msi_upgrade_code
   wix_candle_extension "WixUtilExtension"
   wix_light_extension "WixUtilExtension"
-  signing_identity "13B510D1CF1B3467856A064F1BEA12D0884D2528", machine_store: true
+  signing_identity ENV.fetch("OMNIBUS_SIGNING_IDENTITY", "13B510D1CF1B3467856A064F1BEA12D0884D2528"), machine_store: true
   parameters ChefLogDllPath: windows_safe_path(gem_path("chef-[0-9]*-x64-mingw-ucrt/ext/win32-eventlog/chef-log.dll")),
              ProjectLocationDir: project_location_dir
 end
@@ -104,4 +84,4 @@ package :appx do
   skip_packager true
 end
 
-runtime_dependency "coreutils" if rhel?
+runtime_dependency "coreutils" if rhel?
\ No newline at end of file
diff --git a/omnibus/config/software/chef-foundation.rb b/omnibus/config/software/chef-foundation.rb
new file mode 100644
index 0000000000..7352d6c9e3
--- /dev/null
+++ b/omnibus/config/software/chef-foundation.rb
@@ -0,0 +1,21 @@
+name "chef-foundation"
+license "Apache-2.0"
+license_file "LICENSE"
+
+# Grab accompanying notice file.
+# So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree.
+license_file "NOTICE"
+
+skip_transitive_dependency_licensing true
+
+if windows?
+  source path: "c:/opscode/chef"
+else
+  source path: "/opt/chef"
+end
+
+relative_path "chef-foundation"
+
+build do
+  sync "#{project_dir}", "#{install_dir}"
+end
\ No newline at end of file
diff --git a/omnibus/config/software/chef-local-source.rb b/omnibus/config/software/chef-local-source.rb
new file mode 100644
index 0000000000..528354422d
--- /dev/null
+++ b/omnibus/config/software/chef-local-source.rb
@@ -0,0 +1,115 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# expeditor/ignore: no version pinning
+
+name "chef"
+default_version "main"
+
+license "Apache-2.0"
+license_file "LICENSE"
+
+# Grab accompanying notice file.
+# So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree.
+license_file "NOTICE"
+
+# For the specific super-special version "local_source", build the source from
+# the local git checkout. This is what you'd want to occur by default if you
+# just ran omnibus build locally.
+version("local_source") do
+  source path: "#{project.files_path}/../..",
+         # Since we are using the local repo, we try to not copy any files
+         # that are generated in the process of bundle installing omnibus.
+         # If the install steps are well-behaved, this should not matter
+         # since we only perform bundle and gem installs from the
+         # omnibus cache source directory, but we do this regardless
+         # to maintain consistency between what a local build sees and
+         # what a github based build will see.
+         options: { exclude: [ "omnibus/vendor" ] }
+end
+
+# For any version other than "local_source", fetch from github.
+# This is the behavior the transitive omnibus software deps such as chef-dk
+# expect.
+if version != "local_source"
+  source git: "https://github.com/chef/chef.git"
+end
+
+dependency "chef-foundation"
+
+relative_path "chef"
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+
+  # The --without groups here MUST match groups in https://github.com/chef/chef/blob/main/Gemfile
+  excluded_groups = %w{docgen chefstyle}
+  excluded_groups << "ruby_prof" if aix?
+  excluded_groups << "ruby_shadow" if aix?
+  excluded_groups << "ed25519" if solaris2?
+
+  # these are gems which are not shipped but which must be installed in the testers
+  bundle_excludes = excluded_groups + %w{development test}
+
+  bundle "install --without #{bundle_excludes.join(" ")}", env: env
+
+  ruby "post-bundle-install.rb", env: env
+
+  # use the rake install task to build/install chef-config/chef-utils
+  command "rake install:local", env: env
+
+  gemspec_name = if windows?
+                   # Chef18 is built with ruby3.1 so platform name is changed.
+                   RUBY_PLATFORM == "x64-mingw-ucrt" ? "chef-universal-mingw-ucrt.gemspec" : "chef-universal-mingw32.gemspec"
+                 else
+                   "chef.gemspec"
+                 end
+
+  # This step will build native components as needed - the event log dll is
+  # generated as part of this step.  This is why we need devkit.
+  gem "build #{gemspec_name}", env: env
+
+  # ensure we put the gems in the right place to get picked up by the publish scripts
+  delete "pkg"
+  mkdir "pkg"
+  copy "chef*.gem", "pkg"
+
+  # Always deploy the powershell modules in the correct place.
+  if windows?
+    mkdir "#{install_dir}/modules/chef"
+    copy "distro/powershell/chef/*", "#{install_dir}/modules/chef"
+  end
+
+  block do
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "inspec-core-bin", without: excluded_groups, env: env
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "chef-bin", without: excluded_groups, env: env
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "chef", without: excluded_groups, env: env
+    # cspell:disable-next-line
+    appbundle "chef", lockdir: project_dir, gem: "ohai", without: excluded_groups, env: env
+  end
+
+  # The rubyzip gem ships with some test fixture data compressed in a format Apple's notarization service
+  # cannot understand. We need to delete that archive to pass notarization.
+  block "Delete test folder of rubyzip gem so downstream projects pass notarization" do
+    env["VISUAL"] = "echo"
+    %w{rubyzip}.each do |gem|
+      gem_install_dir = shellout!("#{install_dir}/embedded/bin/gem open #{gem}", env: env).stdout.chomp
+      remove_directory "#{gem_install_dir}/test"
+    end
+  end
+end
\ No newline at end of file
diff --git a/omnibus/config/software/more-ruby-cleanup.rb b/omnibus/config/software/more-ruby-cleanup.rb
index ba3c0d3fd4..7201da3860 100644
--- a/omnibus/config/software/more-ruby-cleanup.rb
+++ b/omnibus/config/software/more-ruby-cleanup.rb
@@ -24,8 +24,6 @@ license :project_license
 
 source path: "#{project.files_path}/#{name}"
 
-dependency "ruby"
-
 build do
   block "Removing console and setup binaries" do
     Dir.glob("#{install_dir}/embedded/lib/ruby/gems/*/gems/*/bin/{console,setup}").each do |f|
@@ -141,4 +139,4 @@ build do
       end
     end
   end
-end
+end
\ No newline at end of file
diff --git a/omnibus/omnibus-test.sh b/omnibus/omnibus-test.sh
index 47b21ae431..83c64ff2a6 100644..100755
--- a/omnibus/omnibus-test.sh
+++ b/omnibus/omnibus-test.sh
@@ -21,7 +21,9 @@ sudo rm -rf "$TMPDIR"
 mkdir -p "$TMPDIR"
 
 # Verify that we kill any orphaned test processes. Kill any orphaned rspec processes.
-sudo kill -9 $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') || true
+if [[ $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') ]]; then
+  sudo kill -9 $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') || true
+fi
 
 export PATH="/opt/chef/bin:$PATH"
 export BIN_DIR="/opt/chef/bin"
@@ -117,5 +119,17 @@ fi
 export CHEF_LICENSE=accept-no-persist
 
 cd "$chef_gem"
-sudo -E bundle install --jobs=3 --retry=3
-sudo -E bundle exec rspec --profile -f progress
+
+# only add -E if not on centos 6
+sudo_path="$(command -v sudo)"
+# cspell:disable-next-line
+rhel_sudo="/opt/rh/devtoolset-7/root/usr/bin/sudo"
+sudo_args=""
+if [[ "$sudo_path" != "$rhel_sudo" ]]; then
+  echo "HERE"
+  sudo -E bundle install --jobs=3 --retry=3
+  sudo -E bundle exec rspec --profile -f progress
+else
+  sudo bundle install --jobs=3 --retry=3
+  sudo bundle exec rspec --profile -f progress
+fi
diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb
index 99817f7caf..b29aea5a70 100644
--- a/omnibus/omnibus.rb
+++ b/omnibus/omnibus.rb
@@ -35,7 +35,7 @@ use_git_caching true
 
 # Enable S3 asset caching
 # ------------------------------
-use_s3_caching true
+use_s3_caching ENV.fetch("OMNIBUS_USE_S3_CACHING", false)
 s3_access_key  ENV["AWS_ACCESS_KEY_ID"]
 s3_secret_key  ENV["AWS_SECRET_ACCESS_KEY"]
 s3_bucket      "opscode-omnibus-cache"