From b12910007eaca937f4e6169c00afefba0d1ebbf1 Mon Sep 17 00:00:00 2001 From: Evan Ahlberg Date: Mon, 23 Jan 2023 13:10:10 -0500 Subject: update verify pipeline and omnibus build/test to use containers (#13489) * update verify pipeline and omnibus build/test to use containers * add execution permission on scripts * add fips back in for opensuse but skip for windows * jesseprieur/BS-159 - Add RPM Signing Key for RHEL/SLES/CentOS * evanahlberg/BS-152 - Add MSI Signing to Windows Builds * fix label on windows platforms * fix role and add check for organization for aws credentials * chmod omnibus test script and fix windows 2019 queue for omnibus test * fix windows queue * Remove dependencies that are part of chef-foundation. * Disable s3 caching. * Rename chef-gem as it conflicts with another omnibus-software. * Speeding up debugging; Removing more-ruby-cleanup gem from omnibus chef package * Adding more-ruby-cleanup back in * Removing more-ruby-cleanup; Removing debugging steps * Remove ruby dependency from more-ruby-cleanup. * add retry and timeouts to all steps * move omnibus test and build to own file and create ad hoc pipeline * Adding in changes to use heredoc in verify pipeline * Adding in retries/timeouts; Fixing missing agent * allow for filtering of the omnibus build and test platforms * only block on chef-oss org * add canary adhoc pipeline and add back trigger for omnibus release * Make omnibus s3 caching configurable via env. Signed-off-by: Gregory Schofield Signed-off-by: Evan Ahlberg Signed-off-by: Jesse Prieur --- omnibus/config/projects/chef.rb | 26 +----- omnibus/config/software/chef-foundation.rb | 21 +++++ omnibus/config/software/chef-local-source.rb | 115 +++++++++++++++++++++++++++ omnibus/config/software/more-ruby-cleanup.rb | 4 +- omnibus/omnibus-test.sh | 20 ++++- omnibus/omnibus.rb | 2 +- 6 files changed, 158 insertions(+), 30 deletions(-) create mode 100644 omnibus/config/software/chef-foundation.rb create mode 100644 omnibus/config/software/chef-local-source.rb mode change 100644 => 100755 omnibus/omnibus-test.sh (limited to 'omnibus') diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb index 217170ffef..2cb807f887 100644 --- a/omnibus/config/projects/chef.rb +++ b/omnibus/config/projects/chef.rb @@ -45,28 +45,8 @@ override :chef, version: "local_source" overrides_path = File.expand_path("../../../../omnibus_overrides.rb", current_file) instance_eval(IO.read(overrides_path), overrides_path) -dependency "preparation" +dependency "chef-local-source" -dependency "chef" - -# -# addons which require omnibus software defns (not direct deps of chef itself - RFC-063) -# -dependency "nokogiri" # (nokogiri cannot go in the Gemfile, see wall of text in the software defn) - -# FIXME?: might make sense to move dependencies below into the omnibus-software chef -# definition or into a chef-complete definition added to omnibus-software. -dependency "gem-permissions" -dependency "shebang-cleanup" -dependency "version-manifest" -dependency "openssl-customization" - -# devkit needs to come dead last these days so we do not use it to compile any gems -dependency "ruby-msys2-devkit" if windows? - -dependency "ruby-cleanup" - -# further gem cleanup other projects might not yet want to use dependency "more-ruby-cleanup" package :rpm do @@ -94,7 +74,7 @@ package :msi do upgrade_code msi_upgrade_code wix_candle_extension "WixUtilExtension" wix_light_extension "WixUtilExtension" - signing_identity "13B510D1CF1B3467856A064F1BEA12D0884D2528", machine_store: true + signing_identity ENV.fetch("OMNIBUS_SIGNING_IDENTITY", "13B510D1CF1B3467856A064F1BEA12D0884D2528"), machine_store: true parameters ChefLogDllPath: windows_safe_path(gem_path("chef-[0-9]*-x64-mingw-ucrt/ext/win32-eventlog/chef-log.dll")), ProjectLocationDir: project_location_dir end @@ -104,4 +84,4 @@ package :appx do skip_packager true end -runtime_dependency "coreutils" if rhel? +runtime_dependency "coreutils" if rhel? \ No newline at end of file diff --git a/omnibus/config/software/chef-foundation.rb b/omnibus/config/software/chef-foundation.rb new file mode 100644 index 0000000000..7352d6c9e3 --- /dev/null +++ b/omnibus/config/software/chef-foundation.rb @@ -0,0 +1,21 @@ +name "chef-foundation" +license "Apache-2.0" +license_file "LICENSE" + +# Grab accompanying notice file. +# So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree. +license_file "NOTICE" + +skip_transitive_dependency_licensing true + +if windows? + source path: "c:/opscode/chef" +else + source path: "/opt/chef" +end + +relative_path "chef-foundation" + +build do + sync "#{project_dir}", "#{install_dir}" +end \ No newline at end of file diff --git a/omnibus/config/software/chef-local-source.rb b/omnibus/config/software/chef-local-source.rb new file mode 100644 index 0000000000..528354422d --- /dev/null +++ b/omnibus/config/software/chef-local-source.rb @@ -0,0 +1,115 @@ +# +# Copyright:: Copyright (c) Chef Software Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# expeditor/ignore: no version pinning + +name "chef" +default_version "main" + +license "Apache-2.0" +license_file "LICENSE" + +# Grab accompanying notice file. +# So that Open4/deep_merge/diff-lcs disclaimers are present in Omnibus LICENSES tree. +license_file "NOTICE" + +# For the specific super-special version "local_source", build the source from +# the local git checkout. This is what you'd want to occur by default if you +# just ran omnibus build locally. +version("local_source") do + source path: "#{project.files_path}/../..", + # Since we are using the local repo, we try to not copy any files + # that are generated in the process of bundle installing omnibus. + # If the install steps are well-behaved, this should not matter + # since we only perform bundle and gem installs from the + # omnibus cache source directory, but we do this regardless + # to maintain consistency between what a local build sees and + # what a github based build will see. + options: { exclude: [ "omnibus/vendor" ] } +end + +# For any version other than "local_source", fetch from github. +# This is the behavior the transitive omnibus software deps such as chef-dk +# expect. +if version != "local_source" + source git: "https://github.com/chef/chef.git" +end + +dependency "chef-foundation" + +relative_path "chef" + +build do + env = with_standard_compiler_flags(with_embedded_path) + + # The --without groups here MUST match groups in https://github.com/chef/chef/blob/main/Gemfile + excluded_groups = %w{docgen chefstyle} + excluded_groups << "ruby_prof" if aix? + excluded_groups << "ruby_shadow" if aix? + excluded_groups << "ed25519" if solaris2? + + # these are gems which are not shipped but which must be installed in the testers + bundle_excludes = excluded_groups + %w{development test} + + bundle "install --without #{bundle_excludes.join(" ")}", env: env + + ruby "post-bundle-install.rb", env: env + + # use the rake install task to build/install chef-config/chef-utils + command "rake install:local", env: env + + gemspec_name = if windows? + # Chef18 is built with ruby3.1 so platform name is changed. + RUBY_PLATFORM == "x64-mingw-ucrt" ? "chef-universal-mingw-ucrt.gemspec" : "chef-universal-mingw32.gemspec" + else + "chef.gemspec" + end + + # This step will build native components as needed - the event log dll is + # generated as part of this step. This is why we need devkit. + gem "build #{gemspec_name}", env: env + + # ensure we put the gems in the right place to get picked up by the publish scripts + delete "pkg" + mkdir "pkg" + copy "chef*.gem", "pkg" + + # Always deploy the powershell modules in the correct place. + if windows? + mkdir "#{install_dir}/modules/chef" + copy "distro/powershell/chef/*", "#{install_dir}/modules/chef" + end + + block do + # cspell:disable-next-line + appbundle "chef", lockdir: project_dir, gem: "inspec-core-bin", without: excluded_groups, env: env + # cspell:disable-next-line + appbundle "chef", lockdir: project_dir, gem: "chef-bin", without: excluded_groups, env: env + # cspell:disable-next-line + appbundle "chef", lockdir: project_dir, gem: "chef", without: excluded_groups, env: env + # cspell:disable-next-line + appbundle "chef", lockdir: project_dir, gem: "ohai", without: excluded_groups, env: env + end + + # The rubyzip gem ships with some test fixture data compressed in a format Apple's notarization service + # cannot understand. We need to delete that archive to pass notarization. + block "Delete test folder of rubyzip gem so downstream projects pass notarization" do + env["VISUAL"] = "echo" + %w{rubyzip}.each do |gem| + gem_install_dir = shellout!("#{install_dir}/embedded/bin/gem open #{gem}", env: env).stdout.chomp + remove_directory "#{gem_install_dir}/test" + end + end +end \ No newline at end of file diff --git a/omnibus/config/software/more-ruby-cleanup.rb b/omnibus/config/software/more-ruby-cleanup.rb index ba3c0d3fd4..7201da3860 100644 --- a/omnibus/config/software/more-ruby-cleanup.rb +++ b/omnibus/config/software/more-ruby-cleanup.rb @@ -24,8 +24,6 @@ license :project_license source path: "#{project.files_path}/#{name}" -dependency "ruby" - build do block "Removing console and setup binaries" do Dir.glob("#{install_dir}/embedded/lib/ruby/gems/*/gems/*/bin/{console,setup}").each do |f| @@ -141,4 +139,4 @@ build do end end end -end +end \ No newline at end of file diff --git a/omnibus/omnibus-test.sh b/omnibus/omnibus-test.sh old mode 100644 new mode 100755 index 47b21ae431..83c64ff2a6 --- a/omnibus/omnibus-test.sh +++ b/omnibus/omnibus-test.sh @@ -21,7 +21,9 @@ sudo rm -rf "$TMPDIR" mkdir -p "$TMPDIR" # Verify that we kill any orphaned test processes. Kill any orphaned rspec processes. -sudo kill -9 $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') || true +if [[ $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') ]]; then + sudo kill -9 $(ps ax | grep 'rspec' | grep -v grep | awk '{ print $1 }') || true +fi export PATH="/opt/chef/bin:$PATH" export BIN_DIR="/opt/chef/bin" @@ -117,5 +119,17 @@ fi export CHEF_LICENSE=accept-no-persist cd "$chef_gem" -sudo -E bundle install --jobs=3 --retry=3 -sudo -E bundle exec rspec --profile -f progress + +# only add -E if not on centos 6 +sudo_path="$(command -v sudo)" +# cspell:disable-next-line +rhel_sudo="/opt/rh/devtoolset-7/root/usr/bin/sudo" +sudo_args="" +if [[ "$sudo_path" != "$rhel_sudo" ]]; then + echo "HERE" + sudo -E bundle install --jobs=3 --retry=3 + sudo -E bundle exec rspec --profile -f progress +else + sudo bundle install --jobs=3 --retry=3 + sudo bundle exec rspec --profile -f progress +fi diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb index 99817f7caf..b29aea5a70 100644 --- a/omnibus/omnibus.rb +++ b/omnibus/omnibus.rb @@ -35,7 +35,7 @@ use_git_caching true # Enable S3 asset caching # ------------------------------ -use_s3_caching true +use_s3_caching ENV.fetch("OMNIBUS_USE_S3_CACHING", false) s3_access_key ENV["AWS_ACCESS_KEY_ID"] s3_secret_key ENV["AWS_SECRET_ACCESS_KEY"] s3_bucket "opscode-omnibus-cache" -- cgit v1.2.1