summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTim Smith <tsmith@chef.io>2020-07-13 15:34:07 -0700
committerGitHub <noreply@github.com>2020-07-13 15:34:07 -0700
commitf3da4e13064aa86f7d5c6f46864c25a6084b9f82 (patch)
tree3b26dd0a3731b33504e8b934eb9529aa3e44a35a
parent05bbf0c483eaa5da8e48eaf14a4230ca741d7a9d (diff)
parent3416ac6d7a0c83f69a38d10b5b001471f9e91a09 (diff)
downloadchef-f3da4e13064aa86f7d5c6f46864c25a6084b9f82.tar.gz
Merge pull request #10102 from chef/csnapp/9282
Test and Promote Habitat builds on Linux
-rw-r--r--.expeditor/config.yml9
-rw-r--r--.expeditor/habitat-test.pipeline.yml34
-rw-r--r--.expeditor/verify.habitat.pipeline.yml20
-rw-r--r--.expeditor/verify.pipeline.yml32
-rw-r--r--habitat/plan.sh137
-rw-r--r--habitat/tests/spec.ps12
-rwxr-xr-xhabitat/tests/test.sh35
-rwxr-xr-xscripts/ci/install-hab.sh23
-rwxr-xr-xscripts/ci/verify-plan.sh37
-rw-r--r--spec/functional/shell_spec.rb10
10 files changed, 232 insertions, 107 deletions
diff --git a/.expeditor/config.yml b/.expeditor/config.yml
index 090a31808a..ea7fba193e 100644
--- a/.expeditor/config.yml
+++ b/.expeditor/config.yml
@@ -26,14 +26,6 @@ docker_images:
pipelines:
- verify:
public: true
- - verify/habitat:
- description: >
- A temporary pipeline to verify the habitat plans. Performs a local build,
- tests it, then throws it away. Its steps should eventually move into the
- "verify" pipeline (if fast enough) or this pipeline should be removed in
- favor of learning about packaging problems after merge.
- definition: .expeditor/verify.habitat.pipeline.yml
- trigger: default
- habitat/build
- habitat/test:
definition: .expeditor/habitat-test.pipeline.yml
@@ -106,6 +98,7 @@ subscriptions:
- bash:.expeditor/update_dockerfile.sh
- built_in:tag_docker_image
- built_in:publish_rubygems
+ - built_in:promote_habitat_packages
- built_in:notify_chefio_slack_channels
# the habitat chain
diff --git a/.expeditor/habitat-test.pipeline.yml b/.expeditor/habitat-test.pipeline.yml
index 8a71df9c17..be35d66017 100644
--- a/.expeditor/habitat-test.pipeline.yml
+++ b/.expeditor/habitat-test.pipeline.yml
@@ -9,12 +9,36 @@ expeditor:
steps:
+- label: ":linux: Validate Linux"
+ commands:
+ - sudo ./scripts/ci/install-hab.sh x86_64-linux
+ - 'echo "--- :hammer_and_wrench: Installing $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUX"'
+ - sudo hab pkg install $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUX
+ - sudo ./habitat/tests/test.sh $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUX
+ expeditor:
+ executor:
+ linux:
+ privileged: true
+ single-use: true
+
+- label: ":linux: Validate Linux (kernel2)"
+ commands:
+ - sudo ./scripts/ci/install-hab.sh x86_64-linux-kernel2
+ - 'echo "--- :hammer_and_wrench: Installing $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUXKERNEL2"'
+ - sudo hab pkg install $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUXKERNEL2
+ - sudo ./habitat/tests/test.sh $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUXKERNEL2
+ expeditor:
+ executor:
+ linux:
+ privileged: true
+ single-use: true
+
- label: ":windows: Validate Habitat Builds of Chef Infra"
commands:
- - powershell -File "./scripts/ci/ensure-minimum-viable-hab.ps1"
+ - powershell -File ./scripts/ci/ensure-minimum-viable-hab.ps1
- 'Write-Host "--- :hammer_and_wrench: Installing $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS"'
- - hab pkg install "$EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS"
- - powershell -File "./habitat/tests/test.ps1" -PackageIdentifier "$EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS"
+ - hab pkg install $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS
+ - powershell -File ./habitat/tests/test.ps1" -PackageIdentifier $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS
expeditor:
executor:
windows:
@@ -26,7 +50,9 @@ steps:
- label: ":habicat: Promoting packages to the current channel."
commands:
- - hab pkg promote "$EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS" current x86_64-windows
+ - hab pkg promote $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUX current x86_64-linux
+ - hab pkg promote $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64LINUXKERNEL2 current x86_64-linux-kernel2
+ - hab pkg promote $EXPEDITOR_PKG_IDENTS_CHEFINFRACLIENTX86_64WINDOWS current x86_64-windows
expeditor:
executor:
docker:
diff --git a/.expeditor/verify.habitat.pipeline.yml b/.expeditor/verify.habitat.pipeline.yml
deleted file mode 100644
index 9c50312db5..0000000000
--- a/.expeditor/verify.habitat.pipeline.yml
+++ /dev/null
@@ -1,20 +0,0 @@
----
-expeditor:
- defaults:
- buildkite:
- timeout_in_minutes: 60
- retry:
- automatic:
- limit: 1
-
-steps:
-
-- label: "Windows plan"
- commands:
- - scripts/ci/verify-plan.ps1
- expeditor:
- executor:
- windows:
- privileged: true
- single-use: true
- shell: ["powershell", "-Command"]
diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml
index 987980aa3a..34a2dee47d 100644
--- a/.expeditor/verify.pipeline.yml
+++ b/.expeditor/verify.pipeline.yml
@@ -579,3 +579,35 @@ steps:
expeditor:
executor:
docker:
+
+- label: ":habicat: Linux plan"
+ commands:
+ - sudo ./scripts/ci/install-hab.sh 'x86_64-linux'
+ - sudo ./scripts/ci/verify-plan.sh
+ timeout_in_minutes: 60
+ expeditor:
+ executor:
+ linux:
+ privileged: true
+ single-use: true
+
+- label: ":habicat: Linux plan (kernel2)"
+ commands:
+ - sudo ./scripts/ci/install-hab.sh 'x86_64-linux-kernel2'
+ - sudo ./scripts/ci/verify-plan.sh
+ timeout_in_minutes: 60
+ expeditor:
+ executor:
+ linux:
+ privileged: true
+ single-use: true
+
+- label: ":habicat: Windows plan"
+ commands:
+ - ./scripts/ci/verify-plan.ps1
+ expeditor:
+ executor:
+ windows:
+ privileged: true
+ single-use: true
+ shell: ["powershell", "-Command"]
diff --git a/habitat/plan.sh b/habitat/plan.sh
index 6e113e8a95..967cdd334e 100644
--- a/habitat/plan.sh
+++ b/habitat/plan.sh
@@ -1,10 +1,13 @@
-pkg_name=chef-infra-client
-pkg_origin=chef
+_chef_client_ruby="core/ruby27"
+pkg_name="chef-infra-client"
+pkg_origin="chef"
pkg_maintainer="The Chef Maintainers <humans@chef.io>"
pkg_description="The Chef Infra Client"
pkg_license=('Apache-2.0')
-pkg_bin_dirs=(bin)
-_chef_client_ruby="core/ruby27"
+pkg_bin_dirs=(
+ bin
+ vendor/bin
+)
pkg_build_deps=(
core/make
core/gcc
@@ -52,10 +55,34 @@ do_verify() {
return 0
}
+do_setup_environment() {
+ push_runtime_env GEM_PATH "${pkg_prefix}/vendor"
+
+ set_runtime_env APPBUNDLER_ALLOW_RVM "true" # prevent appbundler from clearing out the carefully constructed runtime GEM_PATH
+ set_runtime_env SSL_CERT_FILE "$(pkg_path_for cacerts)/ssl/cert.pem"
+ set_runtime_env LANG "en_US.UTF-8"
+ set_runtime_env LC_CTYPE "en_US.UTF-8"
+}
+
do_prepare() {
- export OPENSSL_LIB_DIR=$(pkg_path_for openssl)/lib
- export OPENSSL_INCLUDE_DIR=$(pkg_path_for openssl)/include
- export SSL_CERT_FILE=$(pkg_path_for cacerts)/ssl/cert.pem
+ export GEM_HOME="${pkg_prefix}/vendor"
+ export OPENSSL_LIB_DIR="$(pkg_path_for openssl)/lib"
+ export OPENSSL_INCLUDE_DIR="$(pkg_path_for openssl)/include"
+ export SSL_CERT_FILE="$(pkg_path_for cacerts)/ssl/cert.pem"
+ export CPPFLAGS="${CPPFLAGS} ${CFLAGS}"
+
+ ( cd "$CACHE_PATH"
+ bundle config --local build.nokogiri "--use-system-libraries \
+ --with-zlib-dir=$(pkg_path_for zlib) \
+ --with-xslt-dir=$(pkg_path_for libxslt) \
+ --with-xml2-include=$(pkg_path_for libxml2)/include/libxml2 \
+ --with-xml2-lib=$(pkg_path_for libxml2)/lib"
+ bundle config --local jobs "$(nproc)"
+ bundle config --local without server docgen maintenance pry travis integration ci chefstyle
+ bundle config --local shebang "$(pkg_path_for "$_chef_client_ruby")/bin/ruby"
+ bundle config --local retry 5
+ bundle config --local silence_root_warning 1
+ )
build_line "Setting link for /usr/bin/env to 'coreutils'"
if [ ! -f /usr/bin/env ]; then
@@ -64,58 +91,46 @@ do_prepare() {
}
do_build() {
- local _libxml2_dir
- local _libxslt_dir
- local _zlib_dir
- export CPPFLAGS
- export GEM_HOME
- export GEM_PATH
- export NOKOGIRI_CONFIG
-
- _libxml2_dir=$(pkg_path_for libxml2)
- _libxslt_dir=$(pkg_path_for libxslt)
- _zlib_dir=$(pkg_path_for zlib)
-
- CPPFLAGS="${CPPFLAGS} ${CFLAGS}"
- GEM_HOME=${pkg_prefix}/bundle
- GEM_PATH=${GEM_HOME}
- NOKOGIRI_CONFIG="--use-system-libraries \
- --with-zlib-dir=${_zlib_dir} \
- --with-xslt-dir=${_libxslt_dir} \
- --with-xml2-include=${_libxml2_dir}/include/libxml2 \
- --with-xml2-lib=${_libxml2_dir}/lib"
-
- build_line "Executing bundle install inside hab-cache path. ($CACHE_PATH/chef-config)"
- ( cd "$CACHE_PATH/chef-config" || exit_with "unable to enter hab-cache directory" 1
- bundle config --local build.nokogiri "${NOKOGIRI_CONFIG}"
- bundle config --local silence_root_warning 1
- _bundle_install "${pkg_prefix}/bundle"
+ ( cd "$CACHE_PATH" || exit_with "unable to enter hab-cache directory" 1
+ build_line "Installing gem dependencies ..."
+ bundle install
+ build_line "Installing this project's gems ..."
+ bundle exec rake install
+ for gem in $GEM_HOME/bundler/gems/*; do
+ ( cd $gem
+ build_line "Installing gems from git repos properly ..."
+ rake install
+ )
+ done
)
-
- build_line "Executing bundle install inside source path. ($SRC_PATH)"
- _bundle_install "${pkg_prefix}/bundle"
}
do_install() {
- build_line "Copying directories from source to pkg_prefix"
- mkdir -p "${pkg_prefix}/chef"
- for dir in bin chef-bin chef-config chef-utils lib chef.gemspec Gemfile Gemfile.lock; do
- cp -rv "${SRC_PATH}/${dir}" "${pkg_prefix}/chef/"
- done
-
- # If we generated them on install, bundler thinks our source is in $HAB_CACHE_SOURCE_PATH
- build_line "Generating binstubs with the correct path"
- ( cd "$pkg_prefix/chef" || exit_with "unable to enter pkg prefix directory" 1
- _bundle_install \
- "${pkg_prefix}/bundle" \
- --local \
- --quiet \
- --binstubs "${pkg_prefix}/bin"
+ ( cd "$pkg_prefix" || exit_with "unable to enter pkg prefix directory" 1
+ export BUNDLE_GEMFILE="${CACHE_PATH}/Gemfile"
+ build_line "** fixing binstub shebangs"
+ fix_interpreter "${pkg_prefix}/vendor/bin/*" "$_chef_client_ruby" bin/ruby
+ export BUNDLE_GEMFILE="${CACHE_PATH}/Gemfile"
+ for gem in chef-bin chef inspec-core-bin ohai; do
+ build_line "** generating binstubs for $gem with precise version pins"
+ appbundler $CACHE_PATH $pkg_prefix/bin $gem
+ done
)
+}
- build_line "Fixing bin/ruby and bin/env interpreters"
- fix_interpreter "${pkg_prefix}/bin/*" core/coreutils bin/env
- fix_interpreter "${pkg_prefix}/bin/*" "$_chef_client_ruby" bin/ruby
+do_after() {
+ build_line "Trimming the fat ..."
+
+ # We don't need the cache of downloaded .gem files ...
+ rm -r "$pkg_prefix/vendor/cache"
+ # ... or bundler's cache of git-ref'd gems
+ rm -r "$pkg_prefix/vendor/bundler"
+ # We don't need the gem docs.
+ rm -r "$pkg_prefix/vendor/doc"
+ # We don't need to ship the test suites for every gem dependency,
+ # only Chef's for package verification.
+ find "$pkg_prefix/vendor/gems" -name spec -type d | grep -v "chef-${pkg_version}" \
+ | while read spec_dir; do rm -r "$spec_dir"; done
}
do_end() {
@@ -128,19 +143,3 @@ do_end() {
do_strip() {
return 0
}
-
-# Helper function to wrap up some repetitive bundle install flags
-_bundle_install() {
- local path
- path="$1"
- shift
-
- bundle install ${*:-} \
- --jobs "$(nproc)" \
- --without development:test:docgen \
- --path "$path" \
- --shebang="$(pkg_path_for "$_chef_client_ruby")/bin/ruby" \
- --no-clean \
- --retry 5 \
- --standalone
-}
diff --git a/habitat/tests/spec.ps1 b/habitat/tests/spec.ps1
index 0229ac487f..6cbdd406c2 100644
--- a/habitat/tests/spec.ps1
+++ b/habitat/tests/spec.ps1
@@ -18,7 +18,7 @@ try {
SETX GEM_PATH $($gemPath.Split("=")[1]) /m
hab pkg binlink --force $PackageIdentifier
- /hab/bin/rspec --format progress --tag ~executables --tag ~choco_installed spec/functional
+ /hab/bin/rspec --format documentation --tag ~executables --tag ~choco_installed spec/functional
if (-not $?) { throw "functional testing failed"}
} finally {
Pop-Location
diff --git a/habitat/tests/test.sh b/habitat/tests/test.sh
new file mode 100755
index 0000000000..0882313c8e
--- /dev/null
+++ b/habitat/tests/test.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export CHEF_LICENSE="accept-no-persist"
+export HAB_LICENSE="accept-no-persist"
+
+project_root="$(git rev-parse --show-toplevel)"
+pkg_ident="$1"
+
+# print error message followed by usage and exit
+error () {
+ local message="$1"
+
+ echo -e "\nERROR: ${message}\n" >&2
+
+ exit 1
+}
+
+[[ -n "$pkg_ident" ]] || error 'no hab package identity provided'
+
+package_version=$(awk -F / '{print $3}' <<<"$pkg_ident")
+
+cd "${project_root}"
+
+echo "--- :mag_right: Testing ${pkg_ident} executables"
+[[ "$package_version" = "$(hab pkg exec "${pkg_ident}" chef-client --version | sed 's/.*: //')" ]] || error "chef-client is not the expected version"
+
+for executable in 'chef-client' 'ohai' 'chef-shell' 'chef-apply' 'knife' 'chef-solo'; do
+ echo -en "\t$executable = "
+ hab pkg exec "${pkg_ident}" "${executable}" --version || error "${executable} failed to execute properly"
+done
+
+echo "--- :mag_right: Testing ${pkg_ident} functionality"
+hab pkg exec "${pkg_ident}" rspec -f documentation --tag ~executables spec/functional || error 'failures during rspec tests'
diff --git a/scripts/ci/install-hab.sh b/scripts/ci/install-hab.sh
new file mode 100755
index 0000000000..b65a89d706
--- /dev/null
+++ b/scripts/ci/install-hab.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export HAB_LICENSE="accept"
+
+hab_target="$1"
+
+# print error message followed by usage and exit
+error () {
+ local message="$1"
+
+ echo -e "\nERROR: ${message}\n" >&2
+
+ exit 1
+}
+
+[[ -n "$hab_target" ]] || error 'no hab target provided'
+
+echo "--- :habicat: Installing latest version of Habitat"
+rm -rf /hab
+curl https://raw.githubusercontent.com/habitat-sh/habitat/master/components/hab/install.sh | bash -s -- -t "$hab_target"
+hab license accept
diff --git a/scripts/ci/verify-plan.sh b/scripts/ci/verify-plan.sh
new file mode 100755
index 0000000000..71aac8d767
--- /dev/null
+++ b/scripts/ci/verify-plan.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export HAB_ORIGIN='ci'
+export PLAN='chef-infra-client'
+export CHEF_LICENSE="accept-no-persist"
+export HAB_LICENSE="accept-no-persist"
+
+# print error message followed by usage and exit
+error () {
+ local message="$1"
+
+ echo -e "\nERROR: ${message}\n" >&2
+
+ exit 1
+}
+
+echo "--- :8ball: :linux: Verifying $PLAN"
+project_root="$(git rev-parse --show-toplevel)"
+
+echo "--- :key: Generating fake origin key"
+hab origin key generate "$HAB_ORIGIN"
+
+echo "--- :construction: Building $PLAN (solely for verification testing)"
+(
+ cd "$project_root" || error 'cannot change directory to project root'
+ DO_CHECK=true hab pkg build . || error 'unable to build'
+)
+
+source "${project_root}/results/last_build.env" || error 'unable to determine details about this build'
+
+echo "--- :hammer_and_wrench: Installing $pkg_ident"
+hab pkg install "${project_root}/results/$pkg_artifact" || error 'unable to install this build'
+
+echo "--- :mag_right: Testing $PLAN"
+${project_root}/habitat/tests/test.sh "$pkg_ident" || error 'failures during test of executables'
diff --git a/spec/functional/shell_spec.rb b/spec/functional/shell_spec.rb
index 583440d2b6..2289df3ef1 100644
--- a/spec/functional/shell_spec.rb
+++ b/spec/functional/shell_spec.rb
@@ -97,13 +97,13 @@ describe Shell do
[output, e.status]
end
- it "boots correctly with -lauto" do
+ it "boots correctly with -lauto", :executables do
output, exitstatus = run_chef_shell_with("-lauto")
expect(output).to include("done")
expect(exitstatus).to eq(0)
end
- it "sets the log_level from the command line" do
+ it "sets the log_level from the command line", :executables do
output, exitstatus = run_chef_shell_with("-lfatal") do |out, keyboard|
show_log_level_code = %q[puts "===#{Chef::Log.level}==="]
keyboard.puts(show_log_level_code)
@@ -114,13 +114,13 @@ describe Shell do
end
context "on solo mode" do
- it "starts correctly" do
+ it "starts correctly", :executables do
output, exitstatus = run_chef_shell_with("--solo")
expect(output).to include("done")
expect(exitstatus).to eq(0)
end
- it "should be able to use the API" do
+ it "should be able to use the API", :executables do
output, exitstatus = run_chef_shell_with("-s") do |out, keyboard|
simple_api_get = "api.get('data')"
keyboard.puts(simple_api_get)
@@ -131,7 +131,7 @@ describe Shell do
end
end
- it "sets the override_runlist from the command line" do
+ it "sets the override_runlist from the command line", :executables do
output, exitstatus = run_chef_shell_with("-o 'override::foo,override::bar'") do |out, keyboard|
show_recipes_code = %q[puts "#{node["recipes"].inspect}"]
keyboard.puts(show_recipes_code)