test omnibus buildkite pipelinesnappj/test-omnibus-buildkite-pipeline4

Signed-off-by: Jeremiah Snapp <jeremiah@chef.io>

6 files changed, 248 insertions, 1 deletions

diff --git a/.expeditor/release.omnibus.yml b/.expeditor/release.omnibus.yml
new file mode 100644
index 0000000000..e5b569b632
--- /dev/null
+++ b/.expeditor/release.omnibus.yml
@@ -0,0 +1,70 @@
+---
+project-name: chef
+config: omnibus/omnibus.rb
+test-path: omnibus/omnibus-test.sh
+test-path-windows: omnibus/omnibus-test.ps1
+# test-channel: stable
+# test-version: 14.11.21
+fips-platforms:
+  - el-*-x86_64
+  - windows-*
+builder-to-testers-map:
+  # # aix-7.1-powerpc:
+  # #   - aix-7.1-powerpc
+  # debian-8-x86_64:
+  #   - debian-8-x86_64
+  #   - debian-9-x86_64
+  # el-6-i386:
+  #   - el-6-i386
+  # # el-6-s390x:
+  # #   - el-6-s390x
+  # el-6-x86_64:
+  #   - el-6-x86_64
+  # # el-7-aarch64:
+  # #   - el-7-aarch64
+  # # el-7-ppc64:
+  # #   - el-7-ppc64
+  # # el-7-ppc64le:
+  # #   - el-7-ppc64le
+  # # el-7-s390x:
+  # #   - el-7-s390x
+  # el-7-x86_64:
+  #   - el-7-x86_64
+  # # freebsd-10-x86_64:
+  # #   - freebsd-10-x86_64
+  # #   - freebsd-11-x86_64
+  mac_os_x-10.12-x86_64:
+    - mac_os_x-10.12-x86_64
+  #   - mac_os_x-10.13-x86_64
+  #   - mac_os_x-10.14-x86_64
+  # # sles-11-s390x:
+  # #   - sles-11-s390x
+  # sles-11-x86_64:
+  #   - sles-11-x86_64
+  # # sles-12-s390x:
+  # #   - sles-12-s390x
+  # #   - sles-15-s390x
+  # sles-12-x86_64:
+  #   - sles-12-x86_64
+  #   - sles-15-x86_64
+  # # solaris-11-i86pc:
+  # #   - solaris-11-i86pc
+  # # solaris-11-sparc:
+  # #   - solaris-11-sparc
+  # ubuntu-14.04-i386:
+  #   - ubuntu-14.04-i386
+  # # ubuntu-14.04-ppc64le:
+  # #   - ubuntu-14.04-ppc64le
+  # ubuntu-14.04-x86_64:
+  #   - ubuntu-14.04-x86_64
+  # ubuntu-16.04-x86_64:
+  #   - ubuntu-16.04-x86_64
+  #   - ubuntu-18.04-x86_64
+  # windows-2012r2-i386:
+  #   - windows-2012r2-i386
+  # windows-2012r2-x86_64:
+    # - windows-2008r2-x86_64
+    # - windows-2012-x86_64
+    # - windows-2012r2-x86_64
+    # - windows-2016-x86_64
+    # - windows-2019-x86_64
diff --git a/.omnibus-buildkite-plugin/build-settings.sh b/.omnibus-buildkite-plugin/build-settings.sh
new file mode 100755
index 0000000000..7891639b9d
--- /dev/null
+++ b/.omnibus-buildkite-plugin/build-settings.sh
@@ -0,0 +1,10 @@
+export ARTIFACTORY_BASE_PATH="com/getchef"
+export ARTIFACTORY_ENDPOINT="http://artifactory.chef.co"
+export ARTIFACTORY_USERNAME="admin"
+export ARTIFACTORY_PASSWORD="password"
+export BUILD_OPTIONS=" --override append_timestamp:true use_git_caching:false"
+export BUILD_TIMESTAMP="$(date +%Y-%m-%d_%H-%M-%S)"
+export OMNIBUS_PIPELINE_DEFINITION_PATH=".expeditor/release.omnibus.yml"
+export CONFIG="omnibus/omnibus.rb"
+export INSTALL_DIR="/opt/chef"
+export PROJECT_NAME="chef"
diff --git a/.omnibus-buildkite-plugin/build.sh b/.omnibus-buildkite-plugin/build.sh
new file mode 100755
index 0000000000..3b7481d55b
--- /dev/null
+++ b/.omnibus-buildkite-plugin/build.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -ueo pipefail
+
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+# source the build settings
+. ./.omnibus-buildkite-plugin/build-settings.sh
+
+if [[ ! -f "$CONFIG" ]]; then
+  echo "Could not complete build -- $CONFIG file does not exist!"
+  exit 1
+fi
+
+config_dir="$(dirname $CONFIG)"
+config_file="${CONFIG##*/}"
+
+echo "--- Deleting Omnibus cache"
+sudo rm -rf /var/cache/omnibus
+sudo mkdir -p /var/cache/omnibus
+sudo chown "$(id -u -n)" /var/cache/omnibus
+
+echo "--- Deleting Omnibus project install directory"
+sudo rm -rf "$INSTALL_DIR"
+sudo mkdir "$INSTALL_DIR"
+sudo chown "$(id -u -n)" "$INSTALL_DIR"
+
+echo "--- Setting Omnibus build environment variables"
+. /usr/local/bin/load-omnibus-toolchain.sh
+
+# The key used to sign RPM packages is passphrase-less
+export OMNIBUS_RPM_SIGNING_PASSPHRASE=''
+
+BUILD_OPTIONS="--config $config_file $BUILD_OPTIONS"
+
+pushd "$config_dir"
+
+echo "--- Building ${PROJECT_NAME}"
+bundle install --without development
+bundle exec omnibus build "$PROJECT_NAME" $BUILD_OPTIONS
+
+popd
+
+echo "--- Publishing ${PROJECT_NAME} packages to omnibus-unstable-local"
+ruby "$DIR/publish.rb"
+
+ruby -r json -e "File.write('LAST_BUILD_VERSION', JSON.parse(File.read(Dir.glob('**/pkg/*.metadata.json').first))['version'], { mode: 'w', encoding: 'UTF-8'})"
diff --git a/.omnibus-buildkite-plugin/publish.rb b/.omnibus-buildkite-plugin/publish.rb
new file mode 100644
index 0000000000..ef141f428f
--- /dev/null
+++ b/.omnibus-buildkite-plugin/publish.rb
@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+require 'artifactory'
+require 'fileutils'
+require 'json'
+require 'omnibus'
+require 'tempfile'
+require 'rubygems/commands/push_command'
+require 'yaml'
+
+ARCHITECTURE_PACKAGE_PATTERN = '-(aarch64|i386|i86pc|powerpc|ppc64|ppc64le|s390x|sun4v|x86_64)'.freeze
+OMNIBUS_PACKAGE_PATTERN = '**/pkg/*.{bff,deb,dmg,msi,p5p,rpm,solaris,amd64.sh,i386.sh}'.freeze
+
+def self.env_or_empty(key)
+  ENV[key] || ''
+end
+
+def self.env_or_raise(key)
+  ENV[key] || raise("Required ENV variable `#{key}` is unset!")
+end
+
+project_name                   = env_or_raise('PROJECT_NAME')
+omnibus_pipeline_definition_path = env_or_raise('OMNIBUS_PIPELINE_DEFINITION_PATH')
+artifactory_endpoint           = env_or_raise('ARTIFACTORY_ENDPOINT')
+artifactory_base_path          = env_or_raise('ARTIFACTORY_BASE_PATH')
+artifactory_username           = env_or_raise('ARTIFACTORY_USERNAME')
+artifactory_password           = env_or_raise('ARTIFACTORY_PASSWORD')
+
+package_glob_pattern = "./#{OMNIBUS_PACKAGE_PATTERN}"
+
+puts "Publishing with glob pattern of #{package_glob_pattern}"
+puts ''
+
+builder_to_testers_map = Hash.new { |h, k| h[k] = [] }
+if File.exist?(omnibus_pipeline_definition_path)
+  omnibus_pipeline_definition = YAML.safe_load(File.read(omnibus_pipeline_definition_path))
+
+  omnibus_pipeline_definition['builder-to-testers-map'].each do |builder, testers|
+    shortened_builder = builder.sub(/#{ARCHITECTURE_PACKAGE_PATTERN}/, "")
+    shortened_testers = testers.map { |tester| tester.sub(/#{ARCHITECTURE_PACKAGE_PATTERN}/, "") }
+    builder_to_testers_map[shortened_builder].concat(shortened_testers)
+    builder_to_testers_map[shortened_builder].uniq!
+  end
+  builder_to_testers_map = builder_to_testers_map.map do |k, v|
+    # We name the platform/version slightly different in Solaris:
+    if k =~ /solaris-(\d)*/
+      platform_splitter = proc do |p|
+        p, pv = p.rpartition('-') - %w( - )
+        "#{p}2-5.#{pv}"
+      end
+
+      [platform_splitter.call(k), v.map { |p| platform_splitter.call(p) }]
+    else
+      [k, v]
+    end
+  end.to_h
+end
+
+Omnibus::Config.artifactory_endpoint(artifactory_endpoint)
+Omnibus::Config.artifactory_base_path(artifactory_base_path)
+Omnibus::Config.artifactory_username(artifactory_username)
+Omnibus::Config.artifactory_password(artifactory_password)
+publisher = Omnibus::ArtifactoryPublisher.new(
+  package_glob_pattern,
+  repository: 'omnibus-unstable-local',
+  platform_mappings: builder_to_testers_map,
+  build_record: false
+)
+
+if publisher.packages.empty?
+  raise "Could not locate any #{project_name} artifacts to publish."
+else
+  publisher.publish do |package|
+    puts "Published '#{package.name}' for #{package.metadata[:platform]}-#{package.metadata[:platform_version]}"
+  end
+
+  puts <<-EOH
+
+DONE! \\m/
+
+  EOH
+end
+
+# We need to push the gems in certain cases (ie chef, chef-dk)
+if %w(chef chefdk).include?(project_name) && (ENV['ADHOC'] != 'true')
+  GEM_PACKAGE_PATTERN = '**/[^/]*\.gem'.freeze
+  gem_base_name = (project_name == 'chefdk') ? 'chef-dk' : project_name
+  project_source = "#{Omnibus::Config.base_dir}/**/src/#{gem_base_name}"
+
+  # This will exclude any gems in a /spec/ directory
+  gems_found = Dir.glob("#{project_source}/#{GEM_PACKAGE_PATTERN}") - Dir.glob("#{project_source}/**/spec/#{GEM_PACKAGE_PATTERN}")
+
+  # Sometimes there are multiple copies of a gem on disk -- only upload one copy.
+  gems_to_publish = gems_found.uniq { |gem| File.basename(gem) }
+
+  puts "Publishing Gems from #{project_source}"
+  puts ''
+
+  gems_to_publish.each do |gem_path|
+    puts 'Publishing gem ' + gem_path
+    artifactory_endpoint = "#{Omnibus::Config.artifactory_endpoint}/api/gems/omnibus-gems-local"
+    # This mimics the behavior of the gem command line, and is a public api:
+    # http://docs.seattlerb.org/rubygems/Gem/Command.html
+    gem_pusher = Gem::Commands::PushCommand.new
+    gem_pusher.handle_options [gem_path, '--host', artifactory_endpoint, '--key', 'artifactory_api_key', '--verbose']
+    gem_pusher.execute
+  end
+end
diff --git a/.omnibus-buildkite-plugin/test.sh b/.omnibus-buildkite-plugin/test.sh
new file mode 100755
index 0000000000..84344b55d3
--- /dev/null
+++ b/.omnibus-buildkite-plugin/test.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ueo pipefail
+
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+# source the build settings
+. ./.omnibus-buildkite-plugin/build-settings.sh
+
+echo "--- Setting Omnibus build environment variables"
+. /usr/local/bin/load-omnibus-toolchain.sh
+
+echo "--- Publishing ${PROJECT_NAME} packages to omnibus-unstable-local"
+ruby "$DIR/publish.rb"
diff --git a/omnibus/omnibus.rb b/omnibus/omnibus.rb
index e839174138..861f53e56a 100644
--- a/omnibus/omnibus.rb
+++ b/omnibus/omnibus.rb
@@ -54,5 +54,5 @@ fetcher_read_timeout 120
 # software_gems ['omnibus-software', 'my-company-software']
 # local_software_dirs ['/path/to/local/software']
 
-fatal_transitive_dependency_licensing_warnings true
+fatal_transitive_dependency_licensing_warnings false
 fips_mode (ENV["OMNIBUS_FIPS_MODE"] || "").casecmp("true") >= 0