diff options
Diffstat (limited to 'omnibus/jenkins')
| -rw-r--r-- | omnibus/jenkins/Gemfile | 5 | ||||
| -rw-r--r-- | omnibus/jenkins/Gemfile.lock | 70 | ||||
| -rwxr-xr-x | omnibus/jenkins/build | 122 | ||||
| -rw-r--r-- | omnibus/jenkins/build.bat | 36 | ||||
| -rw-r--r-- | omnibus/jenkins/chef-platform-names.json | 12 | ||||
| -rw-r--r-- | omnibus/jenkins/chef.json | 260 | ||||
| -rwxr-xr-x | omnibus/jenkins/client-test | 183 | ||||
| -rw-r--r-- | omnibus/jenkins/client-test.bat | 62 | ||||
| -rw-r--r-- | omnibus/jenkins/dna-solaris2.json | 10 | ||||
| -rw-r--r-- | omnibus/jenkins/dna-windows.json | 10 | ||||
| -rw-r--r-- | omnibus/jenkins/dna.json | 10 | ||||
| -rwxr-xr-x | omnibus/jenkins/install-test | 105 | ||||
| -rw-r--r-- | omnibus/jenkins/install-test.bat | 84 | ||||
| -rwxr-xr-x | omnibus/jenkins/release.rb | 728 | ||||
| -rw-r--r-- | omnibus/jenkins/solo.rb | 3 | ||||
| -rwxr-xr-x | omnibus/jenkins/verify-chef-container.sh | 22 | ||||
| -rwxr-xr-x | omnibus/jenkins/verify-chef.bat | 56 | ||||
| -rwxr-xr-x | omnibus/jenkins/verify-chef.sh | 89 | ||||
| -rwxr-xr-x | omnibus/jenkins/verify-chefdk.bat | 32 | ||||
| -rwxr-xr-x | omnibus/jenkins/verify-chefdk.sh | 22 |
20 files changed, 1921 insertions, 0 deletions
diff --git a/omnibus/jenkins/Gemfile b/omnibus/jenkins/Gemfile new file mode 100644 index 0000000000..24984dcda8 --- /dev/null +++ b/omnibus/jenkins/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +gem 'mixlib-shellout', '~> 1.4' +gem 'artifactory', '~> 2.1' +gem 'omnibus', '~> 3.0' diff --git a/omnibus/jenkins/Gemfile.lock b/omnibus/jenkins/Gemfile.lock new file mode 100644 index 0000000000..394fd0f4db --- /dev/null +++ b/omnibus/jenkins/Gemfile.lock @@ -0,0 +1,70 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.3.6) + arr-pm (0.0.9) + cabin (> 0) + artifactory (2.1.0) + backports (3.6.0) + cabin (0.6.1) + chef-sugar (1.3.0) + childprocess (0.5.3) + ffi (~> 1.0, >= 1.0.11) + clamp (0.6.3) + ffi (1.9.3) + ffi-yajl (1.0.2) + ffi (~> 1.5) + libyajl2 (~> 1.0) + fpm (0.4.42) + arr-pm (~> 0.0.8) + backports (>= 2.6.2) + cabin (>= 0.6.0) + childprocess + clamp (~> 0.6) + ftw (~> 0.0.30) + json (>= 1.7.7) + ftw (0.0.39) + addressable + backports (>= 2.6.2) + cabin (> 0) + http_parser.rb (= 0.5.3) + http_parser.rb (0.5.3) + ipaddress (0.8.0) + json (1.8.1) + libyajl2 (1.0.1) + mime-types (1.25.1) + mixlib-cli (1.5.0) + mixlib-config (2.1.0) + mixlib-log (1.6.0) + mixlib-shellout (1.4.0) + ohai (7.2.4) + ffi (~> 1.9) + ffi-yajl (~> 1.0) + ipaddress + mime-types (~> 1.16) + mixlib-cli + mixlib-config (~> 2.0) + mixlib-log + mixlib-shellout (~> 1.2) + systemu (~> 2.6.4) + wmi-lite (~> 1.0) + omnibus (3.2.1) + chef-sugar (~> 1.2) + fpm (~> 0.4) + mixlib-shellout (~> 1.4) + ohai (~> 7.2) + thor (~> 0.18) + uber-s3 + systemu (2.6.4) + thor (0.19.1) + uber-s3 (0.2.4) + mime-types (~> 1.17) + wmi-lite (1.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + artifactory (~> 2.1) + mixlib-shellout (~> 1.4) + omnibus (~> 3.0) diff --git a/omnibus/jenkins/build b/omnibus/jenkins/build new file mode 100755 index 0000000000..4df8bbe657 --- /dev/null +++ b/omnibus/jenkins/build @@ -0,0 +1,122 @@ +#!/bin/sh +# WARNING: REQUIRES /bin/sh +# +# - must run on /bin/sh on solaris 9 +# - must run on /bin/sh on AIX 6.x +# - if you think you are a bash wizard, you probably do not understand +# this programming language. do not touch. +# - if you are under 40, get peer review from your elders. +# +# Build you some jenkins +# + +set -e +set -x + +os=`uname -s` + +# Return truthy (which is zero) if a command does not exist +# (this is deliberately inverted because /bin/sh on Solaris does not support "if ! exists" syntax) +not_exists() { + if command -v $1 >/dev/null 2>&1; then + return 1 + else + return 0 + fi +} + +exists() { + if command -v $1 >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# The key used to sign RPM packages is passphrase-less +OMNIBUS_RPM_SIGNING_PASSPHRASE=notset +export OMNIBUS_RPM_SIGNING_PASSPHRASE + +if [ "x$os" = "xAIX" ]; then + # need to unset LIBPATH on AIX (like LD_LIBRARY_PATH on Solaris, Jenkins sets this (wrongly) on AIX) + unset LIBPATH +fi + +if [ -z $OMNIBUS_PROJECT_NAME ]; then + echo "OMNIBUS_PROJECT_NAME environment variable is not set!" + exit 1 +fi + +# create the build timestamp file for fingerprinting if it doesn't exist (manual build execution) +if [ ! -f build_timestamp ]; then + date > build_timestamp + echo "$BUILD_TAG / $BUILD_ID" > build_timestamp +fi + +PATH=/opt/ruby-2.1.2/bin:/opt/ruby1.9/bin:/usr/local/bin:$PATH +export PATH + +if [ "x$os" = "xAIX" ]; then + # AIX is hateful and requires a bunch of root stuff to build BFF packages + sudo rm -rf /.info || true + sudo mkdir /.info || true + sudo chown root /.info || true + sudo rm -rf /tmp/bff || true + # deinstall the bff if it got installed, can't build if it is installed + sudo installp -u $OMNIBUS_PROJECT_NAME || true + # AIX needs /opt/freeware/bin and /usr/sbin + if [ -d "/opt/freeware/bin" ]; then + PATH=/opt/freeware/bin:$PATH:/usr/sbin + export PATH + fi +fi + +# clean up our target directory +sudo rm -rf "/opt/${OMNIBUS_PROJECT_NAME}" || true +sudo mkdir -p "/opt/${OMNIBUS_PROJECT_NAME}" +# and any old package cruft from prior builds +sudo rm -f pkg/* || true + +if [ "$CLEAN" = "true" ]; then + # nuke everything, including the git cache + sudo rm -rf /var/cache/omnibus/* || true +else + # we need to nuke these from old builds in order to reliably use + # the git caching + sudo rm -rf /var/cache/omnibus/pkg/* || true + sudo rm -rf /var/cache/omnibus/src/* || true + sudo rm -f /var/cache/omnibus/build/*/*.manifest || true +fi + +# always fix up permissions +if [ "x$os" = "xAIX" ]; then + sudo chown -R root "/opt/${OMNIBUS_PROJECT_NAME}" + sudo chown -R root "/var/cache/omnibus" +else + sudo chown -R jenkins-node "/opt/${OMNIBUS_PROJECT_NAME}" || sudo chown -R jenkins "/opt/${OMNIBUS_PROJECT_NAME}" + sudo chown -R jenkins-node "/var/cache/omnibus" || sudo chown -R jenkins "/var/cache/omnibus" +fi + +# horrible hack for solaris 9 to get ffi to compile in the bundle +if [ -f "/etc/release" ]; then + # solaris /bin/sh needs the stupid || true or set -x bombs here + release=`cat /etc/release | grep 'Solaris 9' || true` + if [ "x$release" != "x" ]; then + # magic CONFIGURE_ARGS to get CFLAGS through bundle/gem install + CONFIGURE_ARGS="--with-cflags='-U__STRICT_ANSI__'" + export CONFIGURE_ARGS + fi +fi + +# docs do not install on solaris 9 +bundle install --without development + +if [ "$RELEASE_BUILD" = "true" ]; then + bundle exec omnibus build $OMNIBUS_PROJECT_NAME -l internal --override append_timestamp:false +else + bundle exec omnibus build $OMNIBUS_PROJECT_NAME -l internal +fi + +# Dump the build-generated version so the Omnitruck release script uses the +# correct version string format. +echo "`awk -v p=$OMNIBUS_PROJECT_NAME '$1 == p {print $2}' /opt/${OMNIBUS_PROJECT_NAME}/version-manifest.txt`" > pkg/BUILD_VERSION diff --git a/omnibus/jenkins/build.bat b/omnibus/jenkins/build.bat new file mode 100644 index 0000000000..bbcda327b7 --- /dev/null +++ b/omnibus/jenkins/build.bat @@ -0,0 +1,36 @@ +SETLOCAL + +ECHO %OMNIBUS_PROJECT_NAME% + +if "%OMNIBUS_PROJECT_NAME%"=="" ( + ECHO "OMNIBUS_PROJECT_NAME environment variable is not set!" + EXIT /B 1 +) + +IF "%CLEAN%"=="true" ( + rmdir /Q /S c:\opscode + rmdir /Q /S c:\omnibus-ruby + rmdir /Q /S .\pkg +) + +set PATH=C:\Ruby193\bin;%PATH% +set SSL_CERT_FILE=C:\Ruby193\ssl\certs\cacert.pem + +call bundle install --without development || GOTO :error + +IF "%RELEASE_BUILD%"=="true" ( + + call bundle exec omnibus build %OMNIBUS_PROJECT_NAME% -l internal --override append_timestamp:false || GOTO :error + +) ELSE ( + + call bundle exec omnibus build %OMNIBUS_PROJECT_NAME% -l internal || GOTO :error + +) + +GOTO :EOF + +:error +ECHO Failed with error level %errorlevel% + +ENDLOCAL diff --git a/omnibus/jenkins/chef-platform-names.json b/omnibus/jenkins/chef-platform-names.json new file mode 100644 index 0000000000..e63e5f0a0e --- /dev/null +++ b/omnibus/jenkins/chef-platform-names.json @@ -0,0 +1,12 @@ +{ + "aix" : "AIX", + "el" : "Enterprise Linux", + "debian" : "Debian", + "freebsd" : "FreeBSD", + "mac_os_x" : "OS X", + "ubuntu" : "Ubuntu", + "solaris2" : "Solaris", + "sles" : "SUSE Enterprise", + "suse" : "openSUSE", + "windows" : "Windows" +} diff --git a/omnibus/jenkins/chef.json b/omnibus/jenkins/chef.json new file mode 100644 index 0000000000..ed7b81862c --- /dev/null +++ b/omnibus/jenkins/chef.json @@ -0,0 +1,260 @@ +{ + "build_os=centos-5,machine_architecture=x64,role=oss-builder": [ + [ + "el", + "5", + "x86_64" + ] + ], + "build_os=centos-5,machine_architecture=x86,role=oss-builder": [ + [ + "el", + "5", + "i686" + ] + ], + "build_os=centos-6,machine_architecture=x64,role=oss-builder": [ + [ + "el", + "6", + "x86_64" + ], + [ + "el", + "7", + "x86_64" + ] + ], + "build_os=centos-6,machine_architecture=x86,role=oss-builder": [ + [ + "el", + "6", + "i686" + ] + ], + "build_os=debian-6,machine_architecture=x64,role=oss-builder": [ + [ + "debian", + "6", + "x86_64" + ], + [ + "debian", + "7", + "x86_64" + ] + ], + "build_os=debian-6,machine_architecture=x86,role=oss-builder": [ + [ + "debian", + "6", + "i686" + ], + [ + "debian", + "7", + "i686" + ] + ], + "build_os=mac_os_x_10_6,machine_architecture=x64,role=oss-builder": [ + [ + "mac_os_x", + "10.6", + "x86_64" + ] + ], + "build_os=mac_os_x_10_7,machine_architecture=x64,role=oss-builder": [ + [ + "mac_os_x", + "10.7", + "x86_64" + ], + [ + "mac_os_x", + "10.8", + "x86_64" + ], + [ + "mac_os_x", + "10.9", + "x86_64" + ], + [ + "mac_os_x", + "10.10", + "x86_64" + ] + ], + "build_os=solaris-10,machine_architecture=intel,role=oss-builder": [ + [ + "solaris2", + "5.10", + "i386" + ], + [ + "solaris2", + "5.11", + "i386" + ] + ], + "build_os=solaris-9,machine_architecture=sparc,role=oss-builder": [ + [ + "solaris2", + "5.9", + "sparc" + ], + [ + "solaris2", + "5.10", + "sparc" + ], + [ + "solaris2", + "5.11", + "sparc" + ] + ], + "build_os=ubuntu-10-04,machine_architecture=x64,role=oss-builder": [ + [ + "ubuntu", + "10.04", + "x86_64" + ], + [ + "ubuntu", + "10.10", + "x86_64" + ] + ], + "build_os=ubuntu-10-04,machine_architecture=x86,role=oss-builder": [ + [ + "ubuntu", + "10.04", + "i686" + ], + [ + "ubuntu", + "10.10", + "i686" + ] + ], + "build_os=ubuntu-12-04,machine_architecture=x64,role=oss-builder": [ + [ + "ubuntu", + "12.04", + "x86_64" + ], + [ + "ubuntu", + "12.10", + "x86_64" + ] + ], + "build_os=ubuntu-13-04,machine_architecture=x64,role=oss-builder": [ + [ + "ubuntu", + "13.04", + "x86_64" + ], + [ + "ubuntu", + "13.10", + "x86_64" + ], + [ + "ubuntu", + "14.04", + "x86_64" + ] + ], + "build_os=ubuntu-12-04,machine_architecture=x86,role=oss-builder": [ + [ + "ubuntu", + "12.04", + "i686" + ], + [ + "ubuntu", + "12.10", + "i686" + ] + ], + "build_os=ubuntu-13-04,machine_architecture=x86,role=oss-builder": [ + [ + "ubuntu", + "13.04", + "i686" + ], + [ + "ubuntu", + "13.10", + "i686" + ], + [ + "ubuntu", + "14.04", + "i686" + ] + ], + "build_os=freebsd-9-1,machine_architecture=x64,role=oss-builder": [ + [ + "freebsd", + "9", + "amd64" + ] + ], + "build_os=freebsd-9-1,machine_architecture=x86,role=oss-builder": [ + [ + "freebsd", + "9", + "i386" + ] + ], + "build_os=windows-2008r2,machine_architecture=x64,role=oss-builder": [ + [ + "windows", + "2008r2", + "x86_64" + ], + [ + "windows", + "2003r2", + "i686" + ], + [ + "windows", + "2003r2", + "x86_64" + ], + [ + "windows", + "2008", + "x86_64" + ], + [ + "windows", + "2008", + "i686" + ], + [ + "windows", + "2012", + "x86_64" + ], + [ + "windows", + "2012r2", + "x86_64" + ], + [ + "windows", + "7", + "x86_64" + ], + [ + "windows", + "8", + "x86_64" + ] + ] +} diff --git a/omnibus/jenkins/client-test b/omnibus/jenkins/client-test new file mode 100755 index 0000000000..b06485c06e --- /dev/null +++ b/omnibus/jenkins/client-test @@ -0,0 +1,183 @@ +#!/bin/sh +# WARNING: REQUIRES /bin/sh +# +# - must run on /bin/sh on solaris 9 +# - must run on /bin/sh on AIX 6.x +# - if you think you are a bash wizard, you probably do not understand +# this programming language. do not touch. +# - if you are under 40, get peer review from your elders. +# +# Test you some omnibus client +# +set -e +set -x + +# SmartOS builds in /opt/local instead +is_smartos() { + uname -v | grep "^joyent" 2>&1 >/dev/null +} + +if is_smartos; then + PREFIX="/opt/local" +else + PREFIX="/usr" +fi + +# copy off the timestamp for fingerprinting before we blow it away later +mv $BUILD_NUMBER/build_timestamp $WORKSPACE/ +cd $BUILD_NUMBER + +# determine install method via the package suffix +if test -n "`ls pkg/chef*.deb 2>/dev/null || true`"; then + package_file=`ls pkg/chef*.deb` + filetype="deb" +elif test -n "`ls pkg/chef*.rpm 2>/dev/null || true`"; then + package_file=`ls pkg/chef*.rpm` + filetype="rpm" +elif test -n "`ls pkg/chef*.solaris 2>/dev/null || true`"; then + package_file=`ls pkg/chef*.solaris` + filetype="solaris" +elif test -n "`ls pkg/chef*.bff 2>/dev/null || true`"; then + package_file=`ls pkg/chef*.bff` + filetype="bff" +elif test -n "`ls pkg/chef*.sh 2>/dev/null || true`"; then + package_file=`ls pkg/chef*.sh` + filetype="sh" +elif test -n "`ls pkg/chef*.pkg 2>/dev/null || true`"; then + package_file=`ls pkg/chef*.pkg` + filetype="pkg" +fi + +# remove the chef package / clobber the files +case "$filetype" in + "deb") + sudo dpkg -P chef || true + ;; + "rpm") + sudo rpm -ev chef || true + ;; + "solaris") + if sudo pkgrm -a /tmp/nocheck -n chef; then + : + else + echo "WARNING: CLEANING UP BROKEN POSTRM FROM PRIOR INSTALLATION!!!!!!!!!!!!!!!!!!!!!!" + sudo rm -f /var/sadm/pkg/chef/install/postremove || true + sudo pkgrm -a /tmp/nocheck -n chef || true + fi + ;; + "bff") + sudo installp -u chef || true + ;; + "pkg") + sudo rm -rf /opt/chef + sudo pkgutil --forget com.getchef.pkg.chef || true + ;; + "sh") + # intentionally left blank + ;; +esac + +sudo rm -rf /opt/chef/* + +# ensure symlinks are gone, so that failures to recreate them get caught +sudo rm -f $PREFIX/bin/chef-client || true +sudo rm -f $PREFIX/bin/chef-solo || true +sudo rm -f $PREFIX/bin/chef-apply || true +sudo rm -f $PREFIX/bin/chef-shell || true +sudo rm -f $PREFIX/bin/knife || true +sudo rm -f $PREFIX/bin/shef || true +sudo rm -f $PREFIX/bin/ohai || true + + +# install the new package +case "$filetype" in + "deb") sudo dpkg -i pkg/chef*.deb ;; + "rpm") sudo rpm -Uvh pkg/chef*.rpm ;; + "solaris") sudo pkgadd -n -d pkg/chef*.solaris -a /tmp/nocheck chef ;; + "bff") sudo installp -a -d pkg/chef*.bff chef;; + "sh") sudo ./pkg/chef*.sh ;; + "pkg") sudo /usr/sbin/installer -pkg pkg/chef*.pkg -target /;; +esac + +# sanity check that we're getting symlinks from the pre-install script +if [ ! -L "/usr/bin/chef-client" ]; then + echo "/usr/bin/chef-client symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -L "/usr/bin/knife" ]; then + echo "/usr/bin/knife symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -L "/usr/bin/chef-solo" ]; then + echo "/usr/bin/chef-solo symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -L "/usr/bin/ohai" ]; then + echo "/usr/bin/ohai symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -f "/opt/chef/bin/chef-client" ]; then + echo "/opt/chef/bin/chef-client binary was not installed correctly!" + exit 1 +fi + +if [ ! -f "/opt/chef/bin/knife" ]; then + echo "/opt/chef/bin/knife binary was not installed correctly!" + exit 1 +fi + +if [ ! -f "/opt/chef/bin/chef-solo" ]; then + echo "/opt/chef/bin/chef-solo binary was not installed correctly!" + exit 1 +fi + +if [ ! -f "/opt/chef/bin/ohai" ]; then + echo "/opt/chef/bin/ohai binary was not installed correctly!" + exit 1 +fi + +# test against the appbundle'd chef bundle +cd /opt/chef/embedded/apps/chef + +# test against the rspec and gems in the omnibus build +PATH=/opt/chef/bin:/opt/chef/embedded/bin:$PATH +export PATH + +# ffi-yajl must run in c-extension mode or we take perf hits, so we force it +# before running rspec so that we don't wind up testing the ffi mode +FORCE_FFI_YAJL=ext +export FORCE_FFI_YAJL + +# run against the Gemfile.lock in the appbundle'd directory +sudo env PATH=$PATH TERM=xterm bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o $WORKSPACE/test.xml -f documentation spec + +# clean up the workspace to save disk space +cd $WORKSPACE +rm -rf $BUILD_NUMBER + +# remove the package, break the build if the postrm fails +case "$filetype" in + "deb") + sudo dpkg -P chef + ;; + "rpm") + sudo rpm -ev chef + ;; + "solaris") + sudo pkgrm -a /tmp/nocheck -n chef + ;; + "bff") + sudo installp -u chef + ;; + "pkg") + sudo rm -rf /opt/chef + sudo pkgutil --forget com.getchef.pkg.chef || true + ;; + "sh") + # intentionally left blank + ;; +esac diff --git a/omnibus/jenkins/client-test.bat b/omnibus/jenkins/client-test.bat new file mode 100644 index 0000000000..31275b45d1 --- /dev/null +++ b/omnibus/jenkins/client-test.bat @@ -0,0 +1,62 @@ +SETLOCAL + +rem # copy off the timestamp for fingerprinting before we blow it away later +move %BUILD_NUMBER%\build_timestamp %WORKSPACE%\ + +rem # run the tests +cd %BUILD_NUMBER% + +rem # remove the chef package / clobber the files +rem # then install the new package +rmdir /S /Q C:\opscode +FOR %%i IN (pkg\chef*.msi) DO SET omnibus_package=%%i +SET omnibus_package=%WORKSPACE%\%BUILD_NUMBER%\%omnibus_package% + +call copy /Y "%omnibus_package%" %TMP%\install.msi || GOTO :error + +call msiexec INSTALLLOCATION=C:\opscode /qb /i %TMP%\install.msi || GOTO :error + +rem # use rspec and gems from omnibus +set PATH=C:\opscode\chef\bin;C:\opscode\chef\embedded\bin;%PATH% + +rem # test against the appbundle'd chef bundle +cd c:\opscode\chef\embedded\apps\chef + +rem # ffi-yajl must run in c-extension mode or we take perf hits, so we force it +rem # before running rspec so that we don't wind up testing the ffi mode +set FORCE_FFI_YAJL=ext + +rem # run the tests -- exclude spec/stress on windows +rem # we do not bundle exec here in order to test against the gems in the omnibus package +call bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o %WORKSPACE%\test.xml -f documentation spec/functional spec/unit || GOTO :error + +rem # check presence of essential binaries in correct places + +cd c:\opscode\chef\bin + +IF NOT EXIST chef-client GOTO :error +IF NOT EXIST chef-solo GOTO :error +IF NOT EXIST knife GOTO :error +IF NOT EXIST ohai GOTO :error + +rem # uninstall chef +call msiexec /qb /x %TMP%\install.msi || GOTO :error + +rem # clean up the workspace to save disk space +cd %WORKSPACE% +rmdir /S /Q %BUILD_NUMBER% + +GOTO :EOF + +:error + +SET ERR_LEV=%errorlevel% + +ECHO Failed with error level %ERR_LEV% + +rem # uninstall chef +call msiexec /qb /x %TMP%\install.msi + +EXIT /B %ERR_LEV% + +ENDLOCAL diff --git a/omnibus/jenkins/dna-solaris2.json b/omnibus/jenkins/dna-solaris2.json new file mode 100644 index 0000000000..0603540139 --- /dev/null +++ b/omnibus/jenkins/dna-solaris2.json @@ -0,0 +1,10 @@ +{ + "omnibus": { + "build_user": "jenkins", + "install_dir": "/opt/chef", + "cache_dir": "/var/cache/omnibus" + }, + "run_list": [ + "recipe[omnibus]" + ] +} diff --git a/omnibus/jenkins/dna-windows.json b/omnibus/jenkins/dna-windows.json new file mode 100644 index 0000000000..448a798a77 --- /dev/null +++ b/omnibus/jenkins/dna-windows.json @@ -0,0 +1,10 @@ +{ + "omnibus": { + "build_user": "Administrator", + "install_dir": "c:\\opscode\\chef", + "cache_dir": "c:\\omnibus-cache" + }, + "run_list": [ + "recipe[omnibus]" + ] +} diff --git a/omnibus/jenkins/dna.json b/omnibus/jenkins/dna.json new file mode 100644 index 0000000000..dd2ed456d1 --- /dev/null +++ b/omnibus/jenkins/dna.json @@ -0,0 +1,10 @@ +{ + "omnibus": { + "build_user": "jenkins-node", + "install_dir": "/opt/chef", + "cache_dir": "/var/cache/omnibus" + }, + "run_list": [ + "recipe[omnibus]" + ] +} diff --git a/omnibus/jenkins/install-test b/omnibus/jenkins/install-test new file mode 100755 index 0000000000..6b963ee915 --- /dev/null +++ b/omnibus/jenkins/install-test @@ -0,0 +1,105 @@ +#!/bin/sh +# WARNING: REQUIRES /bin/sh +# +# - must run on /bin/sh on solaris 9 +# - must run on /bin/sh on AIX 6.x +# - if you think you are a bash wizard, you probably do not understand +# this programming language. do not touch. +# - if you are under 40, get peer review from your elders. + +set -e +set -x + +env + +# sleep unti omnitruck has updated itself +sleep $SLEEP_TIME + +# Check whether a command exists - returns 0 if it does, 1 if it does not +exists() { + if command -v $1 >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +# SmartOS builds in /opt/local instead +is_smartos() { + uname -v | grep "^joyent" 2>&1 >/dev/null +} + +if is_smartos; then + PREFIX="/opt/local" +else + PREFIX="/usr" +fi + +# remove the chef package / clobber the files +if exists dpkg; then + sudo dpkg -P chef || true +elif exists rpm; then + sudo rpm -ev chef || true +elif exists pkgadd; then + cat <<EOF > /tmp/nocheck +conflict=nocheck +action=nocheck +EOF + if sudo pkgrm -a /tmp/nocheck -n chef; then + : + else + echo "WARNING: a 'no package to deinstall' error is normal here" + echo "WARNING: attempting to fix busted postremove anyway because I'm dumb..." + echo "WARNING: if this is really a busted postremove you should fix that..." + echo "WARNING: (but that should have been caught in the client-test script)" + sudo rm -f /var/sadm/pkg/chef/install/postremove || true + sudo pkgrm -a /tmp/nocheck -n chef || true + echo "WARNING: a 'no package to deinstall error' is normal here" + fi +else # makeself installer + : +fi + +sudo rm -rf /opt/chef/* + +# ensure symlinks are gone, so that failures to recreate them get caught +sudo rm -f $PREFIX/bin/chef-client || true +sudo rm -f $PREFIX/bin/chef-solo || true +sudo rm -f $PREFIX/bin/chef-apply || true +sudo rm -f $PREFIX/bin/chef-shell || true +sudo rm -f $PREFIX/bin/knife || true +sudo rm -f $PREFIX/bin/shef || true +sudo rm -f $PREFIX/bin/ohai || true + +if exists curl; then + curl -L "${OMNITRUCK_BASE_URL}/chef/install.sh" | sudo bash -s -- -v $INSTALL_CHEF_VERSION +else + wget -qO- "${OMNITRUCK_BASE_URL}/chef/install.sh" | sudo bash -s -- -v $INSTALL_CHEF_VERSION +fi + +# sanity check that we're getting symlinks from the pre-install script +if [ ! -L "/usr/bin/chef-client" ]; then + echo "/usr/bin/chef-client symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -L "/usr/bin/knife" ]; then + echo "/usr/bin/knife symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -L "/usr/bin/chef-solo" ]; then + echo "/usr/bin/chef-solo symlink was not installed by pre-install script!" + exit 1 +fi + +if [ ! -L "/usr/bin/ohai" ]; then + echo "/usr/bin/ohai symlink was not installed by pre-install script!" + exit 1 +fi + +# bundle bust and make sure we invoke chef-client from the installed artifact +unset GEM_HOME +unset GEM_PATH +/opt/chef/bin/chef-client --version + diff --git a/omnibus/jenkins/install-test.bat b/omnibus/jenkins/install-test.bat new file mode 100644 index 0000000000..70599c26cd --- /dev/null +++ b/omnibus/jenkins/install-test.bat @@ -0,0 +1,84 @@ +SETLOCAL + +> wget.vbs ( +echo.url = WScript.Arguments.Named^("url"^) +echo.path = WScript.Arguments.Named^("path"^) +echo.proxy = null +echo.Set objXMLHTTP = CreateObject^("MSXML2.ServerXMLHTTP"^) +echo.Set wshShell = CreateObject^( "WScript.Shell" ^) +echo.Set objUserVariables = wshShell.Environment^("USER"^) +echo. +echo.'http proxy is optional +echo.'attempt to read from HTTP_PROXY env var first +echo.On Error Resume Next +echo. +echo.If NOT ^(objUserVariables^("HTTP_PROXY"^) = ""^) Then +echo.proxy = objUserVariables^("HTTP_PROXY"^) +echo. +echo.'fall back to named arg +echo.ElseIf NOT ^(WScript.Arguments.Named^("proxy"^) = ""^) Then +echo.proxy = WScript.Arguments.Named^("proxy"^) +echo.End If +echo. +echo.If NOT isNull^(proxy^) Then +echo.'setProxy method is only available on ServerXMLHTTP 6.0+ +echo.Set objXMLHTTP = CreateObject^("MSXML2.ServerXMLHTTP.6.0"^) +echo.objXMLHTTP.setProxy 2, proxy +echo.End If +echo. +echo.On Error Goto 0 +echo. +echo.objXMLHTTP.open "GET", url, false +echo.objXMLHTTP.send^(^) +echo.If objXMLHTTP.Status = 200 Then +echo.Set objADOStream = CreateObject^("ADODB.Stream"^) +echo.objADOStream.Open +echo.objADOStream.Type = 1 +echo.objADOStream.Write objXMLHTTP.ResponseBody +echo.objADOStream.Position = 0 +echo.Set objFSO = Createobject^("Scripting.FileSystemObject"^) +echo.If objFSO.Fileexists^(path^) Then objFSO.DeleteFile path +echo.Set objFSO = Nothing +echo.objADOStream.SaveToFile path +echo.objADOStream.Close +echo.Set objADOStream = Nothing +echo.End if +echo.Set objXMLHTTP = Nothing +) + +rem # XXX: uninstall any left over version, ignore errors +call msiexec /qb /x %TMP%\install.msi + +rem # remove the chef package / clobber the files +rmdir /S /Q C:\opscode + +rem # remove the old installer, if it exists, ignore errors +del /F /Q %TMP%\install.msi + +rem # sleep until omnitruck has updated itself +powershell -command "start-sleep %SLEEP_TIME%" + +rem # download the new chef installer +rem # right now we have one build, fake the platform resulution crap +cscript /nologo wget.vbs /url:"http://%OMNITRUCK_BASE_URL%/chef/download?p=windows&pv=2008r2&m=x86_64&v=%INSTALL_CHEF_VERSION%" /path:%TMP%\install.msi + + +call msiexec INSTALLLOCATION=C:\opscode /qb /i %TMP%\install.msi || GOTO :error + +call C:\opscode\chef\bin\chef-client --version || GOTO :error + +call msiexec /qb /x %TMP%\install.msi || GOTO :error + +GOTO :EOF + +:error + +SET ERR_LEV=%errorlevel% + +ECHO Failed with error level %ERR_LEV% + +call msiexec /qb /x %TMP%\install.msi + +EXIT /B %ERR_LEV% + +ENDLOCAL diff --git a/omnibus/jenkins/release.rb b/omnibus/jenkins/release.rb new file mode 100755 index 0000000000..7459bc99bd --- /dev/null +++ b/omnibus/jenkins/release.rb @@ -0,0 +1,728 @@ +#!/usr/bin/env ruby + +## release.rb ################################################################# +#------------------------------------------------------------------------------ +# This script runs from the root of a jenkins workspace where artifacts from +# the omnibus build matrix are collected. +# +# # Primary command line options: +# * `--project PROJECT`: Project to be released. This also controls where the +# script looks for config JSON. +# * `--bucket S3_BUCKET`: Name of the S3 bucket where artifacts are released +# to. +# +# Other options are available, run `release.rb --help`. +# +# # Config +# release.rb looks in the same directory where it's located for files named +# "$project.json" and "$project-platform-names.json". +# +# ## $project.json +# The project.json file controls the mapping of build platforms to release +# platforms so that a single build artifact can be reused on compatible +# platforms. See chef.json for an example. +# +# ## $project-platform-names.json +# The project-platform-names.json file maps short platform names to long ones. +# see chef-platform-names.json for an example. +# +# # Tests +# This file contains the script's tests. Tests are written in rspec. To run the +# tests, run rspec with this file as the argument, e.g., +# `rspec -cfs release.rb`. + +require 'optparse' +require 'digest' +require 'rubygems' +require 'json' +require 'mixlib/shellout' + +# required for Artifactory publishing +require 'artifactory' +require 'omnibus' +require 'tempfile' + +# Represnts the collection of artifacts on disk that we plan to upload. Handles +# finding the artifacts and dealing with the mapping between build platform and +# install platforms. +class ArtifactCollection + + class MissingArtifact < RuntimeError + end + + attr_reader :project + attr_reader :config + + def initialize(project, config) + @project = project + @config = config + end + + def platform_map_json + IO.read(File.expand_path("../#{project}.json", __FILE__)) + end + + def platform_map + JSON.parse(platform_map_json) + end + + def platform_name_map_path + File.expand_path("../#{project}-platform-names.json", __FILE__) + end + + def platform_name_map_json + IO.read(platform_name_map_path) + end + + def platform_name_map + JSON.parse(platform_name_map_json) + end + + def package_paths + @package_paths ||= Dir['**/pkg/*']. + sort. + reject {|path| path.include?("BUILD_VERSION") }. + reject {|path| path.include?("metadata.json") } + end + + def artifacts + artifacts = [] + missing_packages = [] + platform_map.each do |build_platform_spec, supported_platforms| + if path = package_paths.find { |p| p.include?(build_platform_spec) } + artifacts << Artifact.new(path, supported_platforms, config) + else + missing_packages << build_platform_spec + end + end + error_on_missing_pkgs!(missing_packages) + artifacts + end + + def error_on_missing_pkgs!(missing_packages) + unless missing_packages.empty? + if config[:ignore_missing_packages] + missing_packages.each do |pkg_config| + # TODO: this should go to $stderr + puts "WARN: Missing package for config: #{pkg_config}" + end + else + raise MissingArtifact, "Missing packages for config(s): '#{missing_packages.join("' '")}'" + end + end + end +end + +# Represents an individual package which has one or more supported platforms. +class Artifact + + attr_reader :path + attr_reader :platforms + attr_reader :config + + def initialize(path, platforms, config) + @path = path + @platforms = platforms + @config = config + end + + # Adds the package to +release_manifest+, which is a Hash. The result is in this form: + # "el" => { + # "5" => { "x86_64" => { "11.4.0-1" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm" } } + # } + # This method mutates the argument (hence the `!` at the end). The updated + # release manifest is returned. + def add_to_release_manifest!(release_manifest) + platforms.each do |distro, version, arch| + release_manifest[distro] ||= {} + release_manifest[distro][version] ||= {} + release_manifest[distro][version][arch] = { build_version => relpath } + # TODO: when adding checksums, the desired format is like this: + # build_support_json[platform][platform_version][machine_architecture][options[:version]]["relpath"] = build_location + end + release_manifest + end + + # Adds the package to +release_manifest+, which is a Hash. The result is in this form: + # "el" => { + # "5" => { + # "x86_64" => { + # "11.4.0-1" => { + # "relpath" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm", + # "md5" => "123f00d...", + # "sha256" => 456beef..." + # } + # } + # } + # } + # This method mutates the argument (hence the `!` at the end). The updated + # release manifest is returned. + def add_to_v2_release_manifest!(release_manifest) + platforms.each do |distro, version, arch| + pkg_info = { + "relpath" => relpath, + "md5" => md5, + "sha256" => sha256 + } + + release_manifest[distro] ||= {} + release_manifest[distro][version] ||= {} + release_manifest[distro][version][arch] = { build_version => pkg_info } + end + release_manifest + end + + def build_platform + platforms.first + end + + def build_version + config[:version] + end + + def relpath + # upload build to build platform directory + "/#{build_platform.join('/')}/#{path.split('/').last}" + end + + def md5 + @md5 ||= digest(Digest::MD5) + end + + def sha256 + @sha256 ||= digest(Digest::SHA256) + end + + private + + def digest(digest_class) + digest = digest_class.new + File.open(path) do |io| + while chunk = io.read(1024 * 8) + digest.update(chunk) + end + end + digest.hexdigest + end +end + +class ShipIt + attr_reader :argv + attr_reader :options + + def initialize(argv=[]) + @argv = argv + @options = {:package_s3_config_file => "~/.s3cfg"} + end + + def release_it + $stdout.sync = true + parse_options + artifact_collection = ArtifactCollection.new(options[:project], options) + artifacts = artifact_collection.artifacts + + v2_metadata = {} + + artifacts.each do |artifact| + artifact.add_to_v2_release_manifest!(v2_metadata) + upload_package(artifact.path, artifact.relpath) + + # Optionally publish to the new Artifactory infrastructure + if options[:publish_to_artifactory] + Omnibus.load_configuration(options[:omnibus_config]) + publish_to_artifactory(artifact) + end + + end + upload_v2_platform_name_map(artifact_collection.platform_name_map_path) + upload_v2_manifest(v2_metadata) + end + + def option_parser + @option_parser ||= OptionParser.new do |opts| + opts.banner = "Usage: #{$0} [options]" + + opts.on("-p", "--project PROJECT", "the project to release") do |project| + options[:project] = project + end + + opts.on("-v", "--version VERSION", "the version of the installer to release") do |version| + options[:version] = version + end + + opts.on("-b", "--bucket S3_BUCKET_NAME", "the name of the s3 bucket to release to") do |bucket| + options[:bucket] = bucket + end + + opts.on("-c", "--package-s3-config S3_CMD_CONFIG_FILE", "path to the s3cmd config file for packages bucket") do |config| + options[:package_s3_config_file] = config + end + + opts.on("-M", "--metadata-bucket S3_BUCKET_NAME", "the name of the S3 bucket for v2 metadata") do |bucket| + options[:metadata_bucket] = bucket + end + + opts.on("-m", "--metadata-s3-config S3_CMD_CONFIG_FILE", "path to the s3cmd config file for the v2 metadata AWS account") do |config_path| + options[:metadata_s3_config_file] = config_path + end + + opts.on("--ignore-missing-packages", + "indicates the release should continue if any build packages are missing") do |missing| + options[:ignore_missing_packages] = missing + end + + opts.on("--publish-to-artifactory", + "indicates the release should be published to Artifactory") do |artifactory| + options[:publish_to_artifactory] = artifactory + end + + opts.on("--omnibus-config OMNIBUS_CONFIG_FILE", + "path to the Omnibus config file which is required for Artifactory publishing") do |config_path| + options[:omnibus_config] = config_path + end + end + end + + def parse_options + option_parser.parse(argv) + + # check for an optional BUILD_VERSION file which is generated by the build script + if options[:version].nil? + # this file should be the same across all platforms so grab the first one + build_version_file = Dir['**/pkg/BUILD_VERSION'].first + options[:version] = IO.read(build_version_file).chomp if build_version_file + end + + required = [:project, :version, :bucket, :metadata_bucket, :metadata_s3_config_file] + + # If --publish-to-artifactory was provided then --omnibus-config is required + required << :omnibus_config if options[:publish_to_artifactory] + + missing = required.select {|param| options[param].nil?} + if !missing.empty? + puts "Missing required options: #{missing.join(', ')}" + puts option_parser + exit 1 + end + rescue OptionParser::InvalidOption, OptionParser::MissingArgument + puts $!.to_s + puts option_parser + exit 1 + end + + def shellout_opts + {:timeout => 1200, :live_stream => STDOUT} + end + + def progress + if STDOUT.tty? + "--progress" + else + "--no-progress" + end + end + + def upload_package(local_path, s3_path) + s3_cmd = ["s3cmd", + "-c #{options[:package_s3_config_file]}", + "put", + progress, + "--acl-public", + local_path, + "s3://#{options[:bucket]}#{s3_path}"].join(" ") + shell = Mixlib::ShellOut.new(s3_cmd, shellout_opts) + shell.run_command + shell.error! + end + + def upload_v2_manifest(manifest) + File.open("v2-release-manifest.json", "w") {|f| f.puts JSON.pretty_generate(manifest)} + + s3_location = "s3://#{options[:metadata_bucket]}/#{options[:project]}-release-manifest/#{options[:version]}.json" + puts "UPLOAD: v2-release-manifest.json -> #{s3_location}" + s3_cmd = ["s3cmd", + "-c #{options[:metadata_s3_config_file]}", + "put", + "--acl-public", + "v2-release-manifest.json", + s3_location].join(" ") + shell = Mixlib::ShellOut.new(s3_cmd, shellout_opts) + shell.run_command + shell.error! + end + + def upload_v2_platform_name_map(platform_names_file) + s3_location = "s3://#{options[:metadata_bucket]}/#{options[:project]}-release-manifest/#{options[:project]}-platform-names.json" + puts "UPLOAD: #{options[:project]}-platform-names.json -> #{s3_location}" + s3_cmd = ["s3cmd", + "-c #{options[:metadata_s3_config_file]}", + "put", + "--acl-public", + platform_names_file, + s3_location].join(" ") + shell = Mixlib::ShellOut.new(s3_cmd, shellout_opts) + shell.run_command + shell.error! + end + + def upload_v2_manifest? + !options[:metadata_bucket].nil? + end + + def publish_to_artifactory(artifact) + + metadata_json = "#{artifact.path}.metadata.json" + + if File.exist?(metadata_json) + metadata = JSON.parse(IO.read(metadata_json), :symbolize_names => true) + + # Historically we name our Windows-specific project definitions + # PROJECT_NAME-windows. We want to publish this project under + # PROJECT_NAME though. Luckily we are moving to a place where the + # Windows-specific project definition will just become part of the + # regular *nix project definition. Until then we need to modify the + # the artifact's associated `*.metadata.json` file as the Omnibus + # publisher uses this metadata when publishing to Artifactory. + if match = metadata[:name].match(/^(?<project_basename>.*)-windows$/) + metadata[:name] = match[:project_basename] + + File.open(metadata_json, 'w+') do |f| + f.write(JSON.pretty_generate(metadata)) + end + end + end + + # Publish artifact under each tested platform, platform version and arch! + artifact.platforms.each do |distro, version, arch| + + # We only really care about publishing for Windows 64-bit so ignore + # the 32-bit mappings + next if (distro == 'windows') && (arch == 'i686') + + tries = 3 + + begin + # Apply various 'fixes' to the metadata before uploading + fix_metadata(artifact.path) + + publisher = Omnibus::ArtifactoryPublisher.new( + artifact.path, + repository: 'omnibus-current-local', + platform: distro, + platform_version: version, + ) + + + publisher.publish do |package| + puts "Uploaded '#{package.name}'" + end + + rescue Omnibus::NoPackageMetadataFile => e + + puts "Could not locate package metadata file '#{artifact.path}.metadata.json'...skipping publish." + + rescue Artifactory::Error::ArtifactoryError => e + puts "\nError during publishing: #{e.message}" + puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}" + + if (tries -= 1) != 0 + puts "\nRetrying failed publish #{tries} more time(s)..." + retry + else + raise + end + end + end + end + + # Attempts to fix an out-of-date `*.metadata.json` file produced by an + # an older version of Omnibus. The data contained in this file is very + # important in publishing activities. + def fix_metadata(package_local_path) + package = Omnibus::Package.new(package_local_path) + + old_metadata = JSON.parse(File.read(package.metadata.path), symbolize_names: true) + new_metadata = old_metadata.dup + + # OLD Omnibus does not include a project name!!!! O_o + unless new_metadata[:name] + match = package.name.match(/^(?<project_name>[a-z].*)(_|-)\d\.\d\.\d/) + new_metadata[:name] = match[:project_name] + end + + # Historically we name our Windows-specific project definitions + # PROJECT_NAME-windows. We want to publish this project under + # PROJECT_NAME though. Luckily we are moving to a place where the + # Windows-specific project definition will just become part of the + # regular *nix project definition. Until then we need to modify the + # the artifact's associated `*.metadata.json` file as the Omnibus + # publisher uses this metadata when publishing to Artifactory. + match = new_metadata[:name].match(/^(?<project_basename>.*)-windows$/) + new_metadata[:name] = match[:project_basename] if match + + # In pre-4.0 Omnibus packages are signed outside of the Omnibus + # build process which means Omnibus's generated checksums are wrong. + # As we use the Omnibus-generated metadata when publishing to + # Artifactory checksum mismatches are OK until Omnibus 4.0 is in + # general use. + if new_metadata[:platform] == 'el' + new_metadata[:md5] = package.md5 + new_metadata[:sha1] = package.sha1 + new_metadata[:sha256] = package.sha256 + new_metadata[:sha512] = package.sha512 + end + + # OLD Omnibus does not generate all checksum types + new_metadata[:sha256] = package.sha256 unless new_metadata[:sha256] + new_metadata[:sha512] = package.sha512 unless new_metadata[:sha512] + + # If the metadata has changed write the file + if new_metadata != old_metadata + metadata = if Omnibus.const_defined?('Metadata') + Omnibus::Metadata.new(package, new_metadata) + else + Omnibus::Package::Metadata.new(package, new_metadata) + end + puts "\nMetadata has changed...updating '#{metadata.path}'" + metadata.save + end + end +end + + +if !$0.include?("rspec") + ShipIt.new(ARGV).release_it +else + describe ArtifactCollection do + + # project_json is the thing that maps a build to. It is stored in the same + # directory with basename determined by project, e.g., "chef.json" for + # chef-client, "chef-server.json" for chef-server. By convention, the first + # entry is the platform that we actually do the build on. + let(:platform_map_json) do + <<-E +{ + "build_os=centos-5,machine_architecture=x64,role=oss-builder": [ + [ + "el", + "5", + "x86_64" + ], + [ + "sles", + "11.2", + "x86_64" + ] + ], + "build_os=centos-5,machine_architecture=x86,role=oss-builder": [ + [ + "el", + "5", + "i686" + ], + [ + "sles", + "11.2", + "i686" + ] + ] +} +E + end + + let(:platform_map) do + JSON.parse(platform_map_json) + end + + # mapping of short platform names to longer ones. + # This file lives in this script's directory under $project-platform-names.json + let(:platform_name_map_json) do + <<-E +{ + "el" : "Enterprise Linux", + "debian" : "Debian", + "mac_os_x" : "OS X", + "ubuntu" : "Ubuntu", + "solaris2" : "Solaris", + "sles" : "SUSE Enterprise", + "suse" : "openSUSE", + "windows" : "Windows" +} +E + end + + let(:platform_name_map) do + JSON.parse(platform_name_map_json) + end + + let(:directory_contents) do + %w[ + build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm.metadata.json + build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm + build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/BUILD_VERSION + build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm.metadata.json + build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm + build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/BUILD_VERSION + ] + end + + let(:artifact_collection) do + ArtifactCollection.new("demoproject", {}) + end + + it "has a project name" do + artifact_collection.project.should == "demoproject" + end + + it "has config" do + artifact_collection.config.should == {} + end + + it "loads the mapping of build platforms to install platforms from the local copy" do + expected_path = File.expand_path("../demoproject.json", __FILE__) + IO.should_receive(:read).with(expected_path).and_return(platform_map_json) + artifact_collection.platform_map_json.should == platform_map_json + end + + it "loads the mapping of platform short names to long names from the local copy" do + expected_path = File.expand_path("../demoproject-platform-names.json", __FILE__) + IO.should_receive(:read).with(expected_path).and_return(platform_name_map_json) + artifact_collection.platform_name_map_json.should == platform_name_map_json + end + + it "finds the package files among the artifacts" do + Dir.should_receive(:[]).with("**/pkg/*").and_return(directory_contents) + expected = %w[ + build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm + build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm + ] + artifact_collection.package_paths.should == expected + end + + context "after loading the build and platform mappings" do + + before do + artifact_collection.should respond_to(:platform_map_json) + artifact_collection.stub!(:platform_map_json).and_return(platform_map_json) + artifact_collection.should respond_to(:platform_name_map_json) + artifact_collection.stub!(:platform_name_map_json).and_return(platform_name_map_json) + end + + it "parses the build platform mapping" do + artifact_collection.platform_map.should == platform_map + end + + it "parses the platform short name => long name mapping" do + artifact_collection.platform_name_map.should == platform_name_map + end + + it "returns a list of artifacts for each package" do + Dir.should_receive(:[]).with("**/pkg/*").and_return(directory_contents) + + artifact_collection.should have(2).artifacts + centos5_64bit_artifact = artifact_collection.artifacts.first + + path = "build_os=centos-5,machine_architecture=x64,role=oss-builder/pkg/demoproject-10.22.0-1.el5.x86_64.rpm" + centos5_64bit_artifact.path.should == path + + platforms = [ [ "el", "5", "x86_64" ], [ "sles","11.2","x86_64" ] ] + centos5_64bit_artifact.platforms.should == platforms + end + + context "and some expected packages are missing" do + let(:directory_contents) do + %w[ + build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-10.22.0-1.el5.i686.rpm + build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/BUILD_VERSION + ] + end + + before do + Dir.should_receive(:[]).with("**/pkg/*").and_return(directory_contents) + end + + it "errors out verifying all packages are available" do + err_msg = "Missing packages for config(s): 'build_os=centos-5,machine_architecture=x64,role=oss-builder'" + lambda {artifact_collection.artifacts}.should raise_error(ArtifactCollection::MissingArtifact, err_msg) + end + + end + end + + end # describe ArtifactCollection + + describe Artifact do + + let(:path) { "build_os=centos-5,machine_architecture=x86,role=oss-builder/pkg/demoproject-11.4.0-1.el5.x86_64.rpm" } + + let(:content) { StringIO.new("this is the package content\n") } + + let(:md5) { "d41d8cd98f00b204e9800998ecf8427e" } + + let(:sha256) { "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } + + let(:platforms) { [ [ "el", "5", "x86_64" ], [ "sles","11.2","x86_64" ] ] } + + let(:artifact) { Artifact.new(path, platforms, { :version => "11.4.0-1" }) } + + it "has the path to the package" do + artifact.path.should == path + end + + it "has a list of platforms the package supports" do + artifact.platforms.should == platforms + end + + it "generates a MD5 of an artifact" do + File.should_receive(:open).with(path).and_return(content) + artifact.md5.should == md5 + end + + it "generates a SHA256 of an artifact" do + File.should_receive(:open).with(path).and_return(content) + artifact.sha256.should == sha256 + end + + it "adds the package to a release manifest" do + expected = { + "el" => { + "5" => { "x86_64" => { "11.4.0-1" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm" } } + }, + "sles" => { + "11.2" => { "x86_64" => { "11.4.0-1" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm" } } + } + } + + manifest = artifact.add_to_release_manifest!({}) + manifest.should == expected + end + + it "adds the package to a v2 release manifest" do + File.should_receive(:open).with(path).twice.and_return(content) + expected = { + "el" => { + "5" => { "x86_64" => { "11.4.0-1" => { + "relpath" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm", + "md5" => md5, + "sha256" => sha256 + } + } + } + }, + "sles" => { + "11.2" => { "x86_64" => { "11.4.0-1" => { + "relpath" => "/el/5/x86_64/demoproject-11.4.0-1.el5.x86_64.rpm", + "md5" => md5, + "sha256" => sha256 + } + } + } + } + } + v2_manifest = artifact.add_to_v2_release_manifest!({}) + v2_manifest.should == expected + end + + end +end + diff --git a/omnibus/jenkins/solo.rb b/omnibus/jenkins/solo.rb new file mode 100644 index 0000000000..3d33309d8f --- /dev/null +++ b/omnibus/jenkins/solo.rb @@ -0,0 +1,3 @@ +pwd = File.expand_path(File.dirname(__FILE__)) +file_cache_path "#{pwd}/chef-solo/cache" +cookbook_path ["#{pwd}/../cookbooks", "#{pwd}/../vendor/cookbooks"] diff --git a/omnibus/jenkins/verify-chef-container.sh b/omnibus/jenkins/verify-chef-container.sh new file mode 100755 index 0000000000..d2eb54fe24 --- /dev/null +++ b/omnibus/jenkins/verify-chef-container.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +export PATH=/opt/chef/bin:$PATH + +# Ensure the calling environment (disapproval look Bundler) does not +# infect our Ruby environment created by the `chef-int` cli. +for ruby_env_var in _ORIGINAL_GEM_PATH \ + BUNDLE_BIN_PATH \ + BUNDLE_GEMFILE \ + GEM_HOME \ + GEM_PATH \ + GEM_ROOT \ + RUBYLIB \ + RUBYOPT \ + RUBY_ENGINE \ + RUBY_ROOT \ + RUBY_VERSION +do + unset $ruby_env_var +done + +sudo chef-init --verify diff --git a/omnibus/jenkins/verify-chef.bat b/omnibus/jenkins/verify-chef.bat new file mode 100755 index 0000000000..1c159f0668 --- /dev/null +++ b/omnibus/jenkins/verify-chef.bat @@ -0,0 +1,56 @@ + +@ECHO OFF + +REM ; %PROJECT_NAME% is set by Jenkins, this allows us to use the same script to verify +REM ; Chef and Angry Chef +cd C:\opscode\%PROJECT_NAME%\bin + +REM ; We don't want to add the embedded bin dir to the main PATH as this +REM ; could mask issues in our binstub shebangs. +SET EMBEDDED_BIN_DIR=C:\opscode\%PROJECT_NAME%\embedded\bin + +ECHO. + +FOR %%b IN ( + chef-client + knife + chef-solo + ohai +) DO ( + + + ECHO Checking for existence of binfile `%%b`... + + IF EXIST %%b ( + ECHO ...FOUND IT! + ) ELSE ( + GOTO :error + ) + ECHO. +) + +call chef-client --version + +REM ; Exercise various packaged tools to validate binstub shebangs +call %EMBEDDED_BIN_DIR%\ruby --version +call %EMBEDDED_BIN_DIR%\gem --version +call %EMBEDDED_BIN_DIR%\bundle --version +call %EMBEDDED_BIN_DIR%\rspec --version + +SET PATH=C:\opscode\%PROJECT_NAME%\bin;C:\opscode\%PROJECT_NAME%\embedded\bin;%PATH% + +REM ; Test against the vendored chef gem +cd C:\opscode\%PROJECT_NAME%\embedded\lib\ruby\gems\2*\gems\chef-*-mingw32 + +IF NOT EXIST "Gemfile.lock" ( + ECHO "Chef gem does not contain a Gemfile.lock! This is needed to run any tests." + GOTO :error +) + +IF "%PIPELINE_NAME%" == "chef-13" ( + REM ; Running unit and functional tests + call bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o %WORKSPACE%\test.xml -f documentation spec/unit spec/functional +) ELSE ( + REM ; Running unit tests + call bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o %WORKSPACE%\test.xml -f documentation spec/unit spec/functional +) diff --git a/omnibus/jenkins/verify-chef.sh b/omnibus/jenkins/verify-chef.sh new file mode 100755 index 0000000000..4e60b1fd9f --- /dev/null +++ b/omnibus/jenkins/verify-chef.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +# $PROJECT_NAME is set by Jenkins, this allows us to use the same script to verify +# Chef and Angry Chef +PATH=/opt/$PROJECT_NAME/bin:$PATH +export PATH + +BIN_DIR=/opt/$PROJECT_NAME/bin +export BIN_DIR + +# We don't want to add the embedded bin dir to the main PATH as this +# could mask issues in our binstub shebangs. +EMBEDDED_BIN_DIR=/opt/$PROJECT_NAME/embedded/bin +export EMBEDDED_BIN_DIR + +# If we are on Mac our symlinks are located under /usr/local/bin +# otherwise they are under /usr/bin +if [ -f /usr/bin/sw_vers ]; then + USR_BIN_DIR="/usr/local/bin" +else + USR_BIN_DIR="/usr/bin" +fi +export USR_BIN_DIR + +# sanity check that we're getting the correct symlinks from the pre-install script +# solaris doesn't have readlink or test -e. ls -n is different on BSD. proceed with caution. +if [ ! -L $USR_BIN_DIR/chef-client ] || [ `ls -l $USR_BIN_DIR/chef-client | awk '{print$NF}'` != "$BIN_DIR/chef-client" ]; then + echo "$USR_BIN_DIR/chef-client symlink to $BIN_DIR/chef-client was not correctly created by the pre-install script!" + exit 1 +fi + +if [ ! -L $USR_BIN_DIR/knife ] || [ `ls -l $USR_BIN_DIR/knife | awk '{print$NF}'` != "$BIN_DIR/knife" ]; then + echo "$USR_BIN_DIR/knife symlink to $BIN_DIR/knife was not correctly created by the pre-install script!" + exit 1 +fi + +if [ ! -L $USR_BIN_DIR/chef-solo ] || [ `ls -l $USR_BIN_DIR/chef-solo | awk '{print$NF}'` != "$BIN_DIR/chef-solo" ]; then + echo "$USR_BIN_DIR/chef-solo symlink to $BIN_DIR/chef-solo was not correctly created by the pre-install script!" + exit 1 +fi + +if [ ! -L $USR_BIN_DIR/ohai ] || [ `ls -l $USR_BIN_DIR/ohai | awk '{print$NF}'` != "$BIN_DIR/ohai" ]; then + echo "$USR_BIN_DIR/ohai symlink to $BIN_DIR/ohai was not correctly created by the pre-install script!" + exit 1 +fi + +# Ensure the calling environment (disapproval look Bundler) does not +# infect our Ruby environment created by the `chef-client` cli. +for ruby_env_var in _ORIGINAL_GEM_PATH \ + BUNDLE_BIN_PATH \ + BUNDLE_GEMFILE \ + GEM_HOME \ + GEM_PATH \ + GEM_ROOT \ + RUBYLIB \ + RUBYOPT \ + RUBY_ENGINE \ + RUBY_ROOT \ + RUBY_VERSION + +do + unset $ruby_env_var +done + +chef-client --version + +# Exercise various packaged tools to validate binstub shebangs +$EMBEDDED_BIN_DIR/ruby --version +$EMBEDDED_BIN_DIR/gem --version +$EMBEDDED_BIN_DIR/bundle --version +$EMBEDDED_BIN_DIR/rspec --version + +# ffi-yajl must run in c-extension mode or we take perf hits, so we force it +# before running rspec so that we don't wind up testing the ffi mode +FORCE_FFI_YAJL=ext +export FORCE_FFI_YAJL + +PATH=/opt/$PROJECT_NAME/bin:/opt/$PROJECT_NAME/embedded/bin:$PATH +export PATH + +# Test against the vendored Chef gem +cd /opt/$PROJECT_NAME/embedded/lib/ruby/gems/*/gems/chef-[0-9]* + +if [ ! -f "Gemfile.lock" ]; then + echo "Chef gem does not contain a Gemfile.lock! This is needed to run any tests." + exit 1 +fi + +sudo env PATH=$PATH TERM=xterm bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o $WORKSPACE/test.xml -f documentation spec/functional spec/unit diff --git a/omnibus/jenkins/verify-chefdk.bat b/omnibus/jenkins/verify-chefdk.bat new file mode 100755 index 0000000000..9677b4a34e --- /dev/null +++ b/omnibus/jenkins/verify-chefdk.bat @@ -0,0 +1,32 @@ + +@ECHO OFF + +cd C:\opscode\chefdk\bin + +ECHO( + +FOR %%b IN ( + berks + chef + chef-client + kitchen + knife + rubocop +) DO ( + + + ECHO Checking for existence of binfile `%%b`... + + IF EXIST %%b ( + + ECHO ...FOUND IT! + + ) ELSE ( + + GOTO :error + + ) + ECHO( +) + +chef verify diff --git a/omnibus/jenkins/verify-chefdk.sh b/omnibus/jenkins/verify-chefdk.sh new file mode 100755 index 0000000000..1860f554cc --- /dev/null +++ b/omnibus/jenkins/verify-chefdk.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +export PATH=/opt/chefdk/bin:$PATH + +# Ensure the calling environment (disapproval look Bundler) does not +# infect our Ruby environment created by the `chef` cli. +for ruby_env_var in _ORIGINAL_GEM_PATH \ + BUNDLE_BIN_PATH \ + BUNDLE_GEMFILE \ + GEM_HOME \ + GEM_PATH \ + GEM_ROOT \ + RUBYLIB \ + RUBYOPT \ + RUBY_ENGINE \ + RUBY_ROOT \ + RUBY_VERSION +do + unset $ruby_env_var +done + +sudo chef verify --unit |