diff options
| author | Lamont Granquist <lamont@scriptkiddie.org> | 2017-05-09 09:39:23 -0700 |
|---|---|---|
| committer | Lamont Granquist <lamont@scriptkiddie.org> | 2017-05-09 10:16:36 -0700 |
| commit | 1b1a8b34c872bc55f2acf77e44ac70e6e1efcab7 (patch) | |
| tree | 3c6789905e64f7f3f9e3343a61bd4f8ce7f5a737 | |
| parent | 0ad389f48d43ebfc4347c41a3573ee855993c5f1 (diff) | |
| download | chef-1b1a8b34c872bc55f2acf77e44ac70e6e1efcab7.tar.gz | |
simplify omnibus config and greenify builds again
this is also necessary for bundler-1.14.x
i'm still not entirely clear why we ever needed all the fussy software gem
configs or what the build-chef / build-chef-gem infrastructure ever
did for us. it seems to have been mostly micro-optimization around
building the software gems before bundle installing the project in order
to take advantage of git caching. i aggressively don't care about that,
this is quite fast enough. we can install nokogiri and libgecode early
and that should take care of 98% of the build optimization issue.
Signed-off-by: Lamont Granquist <lamont@scriptkiddie.org>
37 files changed, 102 insertions, 1389 deletions
diff --git a/.travis.yml b/.travis.yml index 4fa9b65431..91343eb67b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,24 +35,24 @@ matrix: sudo: true script: sudo -E $(which bundle) exec rake spec; # also remove integration / external tests - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen - rvm: 2.4.1 sudo: true script: sudo -E $(which bundle) exec rake spec; # also remove integration / external tests - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen - env: CHEFSTYLE: 1 rvm: 2.4.1 script: bundle exec rake style # also remove integration / external tests - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen - env: AUDIT_CHECK: 1 rvm: 2.4.1 script: bundle exec bundle-audit check --update # also remove integration / external tests - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen # # External tests # @@ -98,7 +98,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -116,7 +116,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -134,7 +134,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -152,7 +152,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -170,7 +170,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -188,7 +188,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -206,7 +206,7 @@ matrix: # before_install: # - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) # - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) -# bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen +# bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen # before_script: # - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) # - cd kitchen-tests @@ -224,7 +224,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -242,7 +242,7 @@ matrix: # before_install: # - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) # - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) -# bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen +# bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen # before_script: # - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) # - cd kitchen-tests @@ -260,7 +260,7 @@ matrix: # before_install: # - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) # - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) -# bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen +# bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen # before_script: # - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) # - cd kitchen-tests @@ -279,7 +279,7 @@ matrix: - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - sudo apt-get update - sudo apt-get -y install squid3 git curl - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen env: - PROXY_TESTS_DIR=proxy_tests/files/default/scripts - PROXY_TESTS_REPO=$PROXY_TESTS_DIR/repo @@ -299,7 +299,7 @@ matrix: before_install: - gem update --system $(grep rubygems omnibus_overrides.rb | cut -d'"' -f2) - gem install bundler -v $(grep bundler omnibus_overrides.rb | cut -d'"' -f2) - bundler_args: --without ci docgen guard integration maintenance omnibus_package tools aix bsd mac_os_x solaris windows --frozen + bundler_args: --without ci docgen guard integration maintenance omnibus_package --frozen before_script: - sudo iptables -L DOCKER || ( echo "DOCKER iptables chain missing" ; sudo iptables -N DOCKER ) - cd kitchen-tests @@ -40,13 +40,12 @@ group(:maintenance, :ci) do end # Everything except AIX -group(:linux, :bsd, :mac_os_x, :solaris, :windows) do - # may need to disable this in insolation on fussy builds like AIX, RHEL4, etc +group(:ruby_prof) do gem "ruby-prof" end # Everything except AIX and Windows -group(:linux, :bsd, :mac_os_x, :solaris) do +group(:ruby_shadow) do gem "ruby-shadow", platforms: :ruby end diff --git a/appveyor.yml b/appveyor.yml index 0ec136a574..c809811bb1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,9 +34,10 @@ install: - gem install bundler -v %OMNIBUS_BUNDLER% --quiet --no-ri --no-rdoc || gem install bundler -v %OMNIBUS_BUNDLER% --quiet --no-ri --no-rdoc || gem install bundler -v %OMNIBUS_BUNDLER% --quiet --no-ri --no-rdoc - gem --version - bundler --version - - SET BUNDLE_WITHOUT=guard:maintenance:tools:integration:ci:docgen:travis:style:omnibus_package:aix:bsd:linux:mac_os_x:solaris + - SET BUNDLE_WITHOUT=server:docgen:maintenance:pry:travis:integration:ci - appveyor DownloadFile http://curl.haxx.se/ca/cacert.pem -FileName C:\cacert.pem - set SSL_CERT_FILE=C:\cacert.pem + - bundle env build_script: - bundle install || bundle install || bundle install diff --git a/ci/bundle_install.sh b/ci/bundle_install.sh index 6914189479..80e56ecbd2 100755 --- a/ci/bundle_install.sh +++ b/ci/bundle_install.sh @@ -1,10 +1,12 @@ #!/bin/sh +# FIXME: someone document what actually calls this +# FIXME: is this really the best place for this or should it go in the rake tasks? set -evx gem environment -bundler_version=$(grep bundler omnibus_overrides.rb | cut -d'"' -f2) -gem install bundler -v $bundler_version --user-install --conservative +omnibus_bundler=$(grep bundler omnibus_overrides.rb | cut -d'"' -f2) +gem uninstall bundler -a -x +gem install bundler -v $omnibus_bundler --user-install --conservative # WITH: ci (for version bumping and changelog creation) -export BUNDLE_WITHOUT=omnibus_package:test:pry:integration:docgen:maintenance:travis:aix:bsd:linux:mac_os_x:solaris:windows:development -bundle _${bundler_version}_ install +bundle install --without omnibus_package test pry integration docgen maintenance travis aix bsd linux mac_os_x solaris windows development diff --git a/ci/dependency_update.sh b/ci/dependency_update.sh index efc0b3ad2f..7930b864d8 100755 --- a/ci/dependency_update.sh +++ b/ci/dependency_update.sh @@ -1,5 +1,7 @@ #!/bin/sh +# FIXME: this seems uselessly trivial, replace with a rake task and have ci call the rake task? + set -evx . ci/bundle_install.sh diff --git a/ci/verify-chef.bat b/ci/verify-chef.bat index 7ba0817938..4ff3e0be77 100755 --- a/ci/verify-chef.bat +++ b/ci/verify-chef.bat @@ -18,6 +18,20 @@ SET TMP=%TMP%\cheftest RMDIR /S /Q %TEMP% MKDIR %TEMP% +REM ; FIXME: we should really use Bundler.with_clean_env in the caller instead of re-inventing it here +set _ORIGINAL_GEM_PATH= +set BUNDLE_BIN_PATH= +set BUNDLE_GEMFILE= +set GEM_HOME= +set GEM_PATH= +set GEM_ROOT= +set RUBYLIB= +set RUBYOPT= +set RUBY_ENGINE= +set RUBY_ROOT= +set RUBY_VERSION= +set BUNDLER_VERSION= + FOR %%b IN ( chef-client knife @@ -46,8 +60,15 @@ call %EMBEDDED_BIN_DIR%\rspec --version SET PATH=C:\opscode\%PROJECT_NAME%\bin;C:\opscode\%PROJECT_NAME%\embedded\bin;%PATH% -REM ; Test against the vendored chef gem (cd into the output of "bundle show chef") -for /f "delims=" %%a in ('bundle show chef') do cd %%a +REM ; Test against the vendored chef gem (cd into the output of "gem which chef") +for /f "delims=" %%a in ('gem which chef') do set CHEFDIR=%%a +call :dirname "%CHEFDIR%" CHEFDIR +call :dirname "%CHEFDIR%" CHEFDIR +cd %CHEFDIR% + +cd + +type Gemfile.lock IF NOT EXIST "Gemfile.lock" ( ECHO "Chef gem does not contain a Gemfile.lock! This is needed to run any tests." @@ -61,7 +82,14 @@ IF "%PIPELINE_NAME%" == "chef-fips" ( REM ; ffi-yajl must run in c-extension mode for perf, so force it so we don't accidentally fall back to ffi set FORCE_FFI_YAJL=ext -set BUNDLE_GEMFILE=C:\opscode\%PROJECT_NAME%\Gemfile -set BUNDLE_IGNORE_CONFIG=true -set BUNDLE_FROZEN=1 -call bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o %WORKSPACE%\test.xml -f documentation spec/functional +set +call %EMBEDDED_BIN_DIR%\rspec -r rspec_junit_formatter -f RspecJunitFormatter -o %WORKSPACE%\test.xml -f documentation spec/functional + +GOTO :EOF + +:dirname file varName + setlocal ENABLEEXTENSIONS ENABLEDELAYEDEXPANSION + SET _dir=%~dp1 + SET _dir=%_dir:~0,-1% + endlocal & set %2=%_dir% +GOTO thout:EOF diff --git a/ci/verify-chef.sh b/ci/verify-chef.sh index 71d4afe0df..0c90d65dbe 100755 --- a/ci/verify-chef.sh +++ b/ci/verify-chef.sh @@ -64,7 +64,8 @@ for ruby_env_var in _ORIGINAL_GEM_PATH \ RUBYOPT \ RUBY_ENGINE \ RUBY_ROOT \ - RUBY_VERSION + RUBY_VERSION \ + BUNDLER_VERSION do unset $ruby_env_var @@ -83,14 +84,17 @@ $EMBEDDED_BIN_DIR/rspec --version FORCE_FFI_YAJL=ext export FORCE_FFI_YAJL +OLD_PATH=$PATH +PATH=/opt/$PROJECT_NAME/bin:/opt/$PROJECT_NAME/embedded/bin:$PATH + +gem_list=`gem which chef` +lib_dir=`dirname $gem_list` +CHEF_GEM=`dirname $lib_dir` + # ACCEPTANCE environment variable will be set on acceptance testers. # If is it set; we run the acceptance tests, otherwise run rspec tests. if [ "x$ACCEPTANCE" != "x" ]; then # Find the Chef gem and cd there. - OLD_PATH=$PATH - PATH=/opt/$PROJECT_NAME/bin:/opt/$PROJECT_NAME/embedded/bin:$PATH - cd /opt/$PROJECT_NAME - CHEF_GEM=`bundle show chef` PATH=$OLD_PATH # On acceptance testers we have Chef DK. We will use its Ruby environment @@ -109,17 +113,7 @@ if [ "x$ACCEPTANCE" != "x" ]; then env PATH=$PATH AWS_SSH_KEY_ID=$AWS_SSH_KEY_ID bundle install --deployment env PATH=$PATH AWS_SSH_KEY_ID=$AWS_SSH_KEY_ID KITCHEN_DRIVER=ec2 KITCHEN_CHEF_CHANNEL=unstable bundle exec chef-acceptance test --force-destroy --data-path $WORKSPACE/chef-acceptance-data else - PATH=/opt/$PROJECT_NAME/bin:/opt/$PROJECT_NAME/embedded/bin:$PATH - export PATH - - # Test against the installed Chef gem - cd /opt/$PROJECT_NAME - CHEF_GEM=`bundle show chef` cd $CHEF_GEM - if [ ! -f "Gemfile.lock" ]; then - echo "Chef gem does not contain a Gemfile.lock! This is needed to run any tests." - exit 1 - fi - sudo env BUNDLE_GEMFILE=/opt/$PROJECT_NAME/Gemfile BUNDLE_IGNORE_CONFIG=true BUNDLE_FROZEN=1 PATH=$PATH TERM=xterm bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o $WORKSPACE/test.xml -f documentation spec/functional + sudo bundle exec rspec -r rspec_junit_formatter -f RspecJunitFormatter -o $WORKSPACE/test.xml -f documentation spec/functional fi diff --git a/ci/version_bump.sh b/ci/version_bump.sh index 0de6b8d40f..9e4c85ebc8 100755 --- a/ci/version_bump.sh +++ b/ci/version_bump.sh @@ -1,5 +1,7 @@ #!/bin/sh +# FIXME: this seems uselessly trivial, replace with a rake task and have ci call the rake task? + set -evx export LANG=en_US.UTF-8 diff --git a/omnibus/Gemfile.lock b/omnibus/Gemfile.lock index 3d32a97a15..caf63bef43 100644 --- a/omnibus/Gemfile.lock +++ b/omnibus/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/chef/license_scout - revision: ff3cb28159e72414d63008f9a0d42e85d4aec4ba + revision: 92e9e27d71f457ea222b09c8fb3819bcb30a330a specs: license_scout (0.1.3) ffi-yajl (~> 2.2) @@ -25,7 +25,7 @@ GIT GIT remote: https://github.com/chef/omnibus-software - revision: cae44c1a3ebf7207516813ba15b372231c253954 + revision: 986940586711c0ea8895120605ed59e2cacdea26 specs: omnibus-software (4.0.0) chef-sugar (>= 3.4.0) @@ -38,13 +38,13 @@ GEM public_suffix (~> 2.0, >= 2.0.2) artifactory (2.8.1) awesome_print (1.7.0) - aws-sdk (2.9.7) - aws-sdk-resources (= 2.9.7) - aws-sdk-core (2.9.7) + aws-sdk (2.9.15) + aws-sdk-resources (= 2.9.15) + aws-sdk-core (2.9.15) aws-sigv4 (~> 1.0) jmespath (~> 1.0) - aws-sdk-resources (2.9.7) - aws-sdk-core (= 2.9.7) + aws-sdk-resources (2.9.15) + aws-sdk-core (= 2.9.15) aws-sigv4 (1.0.0) berkshelf (4.3.5) addressable (~> 2.3, >= 2.3.4) @@ -85,7 +85,7 @@ GEM celluloid-io (0.16.2) celluloid (>= 0.16.0) nio4r (>= 1.1.0) - chef-config (12.19.36) + chef-config (12.20.3) addressable fuzzyurl mixlib-config (~> 2.0) @@ -170,7 +170,7 @@ GEM multipart-post (~> 2.0.0) progressbar zhexdump (>= 0.0.2) - plist (3.2.0) + plist (3.3.0) progressbar (1.8.2) pry (0.10.4) coderay (~> 1.1.0) diff --git a/omnibus/config/projects/chef.rb b/omnibus/config/projects/chef.rb index 10c2cd21b3..2342a7e186 100644 --- a/omnibus/config/projects/chef.rb +++ b/omnibus/config/projects/chef.rb @@ -1,5 +1,5 @@ # -# Copyright 2012-2016, Chef Software, Inc. +# Copyright 2012-2017, Chef Software Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -39,17 +39,28 @@ else install_dir "#{default_root}/#{name}" end +override :chef, version: "local_source" + # Load dynamically updated overrides overrides_path = File.expand_path("../../../../omnibus_overrides.rb", current_file) instance_eval(IO.read(overrides_path), overrides_path) -override :"ruby-windows-devkit", version: "4.5.2-20111229-1559" if windows? && windows_arch_i386? - dependency "preparation" +dependency "chef" + +# FIXME?: might make sense to move dependencies below into the omnibus-software chef +# definition or into a chef-complete definition added to omnibus-software. +dependency "gem-permissions" +dependency "shebang-cleanup" +dependency "version-manifest" +dependency "openssl-customization" -# All actual dependencies are in chef-complete, so that the addition -# or removal of a dependency doesn't dirty the entire project file -dependency "chef-complete" +# devkit needs to come dead last these days so we do not use it to compile any gems +if windows? + override :"ruby-windows-devkit", version: "4.5.2-20111229-1559" if windows_arch_i386? + dependency "ruby-windows-devkit" + dependency "ruby-windows-devkit-bash" +end package :rpm do signing_passphrase ENV["OMNIBUS_RPM_SIGNING_PASSPHRASE"] diff --git a/omnibus/config/software/chef-appbundle.rb b/omnibus/config/software/chef-appbundle.rb deleted file mode 100644 index 8ea21103fb..0000000000 --- a/omnibus/config/software/chef-appbundle.rb +++ /dev/null @@ -1,18 +0,0 @@ -name "chef-appbundle" -default_version "local_source" - -license :project_license -skip_transitive_dependency_licensing true - -source path: project.files_path - -dependency "chef" - -build do - # This is where we get the definitions below - require_relative "../../files/chef-appbundle/build-chef-appbundle" - extend BuildChefAppbundle - - appbundle_gem "chef" - appbundle_gem "ohai" -end diff --git a/omnibus/config/software/chef-cleanup.rb b/omnibus/config/software/chef-cleanup.rb deleted file mode 100644 index 302e841699..0000000000 --- a/omnibus/config/software/chef-cleanup.rb +++ /dev/null @@ -1,30 +0,0 @@ -name "chef-cleanup" -default_version "local_source" - -license :project_license -skip_transitive_dependency_licensing true - -source path: project.files_path - -dependency "chef" - -build do - # This is where we get the definitions below - require_relative "../../files/chef/build-chef" - extend BuildChef - - # Clear the now-unnecessary git caches, cached gems, and git-checked-out gems - block "Delete bundler git cache and git installs" do - gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp - remove_directory "#{gemdir}/cache" - remove_directory "#{gemdir}/bundler" - end - - delete "#{install_dir}/embedded/docs" - delete "#{install_dir}/embedded/share/man" - delete "#{install_dir}/embedded/share/doc" - delete "#{install_dir}/embedded/share/gtk-doc" - delete "#{install_dir}/embedded/ssl/man" - delete "#{install_dir}/embedded/man" - delete "#{install_dir}/embedded/info" -end diff --git a/omnibus/config/software/chef-complete.rb b/omnibus/config/software/chef-complete.rb deleted file mode 100644 index 4e2b9f6ec2..0000000000 --- a/omnibus/config/software/chef-complete.rb +++ /dev/null @@ -1,20 +0,0 @@ -name "chef-complete" - -license :project_license -skip_transitive_dependency_licensing true - -dependency "chef" -dependency "chef-appbundle" -dependency "chef-cleanup" - -dependency "gem-permissions" -dependency "shebang-cleanup" -dependency "version-manifest" -dependency "openssl-customization" - -if windows? - # TODO can this be safely moved to before the chef? - # It would make caching better ... - dependency "ruby-windows-devkit" - dependency "ruby-windows-devkit-bash" -end diff --git a/omnibus/config/software/chef-gem-bcrypt_pbkdf-ruby.rb b/omnibus/config/software/chef-gem-bcrypt_pbkdf-ruby.rb deleted file mode 100644 index 1f7f4d3207..0000000000 --- a/omnibus/config/software/chef-gem-bcrypt_pbkdf-ruby.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://raw.githubusercontent.com/mfazekas/bcrypt_pbkdf-ruby/master/COPYING" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-binding_of_caller.rb b/omnibus/config/software/chef-gem-binding_of_caller.rb deleted file mode 100644 index 3e7a9f9c70..0000000000 --- a/omnibus/config/software/chef-gem-binding_of_caller.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://github.com/banister/binding_of_caller/blob/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-byebug.rb b/omnibus/config/software/chef-gem-byebug.rb deleted file mode 100644 index 3aef706e82..0000000000 --- a/omnibus/config/software/chef-gem-byebug.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://github.com/deivid-rodriguez/byebug/blob/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-debug_inspector.rb b/omnibus/config/software/chef-gem-debug_inspector.rb deleted file mode 100644 index ab818768ea..0000000000 --- a/omnibus/config/software/chef-gem-debug_inspector.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://github.com/banister/debug_inspector/blob/master/README.md" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-ffi-yajl.rb b/omnibus/config/software/chef-gem-ffi-yajl.rb deleted file mode 100644 index 44f98446bd..0000000000 --- a/omnibus/config/software/chef-gem-ffi-yajl.rb +++ /dev/null @@ -1,12 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://github.com/chef/ffi-yajl/blob/master/LICENSE" -skip_transitive_dependency_licensing true - -dependency "chef-gem-libyajl2" diff --git a/omnibus/config/software/chef-gem-ffi.rb b/omnibus/config/software/chef-gem-ffi.rb deleted file mode 100644 index ea8879c2ac..0000000000 --- a/omnibus/config/software/chef-gem-ffi.rb +++ /dev/null @@ -1,12 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "BSD-3-Clause" -license_file "https://github.com/ffi/ffi/blob/master/LICENSE" -license_file "https://github.com/ffi/ffi/blob/master/COPYING" -license_file "https://github.com/ffi/ffi/blob/master/LICENSE.SPECS" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-inspec.rb b/omnibus/config/software/chef-gem-inspec.rb deleted file mode 100644 index 8c5e1cbf26..0000000000 --- a/omnibus/config/software/chef-gem-inspec.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "Apache-2.0" -license_file "https://raw.githubusercontent.com/chef/inspec/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-json.rb b/omnibus/config/software/chef-gem-json.rb deleted file mode 100644 index 9217359ba2..0000000000 --- a/omnibus/config/software/chef-gem-json.rb +++ /dev/null @@ -1,11 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "Ruby" -license_file "https://github.com/flori/json/blob/master/README.md" -license_file "https://www.ruby-lang.org/en/about/license.txt" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-libyajl2.rb b/omnibus/config/software/chef-gem-libyajl2.rb deleted file mode 100644 index 47ef42e1cf..0000000000 --- a/omnibus/config/software/chef-gem-libyajl2.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "Apache-2.0" -license_file "https://github.com/chef/libyajl2-gem/blob/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-mini_portile2.rb b/omnibus/config/software/chef-gem-mini_portile2.rb deleted file mode 100644 index 36a2b833dd..0000000000 --- a/omnibus/config/software/chef-gem-mini_portile2.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://github.com/flavorjones/mini_portile/blob/master/LICENSE.txt" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-nokogiri.rb b/omnibus/config/software/chef-gem-nokogiri.rb deleted file mode 100644 index c6b8d03822..0000000000 --- a/omnibus/config/software/chef-gem-nokogiri.rb +++ /dev/null @@ -1,13 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://github.com/ruby-prof/ruby-prof/blob/master/LICENSE" -skip_transitive_dependency_licensing true - -dependency "chef-gem-pkg-config" -dependency "chef-gem-mini_portile2" diff --git a/omnibus/config/software/chef-gem-ohai.rb b/omnibus/config/software/chef-gem-ohai.rb deleted file mode 100644 index 7ee7ff4909..0000000000 --- a/omnibus/config/software/chef-gem-ohai.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "Apache-2.0" -license_file "https://github.com/chef/ohai/blob/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-pkg-config.rb b/omnibus/config/software/chef-gem-pkg-config.rb deleted file mode 100644 index 051091b73f..0000000000 --- a/omnibus/config/software/chef-gem-pkg-config.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "LGPL-2.1" -license_file "https://github.com/ruby-gnome2/pkg-config/blob/master/LGPL-2.1" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-rbnacl-libsodium.rb b/omnibus/config/software/chef-gem-rbnacl-libsodium.rb deleted file mode 100644 index 90bbc69dfb..0000000000 --- a/omnibus/config/software/chef-gem-rbnacl-libsodium.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "MIT" -license_file "https://raw.githubusercontent.com/cryptosphere/rbnacl-libsodium/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-ruby-prof.rb b/omnibus/config/software/chef-gem-ruby-prof.rb deleted file mode 100644 index af90212d23..0000000000 --- a/omnibus/config/software/chef-gem-ruby-prof.rb +++ /dev/null @@ -1,10 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "BSD-2-Clause" -license_file "https://github.com/ruby-prof/ruby-prof/blob/master/LICENSE" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef-gem-ruby-shadow.rb b/omnibus/config/software/chef-gem-ruby-shadow.rb deleted file mode 100644 index 02fc906d9d..0000000000 --- a/omnibus/config/software/chef-gem-ruby-shadow.rb +++ /dev/null @@ -1,11 +0,0 @@ -# gem installs this gem from the version specified in chef's Gemfile.lock -# so we can take advantage of omnibus's caching. Just duplicate this file and -# add the new software def to chef software def if you want to separate -# another gem's installation. -require_relative "../../files/chef-gem/build-chef-gem/gem-install-software-def" -BuildChefGem::GemInstallSoftwareDef.define(self, __FILE__) - -license "Public-Domain" -license_file "https://github.com/apalmblad/ruby-shadow/blob/master/LICENSE" -license_file "http://creativecommons.org/licenses/publicdomain/" -skip_transitive_dependency_licensing true diff --git a/omnibus/config/software/chef.rb b/omnibus/config/software/chef.rb deleted file mode 100644 index 4726f8b687..0000000000 --- a/omnibus/config/software/chef.rb +++ /dev/null @@ -1,88 +0,0 @@ -name "chef" -default_version "local_source" - -license :project_license - -# For the specific super-special version "local_source", build the source from -# the local git checkout. This is what you'd want to occur by default if you -# just ran omnibus build locally. -version("local_source") do - source path: File.expand_path("../..", project.files_path), - # Since we are using the local repo, we try to not copy any files - # that are generated in the process of bundle installing omnibus. - # If the install steps are well-behaved, this should not matter - # since we only perform bundle and gem installs from the - # omnibus cache source directory, but we do this regardless - # to maintain consistency between what a local build sees and - # what a github based build will see. - options: { exclude: [ "omnibus/vendor" ] } -end - -# For any version other than "local_source", fetch from github. -if version != "local_source" - source git: "git://github.com/chef/chef.git" -end - -# For nokogiri -dependency "libxml2" -dependency "libxslt" -dependency "liblzma" -dependency "zlib" - -# ruby and bundler and friends -dependency "ruby" -dependency "rubygems" -dependency "bundler" - -# Install all the native gems separately -# Worst offenders first to take best advantage of cache: -dependency "chef-gem-ffi-yajl" -dependency "chef-gem-ohai" -dependency "chef-gem-nokogiri" unless windows? -dependency "chef-gem-libyajl2" -dependency "chef-gem-ruby-prof" -dependency "chef-gem-byebug" -dependency "chef-gem-debug_inspector" -dependency "chef-gem-binding_of_caller" -dependency "chef-gem-inspec" -unless ios_xr? || solaris? - dependency "chef-gem-rbnacl-libsodium" - dependency "chef-gem-bcrypt_pbkdf-ruby" -end - -# Now everyone else, in alphabetical order because we don't care THAT much -Dir.entries(File.dirname(__FILE__)).sort.each do |gem_software| - if gem_software =~ /^(chef-gem-.+)\.rb$/ - dependency $1 - end -end - -build do - # This is where we get the definitions below - require_relative "../../files/chef/build-chef" - extend BuildChef - - project_env = env.dup - project_env["BUNDLE_GEMFILE"] = project_gemfile - - # Prepare to install: build config, retries, job, frozen=true - # TODO Windows install seems to sometimes install already-installed gems such - # as gherkin (and fail as a result) if you use jobs > 1. - create_bundle_config(project_gemfile, retries: 4, jobs: windows? ? 1 : 7, frozen: true) - - # Install all the things. Arguments are specified in .bundle/config (see create_bundle_config) - block { log.info(log_key) { "" } } - bundle "install --verbose", env: project_env - - # Check that it worked - block { log.info(log_key) { "" } } - bundle "check", env: project_env - - # fix up git-sourced gems - properly_reinstall_git_and_path_sourced_gems - install_shared_gemfile - - # Check that the final gemfile worked - block { log.info(log_key) { "" } } - bundle "check", env: env, cwd: File.dirname(shared_gemfile) -end diff --git a/omnibus/files/chef-appbundle/build-chef-appbundle.rb b/omnibus/files/chef-appbundle/build-chef-appbundle.rb deleted file mode 100644 index eaf4904501..0000000000 --- a/omnibus/files/chef-appbundle/build-chef-appbundle.rb +++ /dev/null @@ -1,93 +0,0 @@ -require_relative "../chef-gem/build-chef-gem" - -module BuildChefAppbundle - include BuildChefGem - - def lockdown_gem(gem_name) - shared_gemfile = self.shared_gemfile - - # Update the Gemfile to restrict to built versions so that bundle installs - # will do the right thing - block "Lock down the #{gem_name} gem" do - installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp - installed_gemfile = File.join(installed_path, "Gemfile") - - # - # Include the main distribution Gemfile in the gem's Gemfile - # - # NOTE: if this fails and the build retries, you will see this multiple - # times in the file. - # - distribution_gemfile = Pathname(shared_gemfile).relative_path_from(Pathname(installed_gemfile)).to_s - gemfile_text = <<-EOM.gsub(/^\s+/, "") - # Lock gems that are part of the distribution - distribution_gemfile = File.expand_path(#{distribution_gemfile.inspect}, __FILE__) - instance_eval(IO.read(distribution_gemfile), distribution_gemfile) - EOM - gemfile_text << IO.read(installed_gemfile) - create_file(installed_gemfile) { gemfile_text } - - # Remove the gemfile.lock - remove_file("#{installed_gemfile}.lock") if File.exist?("#{installed_gemfile}.lock") - - # If it's frozen, make it not be. - shellout!("#{bundle_bin} config --delete frozen") - - # This could be changed to `bundle install` if we wanted to actually - # install extra deps out of their gemfile ... - shellout!("#{bundle_bin} lock", env: env, cwd: installed_path) - # bundle lock doesn't always tell us when it fails, so we have to check :/ - unless File.exist?("#{installed_gemfile}.lock") - raise "bundle lock failed: no #{installed_gemfile}.lock created!" - end - - # Ensure all the gems we need are actually installed (if the bundle adds - # something, we need to know about it so we can include it in the main - # solve). - # Save bundle config and modify to use --without development before checking - bundle_config = File.expand_path("../.bundle/config", installed_gemfile) - orig_config = IO.read(bundle_config) if File.exist?(bundle_config) - # "test", "changelog" and "guard" come from berkshelf, "maintenance" comes from chef - # "tools" and "integration" come from inspec - shellout!("#{bundle_bin} config --local without #{without_groups.join(":")}", env: env, cwd: installed_path) - shellout!("#{bundle_bin} config --local frozen 1") - - shellout!("#{bundle_bin} check", env: env, cwd: installed_path) - - # Restore bundle config - if orig_config - create_file(bundle_config) { orig_config } - else - remove_file bundle_config - end - end - end - - # appbundle the gem, making /opt/chef/bin/<binary> do the superfast pinning - # thing. - # - # To protect the app from loading the wrong versions of things, it uses - # appbundler against the resulting file. - # - # Relocks the Gemfiles inside the specified gems (e.g. berkshelf, test-kitchen, - # chef) to use the distribution's chosen gems. - def appbundle_gem(gem_name) - # First lock the gemfile down. - lockdown_gem(gem_name) - - shared_gemfile = self.shared_gemfile - - # Ensure the main bin dir exists - bin_dir = File.join(install_dir, "bin") - mkdir(bin_dir) - - block "Lock down the #{gem_name} gem" do - installed_path = shellout!("#{bundle_bin} show #{gem_name}", env: env, cwd: install_dir).stdout.chomp - - # appbundle the gem - appbundler_args = [ installed_path, bin_dir, gem_name ] - appbundler_args = appbundler_args.map { |a| ::Shellwords.escape(a) } - shellout!("#{appbundler_bin} #{appbundler_args.join(" ")}", env: env, cwd: installed_path) - end - end -end diff --git a/omnibus/files/chef-gem/build-chef-gem.rb b/omnibus/files/chef-gem/build-chef-gem.rb deleted file mode 100644 index c9aaaada1d..0000000000 --- a/omnibus/files/chef-gem/build-chef-gem.rb +++ /dev/null @@ -1,128 +0,0 @@ -require "shellwords" -require "pathname" -require "bundler" -require_relative "../../../version_policy" - -# Common definitions and helpers (like compile environment and binary -# locations) for all software definitions. -module BuildChefGem - PLATFORM_FAMILY_FAMILIES = { - "linux" => %w{wrlinux debian fedora rhel suse gentoo slackware arch exherbo alpine}, - "bsd" => %w{dragonflybsd freebsd netbsd openbsd}, - "solaris" => %w{smartos omnios openindiana opensolaris solaris2 nextentacore}, - "aix" => %w{aix}, - "windows" => %w{windows}, - "mac_os_x" => %w{mac_os_x}, - } - def platform_family_families - PLATFORM_FAMILY_FAMILIES.keys - end - - def platform_family_family - PLATFORM_FAMILY_FAMILIES. - select { |key, families| families.include?(Omnibus::Ohai["platform_family"]) }. - first[0] - end - - def embedded_bin(binary) - windows_safe_path("#{install_dir}/embedded/bin/#{binary}") - end - - def appbundler_bin - embedded_bin("appbundler") - end - - def bundle_bin - embedded_bin("bundle") - end - - def gem_bin - embedded_bin("gem") - end - - def rake_bin - embedded_bin("rake") - end - - def without_groups - # Add --without for every known OS except the one we're in. - exclude_os_groups = platform_family_families - [ platform_family_family ] - (INSTALL_WITHOUT_GROUPS + exclude_os_groups).map { |g| g.to_sym } - end - - # - # Get the path to the top level shared Gemfile included by all individual - # Gemfiles - # - def shared_gemfile - File.join(install_dir, "Gemfile") - end - - # A common env for building everything including nokogiri and dep-selector-libgecode - def env - env = with_standard_compiler_flags(with_embedded_path, bfd_flags: true) - - # From dep-selector-libgecode - # On some RHEL-based systems, the default GCC that's installed is 4.1. We - # need to use 4.4, which is provided by the gcc44 and gcc44-c++ packages. - # These do not use the gcc binaries so we set the flags to point to the - # correct version here. - if File.exist?("/usr/bin/gcc44") - env["CC"] = "gcc44" - env["CXX"] = "g++44" - end - - if solaris_11? - env["CFLAGS"] << " -std=c99" - env["CPPFLAGS"] << " -D_XOPEN_SOURCE=600 -D_XPG6" - end - - # From dep-selector-libgecode - # Ruby DevKit ships with BSD Tar - env["PROG_TAR"] = "bsdtar" if windows? - env["ARFLAGS"] = "rv #{env["ARFLAGS"]}" if env["ARFLAGS"] - - # Set up nokogiri environment and args - env["NOKOGIRI_USE_SYSTEM_LIBRARIES"] = "true" - env - end - - # - # Install arguments for various gems (to be passed to `gem install` or set in - # `bundle config build.<gemname>`). - # - def all_install_args - @all_install_args = { - "nokogiri" => %W{ - --use-system-libraries - --with-xml2-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")} - --with-xml2-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxml2")} - --with-xslt-lib=#{Shellwords.escape("#{install_dir}/embedded/lib")} - --with-xslt-include=#{Shellwords.escape("#{install_dir}/embedded/include/libxslt")} - --without-iconv-dir - --with-zlib-dir=#{Shellwords.escape("#{install_dir}/embedded")} - }.join(" "), - } - end - - # gem install arguments for a particular gem. "" if no special args. - def install_args_for(gem_name) - all_install_args[gem_name] || "" - end - - # Give block all the variables - def block(*args, &block) - super do - extend BuildChefGem - instance_eval(&block) - end - end - - # Give build all the variables - def build(*args, &block) - super do - extend BuildChefGem - instance_eval(&block) - end - end -end diff --git a/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb b/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb deleted file mode 100644 index ea6c32e94a..0000000000 --- a/omnibus/files/chef-gem/build-chef-gem/gem-install-software-def.rb +++ /dev/null @@ -1,155 +0,0 @@ -require "bundler" -require "omnibus" -require_relative "../build-chef-gem" -require_relative "../../../../tasks/gemfile_util" - -module BuildChefGem - class GemInstallSoftwareDef - def self.define(software, software_filename) - new(software, software_filename).send(:define) - end - - include BuildChefGem - include Omnibus::Logging - - protected - - def initialize(software, software_filename) - @software = software - @software_filename = software_filename - end - - attr_reader :software, :software_filename - - # XXX: why are we programmatically defining config that is already expressed as code-as-configuration? - - def define - # this has to come first because gem_metadata depends on it - software.name "#{File.basename(software_filename)[0..-4]}" - if installing_from_git? - define_git - else - define_gem - end - end - - def define_git - software.default_version gem_metadata[:ref] - - # If the source directory for building stuff changes, tell omnibus to de-cache us - software.source git: gem_metadata[:git] - - # ruby and bundler and friends - software.dependency "ruby" - software.dependency "rubygems" - - software.relative_path gem_name - - gem_name = self.gem_name - - software.build do - extend BuildChefGem - gem "build #{gem_name}.gemspec", env: env - gem "install #{gem_name}*.gem --no-ri --no-rdoc", env: env - end - end - - def define_gem - software.default_version gem_version - - # If the source directory for building stuff changes, tell omnibus to - # de-cache us - software.source path: File.expand_path("../..", __FILE__) - - # ruby and bundler and friends - software.dependency "ruby" - software.dependency "rubygems" - - gem_name = self.gem_name - gem_version = self.gem_version - gem_metadata = self.gem_metadata - lockfile_path = self.lockfile_path - - software.build do - extend BuildChefGem - - if gem_version == "<skip>" - if gem_metadata - block do - raise "can we just remove this use case? what is it for?" - #log.info(log_key) { "#{gem_name} has source #{gem_metadata} in #{lockfile_path}. We only cache rubygems.org installs in omnibus to keep things simple. The chef step will build #{gem_name} ..." } - end - else - block do - log.info(log_key) { "#{gem_name} is not in the #{lockfile_path}. This can happen if your OS doesn't build it, or if chef no longer depends on it. Skipping ..." } - end - end - else - block do - log.info(log_key) { "Found version #{gem_version} of #{gem_name} in #{lockfile_path}. Building early to take advantage of omnibus caching ..." } - end - gem "install #{gem_name} -v #{gem_version} --no-doc --no-ri --ignore-dependencies --verbose -- #{install_args_for(gem_name)}", env: env - end - end - end - - # Path above omnibus (where Gemfile is) - def root_path - File.expand_path("../../../../..", __FILE__) - end - - def gemfile_path - File.join(root_path, "Gemfile") - end - - def lockfile_path - "#{gemfile_path}.lock" - end - - def gem_name - @gem_name ||= begin - # File must be named chef-<gemname>.rb - # Will look at chef/Gemfile.lock and install that version of the gem using "gem install" - # (and only that version) - if File.basename(software_filename) =~ /^chef-gem-(.+)\.rb$/ - $1 - else - raise "#{software_filename} must be named chef-<gemname>.rb to build a gem automatically" - end - end - end - - def gem_metadata - @gem_metadata ||= begin - bundle = GemfileUtil::Bundle.parse(gemfile_path, lockfile_path) - result = bundle.gems[gem_name] - if result - if bundle.select_gems(without_groups: without_groups).include?(gem_name) - log.info(software.name) { "Using #{gem_name} version #{result[:version]} from #{gemfile_path}" } - result - else - log.info(software.name) { "#{gem_name} not loaded from #{gemfile_path} because it was only in groups #{without_groups.join(", ")}. Skipping ..." } - nil - end - else - log.info(software.name) { "#{gem_name} was not found in #{lockfile_path}. Skipping ..." } - nil - end - end - end - - def installing_from_git? - gem_metadata && gem_metadata[:git] && gem_metadata[:ref] - end - - def gem_version - @gem_version ||= begin - if gem_metadata && URI(gem_metadata[:source]) == URI("https://rubygems.org/") - gem_metadata[:version] - else - "<skip>" - end - end - end - end -end diff --git a/omnibus/files/chef/build-chef.rb b/omnibus/files/chef/build-chef.rb deleted file mode 100644 index 4b8ec78054..0000000000 --- a/omnibus/files/chef/build-chef.rb +++ /dev/null @@ -1,127 +0,0 @@ -require "shellwords" -require "pathname" -require "bundler" -require_relative "../chef-gem/build-chef-gem" -require_relative "../../../version_policy" - -# We use this to break up the `build` method into readable parts -module BuildChef - include BuildChefGem - - def create_bundle_config(gemfile, without: without_groups, retries: nil, jobs: nil, frozen: nil) - bundle_config = File.expand_path("../.bundle/config", gemfile) - - block "Put build config into #{bundle_config}: #{{ without: without, retries: retries, jobs: jobs, frozen: frozen }}" do - # bundle config build.nokogiri #{nokogiri_build_config} messes up the line, - # so we write it directly ourselves. - new_bundle_config = "---\n" - new_bundle_config << "BUNDLE_WITHOUT: #{Array(without).join(":")}\n" if without - new_bundle_config << "BUNDLE_RETRY: #{retries}\n" if retries - new_bundle_config << "BUNDLE_JOBS: #{jobs}\n" if jobs - new_bundle_config << "BUNDLE_FROZEN: '1'\n" if frozen - all_install_args.each do |gem_name, install_args| - new_bundle_config << "BUNDLE_BUILD__#{gem_name.upcase}: #{install_args}\n" - end - create_file(bundle_config) { new_bundle_config } - end - end - - # - # Get the (possibly platform-specific) path to the Gemfile. - # - def project_gemfile - File.join(project_dir, "Gemfile") - end - - # - # Some gems we installed don't end up in the `gem list` due to the fact that - # they have git sources (`gem 'chef', github: 'chef/chef'`) or paths (`gemspec` - # or `gem 'chef-config', path: 'chef-config'`). To get them in there, we need - # to go through these gems, run `rake install` from their top level, and - # then delete the git cached versions. - # - # Once we finish with all this, we update the Gemfile that will end up in the - # top-level install so that it doesn't have git or path references anymore. - # - def properly_reinstall_git_and_path_sourced_gems - # Emit blank line to separate different tasks - block { log.info(log_key) { "" } } - project_env = env.dup.merge("BUNDLE_GEMFILE" => project_gemfile) - - # Reinstall git-sourced or path-sourced gems, and delete the originals - block "Reinstall git-sourced gems properly" do - # Grab info about the gem environment so we can make decisions - gemdir = shellout!("#{gem_bin} environment gemdir", env: env).stdout.chomp - gem_install_dir = File.join(gemdir, "gems") - - # bundle list --paths gets us the list of gem install paths. Get the ones - # that are installed local (git and path sources like `gem :x, github: 'chef/x'` - # or `gem :x, path: '.'` or `gemspec`). To do this, we just detect which ones - # have properly-installed paths (in the `gems` directory that shows up when - # you run `gem list`). - locally_installed_gems = shellout!("#{bundle_bin} list --paths", env: project_env, cwd: project_dir). - stdout.lines.select { |gem_path| !gem_path.start_with?(gem_install_dir) } - - # Install the gems for real using `rake install` in their directories - locally_installed_gems.each do |gem_path| - gem_path = gem_path.chomp - # We use the already-installed bundle to rake install, because (hopefully) - # just rake installing doesn't require anything special. - # Emit blank line to separate different tasks - log.info(log_key) { "" } - log.info(log_key) { "Properly installing git or path sourced gem #{gem_path} using rake install" } - shellout!("#{bundle_bin} exec #{rake_bin} install", env: project_env, cwd: gem_path) - end - end - end - - def install_shared_gemfile - # Emit blank line to separate different tasks - block { log.info(log_key) { "" } } - - shared_gemfile = self.shared_gemfile - project_env = env.dup.merge("BUNDLE_GEMFILE" => project_gemfile) - - # Show the config for good measure - bundle "config", env: project_env - - # Make `Gemfile` point to these by removing path and git sources and pinning versions. - block "Rewrite Gemfile using all properly-installed gems" do - gem_pins = "" - result = [] - shellout!("#{bundle_bin} list", env: project_env).stdout.lines.map do |line| - if line =~ /^\s*\*\s*(\S+)\s+\((\S+).*\)\s*$/ - name, version = $1, $2 - # rubocop is an exception, since different projects disagree - next if GEMS_ALLOWED_TO_FLOAT.include?(name) - gem_pins << "gem #{name.inspect}, #{version.inspect}, override: true\n" - end - end - - # Find the installed chef gem by looking for lib/chef.rb - chef_gem = File.expand_path("../..", shellout!("#{gem_bin} which chef", env: project_env).stdout.chomp) - # Figure out the path to gemfile_util from there - gemfile_util = Pathname.new(File.join(chef_gem, "tasks", "gemfile_util")) - gemfile_util = gemfile_util.relative_path_from(Pathname.new(shared_gemfile).dirname) - - create_file(shared_gemfile) { <<-EOM } - # Meant to be included in component Gemfiles at the beginning with: - # - # instance_eval(IO.read("#{install_dir}/Gemfile"), "#{install_dir}/Gemfile") - # - # Override any existing gems with our own. - require_relative "#{gemfile_util}" - extend GemfileUtil - #{gem_pins} - EOM - end - - shared_gemfile_env = env.dup.merge("BUNDLE_GEMFILE" => shared_gemfile) - - # Create a `Gemfile.lock` at the final location - bundle "lock", env: shared_gemfile_env - - # Freeze the location's Gemfile.lock. - create_bundle_config(shared_gemfile, frozen: true) - end -end diff --git a/tasks/bin/create-override-gemfile b/tasks/bin/create-override-gemfile deleted file mode 100755 index b67da025d2..0000000000 --- a/tasks/bin/create-override-gemfile +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env ruby - -require "rubygems" -require "bundler" - -Bundler.with_clean_env do - require_relative "../gemfile_util" - - options = {} - opts = OptionParser.new do |opts| - opts.banner = "Usage: create-override-gemfile [OPTIONS]" - - opts.on("--gemfile GEMFILE", "The Gemfile to read (default: Gemfile).") { |path| options[:gemfile_path] = path } - opts.on("--lockfile GEMFILE", "The lockfile to read (default: <gemfile>.lock).") { |path| options[:lockfile_path] = path } - - opts.on("--group GROUP", "Groups to include (whitelist).") do |group| - options[:groups] ||= [] - options[:groups] << group.to_sym - end - - opts.on("--without GROUP", "Groups to exclude.") do |group| - options[:without_groups] ||= [] - options[:without_groups] << group.to_sym - end - - opts.on("--gem GEM", "Gems to include regardless of groups.") do |name| - options[:gems] ||= [] - options[:gems] << name - end - - opts.on("--relative-to PATH", "A path to prepend to any relative paths in the Gemfile.") do |path| - unless Pathname.new(path).absolute? - puts opts - raise "--relative-to #{path} was not an absolute path!" - end - options[:relative_to] = path - end - - opts.on("--[no-]copy-groups", "Whether to copy groups over from the original Gemfile or not (default: false).") { |val| options[:copy_groups] = val } - - opts.on("--[no-]override", "Whether to emit override: true on each gem line (default: false).") { |val| options[:override] = val } - - opts.on("-h", "--help", "Print this message.") do - puts opts - exit(0) - end - end - - args = opts.parse(ARGV) - - if args.size > 0 - puts opts - raise "Invalid arguments #{args}" - end - - def create_override_gemfile(gemfile_path: "Gemfile", lockfile_path: "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false, relative_to: ".", override: false) - relative_to = Pathname.new(relative_to).realpath - # Select the gems we want - bundle = GemfileUtil::Bundle.parse(gemfile_path, lockfile_path) - gems_to_include = bundle.select_gems(groups: groups, without_groups: without_groups) - gems.each do |name| - raise "Requested gem #{name} is not in #{gemfile_path}.lock!" if !bundle.gems[name] - gems_to_include[name] ||= bundle.gems[name] - gems_to_include[name][:dependencies].each do |dep| - gems_to_include[name] ||= bundle.gems[dep] - end - end - - # Add the gems to the Gemfile - gem_root = Pathname.new(gemfile_path).dirname.realpath - gems_to_include.sort_by { |name, options| options[:declared_groups].empty? ? 1 : 0 }.each do |name, options| - comment = nil - options = options.dup - version = options.delete(:version) - if copy_groups - # Some dependencies have no groups (are not in the Gemfile--just runtime - # dependencies). If we actually record that they have no groups, they - # will *always* be installed (or perhaps never). We only want them to - # install if their other deps do, so we mark them with the groups of the - # things that brought them in (the gems that depended on them). To do - # this, we just leave :groups intact. - if options[:declared_groups].empty? - options.delete(:declared_groups) - comment = " # Transitive dependency, not actually in original Gemfile" - else - # For other things, we want to copy the actual declared_groups--the - # ones that were in the Gemfile. We want the same --with and --without - # options to include and exclude them as worked with the original - # Gemfile. - options[:groups] = options.delete(:declared_groups) - end - else - options.delete(:groups) - options.delete(:declared_groups) - end - options.delete(:dependencies) - options.delete(:development_dependencies) - options[:override] = true if override - options[:path] = Pathname.new(options[:path]).expand_path(gem_root).relative_path_from(relative_to).to_s if options[:path] - line = "gem #{name.inspect}, #{version.inspect}" - options.each do |name, value| - line << ", #{name}: #{value.inspect}" - end - line << comment if comment - puts line - end - end - - create_override_gemfile(options) -end diff --git a/tasks/dependencies.rb b/tasks/dependencies.rb index e6e11c0235..62c62149de 100644 --- a/tasks/dependencies.rb +++ b/tasks/dependencies.rb @@ -25,11 +25,13 @@ namespace :dependencies do # dependencies locally is by running the dependency update script. desc "Update all dependencies. dependencies:update to update as little as possible." task :update do |t, rake_args| + # FIXME: probably broken, and needs less indirection system("#{File.join(Dir.pwd, "ci", "dependency_update.sh")}") end desc "Force update (when adding new gems to Gemfiles)" task :force_update do |t, rake_args| + # FIXME: probably broken, and needs less indirection FileUtils.rm_f(File.join(Dir.pwd, ".bundle", "config")) system("#{File.join(Dir.pwd, "ci", "dependency_update.sh")}") end diff --git a/tasks/gemfile_util.rb b/tasks/gemfile_util.rb deleted file mode 100644 index 03a729148a..0000000000 --- a/tasks/gemfile_util.rb +++ /dev/null @@ -1,390 +0,0 @@ -require "rubygems" -require "bundler" -require "shellwords" -require "set" - -module GemfileUtil - # - # Adds `override: true`, which allows your statement to override any other - # gem statement about the same gem in the Gemfile. - # - def gem(name, *args) - options = args[-1].is_a?(Hash) ? args[-1] : {} - - # Unless we're finished with everything, ignore gems that are being overridden - unless overridden_gems == :finished - # If it's a path or override gem, it overrides whatever else is there. - if options[:path] || options[:override] - options.delete(:override) - warn_if_replacing(name, overridden_gems[name], args) - overridden_gems[name] = args - return - - # If there's an override gem, and we're *not* an override gem, don't do anything - elsif overridden_gems[name] - warn_if_replacing(name, args, overridden_gems[name]) - return - end - end - - # Otherwise, add the gem normally - super - rescue - puts $!.backtrace - raise - end - - def overridden_gems - @overridden_gems ||= {} - end - - # - # Just before we finish the Gemfile, finish up the override gems - # - def to_definition(*args) - complete_overrides - super - end - - def complete_overrides - to_override = overridden_gems - unless to_override == :finished - @overridden_gems = :finished - to_override.each do |name, args| - gem name, *args - end - end - end - - # - # Include all gems in the locked gemfile. - # - # @param gemfile_path Path to the Gemfile to load (relative to your Gemfile) - # @param lockfile_path Path to the Gemfile to load (relative to your Gemfile). - # Defaults to <gemfile_path>.lock. - # @param groups A list of groups to include (whitelist). If not passed (or set - # to nil), all gems will be selected. - # @param without_groups A list of groups to ignore. Gems will be excluded from - # the results if all groups they belong to are ignored. This matches - # bundler's `without` behavior. - # @param gems A list of gems to include above and beyond the given groups. - # Gems in this list must be explicitly included in the Gemfile - # with a `gem "gem_name", ...` line or they will be silently - # ignored. - # @param copy_groups Whether to copy the groups over from the old lockfile to - # the new. Use this when the new lockfile has the same convention for - # groups as the old. Defaults to `false`. - # - def include_locked_gemfile(gemfile_path, lockfile_path = "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false) - # Parse the desired lockfile - gemfile_path = Pathname.new(gemfile_path).expand_path(Bundler.default_gemfile.dirname).realpath - lockfile_path = Pathname.new(lockfile_path).expand_path(Bundler.default_gemfile.dirname).realpath - - # Calculate relative_to - relative_to = Bundler.default_gemfile.dirname.realpath - - # Call out to create-override-gemfile to read the Gemfile+Gemfile.lock (bundler does not work well if you do two things in one process) - create_override_gemfile_bin = File.expand_path("../bin/create-override-gemfile", __FILE__) - arguments = [ - "--gemfile", gemfile_path, - "--lockfile", lockfile_path, - "--override" - ] - arguments += [ "--relative-to", relative_to ] if relative_to != "." - arguments += Array(groups).flat_map { |group| [ "--group", group ] } - arguments += Array(without_groups).flat_map { |without| [ "--without", without ] } - arguments += Array(gems).flat_map { |name| [ "--gem", name ] } - arguments << "--copy-groups" if copy_groups - cmd = Shellwords.join([ Gem.ruby, "-S", create_override_gemfile_bin, *arguments ]) - output = nil - Bundler.ui.info("> #{cmd}") - Bundler.with_clean_env do - output = `#{cmd}` - end - instance_eval(output, cmd, 1) - end - - # - # Include all gems in the locked gemfile. - # - # @param current_gemfile The Gemfile you are currently loading (`self`). - # @param gemfile_path Path to the Gemfile to load (relative to your Gemfile) - # @param lockfile_path Path to the Gemfile to load (relative to your Gemfile). - # Defaults to <gemfile_path>.lock. - # @param groups A list of groups to include (whitelist). If not passed (or set - # to nil), all gems will be selected. - # @param without_groups A list of groups to ignore. Gems will be excluded from - # the results if all groups they belong to are ignored. This matches - # bundler's `without` behavior. - # @param gems A list of gems to include above and beyond the given groups. - # Gems in this list must be explicitly included in the Gemfile - # with a `gem "gem_name", ...` line or they will be silently - # ignored. - # @param copy_groups Whether to copy the groups over from the old lockfile to - # the new. Use this when the new lockfile has the same convention for - # groups as the old. Defaults to `false`. - # - def self.include_locked_gemfile(current_gemfile, gemfile_path, lockfile_path = "#{gemfile_path}.lock", groups: nil, without_groups: nil, gems: [], copy_groups: false) - current_gemfile.instance_eval do - extend GemfileUtil - include_locked_gemfile(gemfile_path, lockfile_path, groups: groups, without_groups: without_groups, gems: gems, copy_groups: copy_groups) - end - end - - def warn_if_replacing(name, old_args, new_args) - return if !old_args || !new_args - if args_to_dep(name, *old_args) =~ args_to_dep(name, *new_args) - Bundler.ui.debug "Replaced Gemfile dependency #{name} (#{old_args}) with (#{new_args})" - else - Bundler.ui.warn "Replaced Gemfile dependency #{name} (#{old_args}) with (#{new_args})" - end - end - - def args_to_dep(name, *version, **options) - version = [">= 0"] if version.empty? - Bundler::Dependency.new(name, version, options) - end - - # - # Reads a bundle, including a gemfile and lockfile. - # - # Does no validation, does not update the lockfile or its gems in any way. - # - class Bundle - # - # Parse the given gemfile/lockfile pair. - # - # @return [Bundle] The parsed bundle. - # - def self.parse(gemfile_path, lockfile_path = "#{gemfile_path}.lock") - result = new(gemfile_path, lockfile_path) - result.gems - result - end - - # - # Create a new Bundle to parse the given gemfile/lockfile pair. - # - def initialize(gemfile_path, lockfile_path = "#{gemfile_path}.lock") - @gemfile_path = gemfile_path - @lockfile_path = lockfile_path - end - - # - # The path to the Gemfile - # - attr_reader :gemfile_path - - # - # The path to the Lockfile - # - attr_reader :lockfile_path - - # - # The list of gems. - # - # @return [Hash<String, Hash>] The resulting gems, where key = gem_name, and the - # hash has: - # - version: version of the gem. - # - source info (:source/:git/:ref/:path) from the lockfile - # - dependencies: A list of gem names this gem has a runtime - # dependency on. Dependencies are transitive: if A depends on B, - # and B depends on C, then A has C in its :dependencies list. - # - development_dependencies: - A list of gem names this gem has a - # development dependency on. Dependencies are transitive: if A - # depends on B, and B depends on C, then A has C in its - # :development_dependencies list. development dependencies *include* - # runtime dependencies. - # - groups: The list of groups (symbols) this gem is in. Groups - # are transitive: if A has a runtime dependency on B, and A is - # in group X, then B is also in group X. - # - declared_groups: The list of groups (symbols) this gem was - # declared in the Gemfile. - # - def gems - @gems ||= begin - gems = locks.dup - gems.each do |name, g| - if gem_declarations.has_key?(name) - g[:declared_groups] = gem_declarations[name][:groups] - else - g[:declared_groups] = [] - end - g[:groups] = g[:declared_groups].dup - end - # Transitivize groups (since dependencies are already transitive, this is easy) - gems.each do |name, g| - g[:dependencies].each do |dep| - gems[dep][:groups] |= gems[name][:declared_groups].dup - end - end - gems - end - end - - # - # Get the gems (and their deps) in the given group. - # - # @param groups A list of groups to include (whitelist). If not passed (or set - # to nil), all gems will be selected. - # @param without_groups A list of groups to ignore. Gems will be excluded from - # the results if all groups they belong to are ignored. - # This matches bundler's `without` behavior. - # @param gems A list of gems to include regardless of what groups are included. - # - # @return Hash[String, Hash] The resulting gems, where key = gem_name, and the - # hash has: - # - version: version of the gem. - # - source info (:source/:git/:ref/:path) from the lockfile - # - dependencies: A list of gem names this gem has a runtime - # dependency on. Dependencies are transitive: if A depends on B, - # and B depends on C, then A has C in its :dependencies list. - # - development_dependencies: - A list of gem names this gem has a - # development dependency on. Dependencies are transitive: if A - # depends on B, and B depends on C, then A has C in its - # :development_dependencies list. development dependencies - # *include* runtime dependencies. - # - groups: The list of groups (symbols) this gem is in. Groups - # are transitive: if A has a runtime dependency on B, and A is - # in group X, then B is also in group X. - # - declared_groups: The list of groups (symbols) this gem was - # declared in the Gemfile. - # - def select_gems(groups: nil, without_groups: nil) - # First, select the gems that match - result = {} - gems.each do |name, g| - dep_groups = g[:declared_groups] - [ :only_a_runtime_dependency_of_other_gems ] - dep_groups &= groups if groups - dep_groups -= without_groups if without_groups - if dep_groups.any? - result[name] ||= g - g[:dependencies].each do |dep| - result[dep] ||= gems[dep] - end - end - end - result - end - - # - # Get all locks from the given lockfile. - # - # @return Hash[String, Hash] The resulting gems, where key = gem_name, and the - # hash has: - # - version: version of the gem. - # - source info (:source/:git/:ref/:path) - # - dependencies: A list of gem names this gem has a runtime - # dependency on. Dependencies are transitive: if A depends on B, - # and B depends on C, then A has C in its :dependencies list. - # - development_dependencies: - A list of gem names this gem has a - # development dependency on. Dependencies are transitive: if A - # depends on B, and B depends on C, then A has C in its - # :development_dependencies list. development dependencies *include* - # runtime dependencies. - # - def locks - @locks ||= begin - # Grab all the specs from the lockfile - locks = {} - parsed_lockfile = Bundler::LockfileParser.new(IO.read(lockfile_path)) - parsed_lockfile.specs.each do |spec| - # Never include bundler, it can't be bundled and doesn't put itself in - # the lockfile correctly anyway - next if spec.name == "bundler" - # Only the platform-specific locks for now (TODO make it possible to emit all locks) - next if spec.platform && spec.platform != Gem::Platform::RUBY - lock = lock_source_metadata(spec) - lock[:version] = spec.version.to_s - runtime = spec.dependencies.select { |dep| dep.type == :runtime } - lock[:dependencies] = Set.new(runtime.map { |dep| dep.name }) - lock[:development_dependencies] = Set.new(spec.dependencies.map { |dep| dep.name }) - lock[:dependencies].delete("bundler") - lock[:development_dependencies].delete("bundler") - locks[spec.name] = lock - end - - # Transitivize the deps. - locks.each do |name, lock| - # Not all deps were brought over (platform-specific ones) so weed them out - lock[:dependencies] &= locks.keys - lock[:development_dependencies] &= locks.keys - - lock[:dependencies] = transitive_dependencies(locks, name, :dependencies) - lock[:development_dependencies] = transitive_dependencies(locks, name, :development_dependencies) - end - - locks - end - end - - # - # Get all desired gems, sans dependencies, from the gemfile. - # - # @param gemfile Path to the Gemfile to load - # - # @return Hash<String, Hash> An array of hashes where key = gem name and value - # has :groups (an array of symbols representing the groups the gem - # is in). :groups are not transitive, since we don't know the - # dependency tree yet. - # - def gem_declarations - @gem_declarations ||= begin - Bundler.with_clean_env do - # Set BUNDLE_GEMFILE to the new gemfile temporarily so all bundler's things work - # This works around some issues in bundler 1.11.2. - ENV["BUNDLE_GEMFILE"] = gemfile_path - - parsed_gemfile = Bundler::Dsl.new - parsed_gemfile.eval_gemfile(gemfile_path) - parsed_gemfile.complete_overrides if parsed_gemfile.respond_to?(:complete_overrides) - - result = {} - parsed_gemfile.dependencies.each do |dep| - groups = dep.groups.empty? ? [:default] : dep.groups - result[dep.name] = { groups: groups, platforms: dep.platforms } - end - result - end - end - end - - private - - # - # Given a bunch of locks (name -> { dependencies: [name,name] }) and a - # dependency name, add its dependencies to the result transitively. - # - def transitive_dependencies(locks, name, dep_key, result = Set.new) - locks[name][dep_key].each do |dep| - # Only ever add a dep once, so we don't infinitely recurse - if result.add?(dep) - transitive_dependencies(locks, dep, dep_key, result) - end - end - result - end - - # - # Get source and version metadata for the given Bundler spec (coming from a lockfile). - # - # @return Hash { version: <version>, git: <git>, path: <path>, source: <source>, ref: <ref> } - # - def lock_source_metadata(spec) - # Copy source information from included Gemfile - result = {} - case spec.source - when Bundler::Source::Rubygems - result[:source] = spec.source.remotes.first.to_s - when Bundler::Source::Git - result[:git] = spec.source.uri.to_s - result[:ref] = spec.source.revision - when Bundler::Source::Path - result[:path] = spec.source.path.to_s - else - raise "Unknown source #{spec.source} for gem #{spec.name}" - end - result - end - end -end |