From d0bb671cabd8ef5d60eb2f2d1c22d9d423b5921b Mon Sep 17 00:00:00 2001 From: Lamont Granquist Date: Mon, 15 Aug 2016 12:44:59 -0700 Subject: rubygems memory perf issues set rubygems_cache_enabled to true in Chef::Config to get the old behavior back Signed-off-by: Lamont Granquist --- chef-config/lib/chef-config/config.rb | 5 ++ lib/chef/provider/package/rubygems.rb | 29 +++++----- spec/unit/provider/package/rubygems_spec.rb | 84 ++++++++++++++++------------- 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb index a194edc80e..fce72e28ff 100644 --- a/chef-config/lib/chef-config/config.rb +++ b/chef-config/lib/chef-config/config.rb @@ -791,6 +791,11 @@ module ChefConfig default :normal_attribute_whitelist, nil default :override_attribute_whitelist, nil + # Pull down all the rubygems versions from rubygems and cache them the first time we do a gem_package or + # chef_gem install. This is memory-expensive and will grow without bounds, but will reduce network + # round trips. + default :rubygems_cache_enabled, false + config_context :windows_service do # Set `watchdog_timeout` to the number of seconds to wait for a chef-client run # to finish diff --git a/lib/chef/provider/package/rubygems.rb b/lib/chef/provider/package/rubygems.rb index 0aeec951b1..0fd9373dbf 100644 --- a/lib/chef/provider/package/rubygems.rb +++ b/lib/chef/provider/package/rubygems.rb @@ -159,19 +159,22 @@ class Chef # Find the newest gem version available from Gem.sources that satisfies # the constraints of +gem_dependency+ def find_newest_remote_version(gem_dependency, *sources) - available_gems = dependency_installer.find_gems_with_sources(gem_dependency) - spec, source = if available_gems.respond_to?(:last) - # DependencyInstaller sorts the results such that the last one is - # always the one it considers best. - spec_with_source = available_gems.last - spec_with_source && spec_with_source - else - # Rubygems 2.0 returns a Gem::Available set, which is a - # collection of AvailableSet::Tuple structs - available_gems.pick_best! - best_gem = available_gems.set.first - best_gem && [best_gem.spec, best_gem.source] - end + spec, source = + if Chef::Config[:rubygems_cache_enabled] + # This code caches every gem on rubygems.org and uses lots of RAM + available_gems = dependency_installer.find_gems_with_sources(gem_dependency) + available_gems.pick_best! + best_gem = available_gems.set.first + best_gem && [best_gem.spec, best_gem.source] + else + # Use the API that 'gem install' calls which does not pull down the rubygems universe + begin + rs = dependency_installer.resolve_dependencies gem_dependency.name, gem_dependency.requirement + rs.specs.first + rescue Gem::UnsatisfiableDependencyError + nil + end + end version = spec && spec.version if version diff --git a/spec/unit/provider/package/rubygems_spec.rb b/spec/unit/provider/package/rubygems_spec.rb index f87c261ec0..ed109bf20f 100644 --- a/spec/unit/provider/package/rubygems_spec.rb +++ b/spec/unit/provider/package/rubygems_spec.rb @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -require "pp" module GemspecBackcompatCreator def gemspec(name, version) @@ -86,34 +85,55 @@ describe Chef::Provider::Package::Rubygems::CurrentGemEnvironment do expect(Gem.sources).to eq(normal_sources) end - it "finds a matching gem candidate version" do - dep = Gem::Dependency.new("rspec", ">= 0") - dep_installer = Gem::DependencyInstaller.new - allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) - latest = [[gemspec("rspec", Gem::Version.new("1.3.0")), "https://rubygems.org/"]] - expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(latest) - expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">= 0"))).to eq(Gem::Version.new("1.3.0")) - end + context "new default rubygems behavior" do + before do + Chef::Config[:rubygems_cache_enabled] = false + end - it "finds a matching gem candidate version on rubygems 2.0.0+" do - dep = Gem::Dependency.new("rspec", ">= 0") - dep_installer = Gem::DependencyInstaller.new - allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) - best_gem = double("best gem match", :spec => gemspec("rspec", Gem::Version.new("1.3.0")), :source => "https://rubygems.org") - available_set = double("Gem::AvailableSet test double") - expect(available_set).to receive(:pick_best!) - expect(available_set).to receive(:set).and_return([best_gem]) - expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(available_set) - expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">= 0"))).to eq(Gem::Version.new("1.3.0")) + it "finds a matching gem candidate version on rubygems 2.0.0+" do + dep = Gem::Dependency.new("rspec", ">= 0") + dep_installer = Gem::DependencyInstaller.new + allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) + expect(dep_installer).not_to receive(:find_gems_with_sources).with(dep).and_call_original + expect(@gem_env.candidate_version_from_remote(dep)).to be_kind_of(Gem::Version) + end + + it "gives the candidate version as nil if none is found" do + dep = Gem::Dependency.new("lksdjflksdjflsdkfj", ">= 0") + dep_installer = Gem::DependencyInstaller.new + allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) + expect(dep_installer).not_to receive(:find_gems_with_sources).with(dep).and_call_original + expect(@gem_env.candidate_version_from_remote(dep)).to be_nil + end + + it "finds a matching gem from a specific gemserver when explicit sources are given (to a server that doesn't respond to api requests)" do + dep = Gem::Dependency.new("rspec", ">= 0") + dep_installer = Gem::DependencyInstaller.new + allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) + expect(dep_installer).not_to receive(:find_gems_with_sources).with(dep).and_call_original + expect(@gem_env.candidate_version_from_remote(dep, "http://production.cf.rubygems.org")).to be_kind_of(Gem::Version) + end end - it "gives the candidate version as nil if none is found" do - dep = Gem::Dependency.new("rspec", ">= 0") - latest = [] - dep_installer = Gem::DependencyInstaller.new - allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) - expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(latest) - expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">= 0"))).to be_nil + context "old rubygems caching behavior" do + before do + Chef::Config[:rubygems_cache_enabled] = true + end + + it "finds a matching gem candidate version on rubygems 2.0.0+" do + dep = Gem::Dependency.new("rspec", ">= 0") + expect(@gem_env.candidate_version_from_remote(dep)).to be_kind_of(Gem::Version) + end + + it "gives the candidate version as nil if none is found" do + dep = Gem::Dependency.new("lksdjflksdjflsdkfj", ">= 0") + expect(@gem_env.candidate_version_from_remote(dep)).to be_nil + end + + it "finds a matching gem from a specific gemserver when explicit sources are given" do + dep = Gem::Dependency.new("rspec", ">= 0") + expect(@gem_env.candidate_version_from_remote(dep, "http://production.cf.rubygems.org")).to be_kind_of(Gem::Version) + end end it "finds a matching candidate version from a .gem file when the path to the gem is supplied" do @@ -122,17 +142,6 @@ describe Chef::Provider::Package::Rubygems::CurrentGemEnvironment do expect(@gem_env.candidate_version_from_file(Gem::Dependency.new("chef-integration-test", ">= 0.2.0"), location)).to be_nil end - it "finds a matching gem from a specific gemserver when explicit sources are given" do - dep = Gem::Dependency.new("rspec", ">= 0") - latest = [[gemspec("rspec", Gem::Version.new("1.3.0")), "https://rubygems.org/"]] - - expect(@gem_env).to receive(:with_gem_sources).with("http://gems.example.com").and_yield - dep_installer = Gem::DependencyInstaller.new - allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer) - expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(latest) - expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">=0"), "http://gems.example.com")).to eq(Gem::Version.new("1.3.0")) - end - it "installs a gem with a hash of options for the dependency installer" do dep_installer = Gem::DependencyInstaller.new expect(@gem_env).to receive(:dependency_installer).with(:install_dir => "/foo/bar").and_return(dep_installer) @@ -545,7 +554,6 @@ describe Chef::Provider::Package::Rubygems do expect(provider.candidate_version).to eq("0.1.0") end end - end describe "when installing a gem" do -- cgit v1.2.1