rubygems memory perf issueslcg/rubygems_perf2

set rubygems_cache_enabled to true in Chef::Config to get the old behavior back

Signed-off-by: Lamont Granquist <lamont@scriptkiddie.org>

3 files changed, 67 insertions, 51 deletions

diff --git a/chef-config/lib/chef-config/config.rb b/chef-config/lib/chef-config/config.rb
index a194edc80e..fce72e28ff 100644
--- a/chef-config/lib/chef-config/config.rb
+++ b/chef-config/lib/chef-config/config.rb
@@ -791,6 +791,11 @@ module ChefConfig
     default :normal_attribute_whitelist, nil
     default :override_attribute_whitelist, nil
 
+    # Pull down all the rubygems versions from rubygems and cache them the first time we do a gem_package or
+    # chef_gem install.  This is memory-expensive and will grow without bounds, but will reduce network
+    # round trips.
+    default :rubygems_cache_enabled, false
+
     config_context :windows_service do
       # Set `watchdog_timeout` to the number of seconds to wait for a chef-client run
       # to finish
diff --git a/lib/chef/provider/package/rubygems.rb b/lib/chef/provider/package/rubygems.rb
index 0aeec951b1..0fd9373dbf 100644
--- a/lib/chef/provider/package/rubygems.rb
+++ b/lib/chef/provider/package/rubygems.rb
@@ -159,19 +159,22 @@ class Chef
           # Find the newest gem version available from Gem.sources that satisfies
           # the constraints of +gem_dependency+
           def find_newest_remote_version(gem_dependency, *sources)
-            available_gems = dependency_installer.find_gems_with_sources(gem_dependency)
-            spec, source = if available_gems.respond_to?(:last)
-                             # DependencyInstaller sorts the results such that the last one is
-                             # always the one it considers best.
-                             spec_with_source = available_gems.last
-                             spec_with_source && spec_with_source
-                           else
-                             # Rubygems 2.0 returns a Gem::Available set, which is a
-                             # collection of AvailableSet::Tuple structs
-                             available_gems.pick_best!
-                             best_gem = available_gems.set.first
-                             best_gem && [best_gem.spec, best_gem.source]
-                           end
+            spec, source =
+              if Chef::Config[:rubygems_cache_enabled]
+                # This code caches every gem on rubygems.org and uses lots of RAM
+                available_gems = dependency_installer.find_gems_with_sources(gem_dependency)
+                available_gems.pick_best!
+                best_gem = available_gems.set.first
+                best_gem && [best_gem.spec, best_gem.source]
+              else
+                # Use the API that 'gem install' calls which does not pull down the rubygems universe
+                begin
+                  rs = dependency_installer.resolve_dependencies gem_dependency.name, gem_dependency.requirement
+                  rs.specs.first
+                rescue Gem::UnsatisfiableDependencyError
+                  nil
+                end
+              end
 
             version = spec && spec.version
             if version
diff --git a/spec/unit/provider/package/rubygems_spec.rb b/spec/unit/provider/package/rubygems_spec.rb
index f87c261ec0..ed109bf20f 100644
--- a/spec/unit/provider/package/rubygems_spec.rb
+++ b/spec/unit/provider/package/rubygems_spec.rb
@@ -16,7 +16,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-require "pp"
 
 module GemspecBackcompatCreator
   def gemspec(name, version)
@@ -86,34 +85,55 @@ describe Chef::Provider::Package::Rubygems::CurrentGemEnvironment do
     expect(Gem.sources).to eq(normal_sources)
   end
 
-  it "finds a matching gem candidate version" do
-    dep = Gem::Dependency.new("rspec", ">= 0")
-    dep_installer = Gem::DependencyInstaller.new
-    allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
-    latest = [[gemspec("rspec", Gem::Version.new("1.3.0")), "https://rubygems.org/"]]
-    expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(latest)
-    expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">= 0"))).to eq(Gem::Version.new("1.3.0"))
-  end
+  context "new default rubygems behavior" do
+    before do
+      Chef::Config[:rubygems_cache_enabled] = false
+    end
 
-  it "finds a matching gem candidate version on rubygems 2.0.0+" do
-    dep = Gem::Dependency.new("rspec", ">= 0")
-    dep_installer = Gem::DependencyInstaller.new
-    allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
-    best_gem = double("best gem match", :spec => gemspec("rspec", Gem::Version.new("1.3.0")), :source => "https://rubygems.org")
-    available_set = double("Gem::AvailableSet test double")
-    expect(available_set).to receive(:pick_best!)
-    expect(available_set).to receive(:set).and_return([best_gem])
-    expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(available_set)
-    expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">= 0"))).to eq(Gem::Version.new("1.3.0"))
+    it "finds a matching gem candidate version on rubygems 2.0.0+" do
+      dep = Gem::Dependency.new("rspec", ">= 0")
+      dep_installer = Gem::DependencyInstaller.new
+      allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
+      expect(dep_installer).not_to receive(:find_gems_with_sources).with(dep).and_call_original
+      expect(@gem_env.candidate_version_from_remote(dep)).to be_kind_of(Gem::Version)
+    end
+
+    it "gives the candidate version as nil if none is found" do
+      dep = Gem::Dependency.new("lksdjflksdjflsdkfj", ">= 0")
+      dep_installer = Gem::DependencyInstaller.new
+      allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
+      expect(dep_installer).not_to receive(:find_gems_with_sources).with(dep).and_call_original
+      expect(@gem_env.candidate_version_from_remote(dep)).to be_nil
+    end
+
+    it "finds a matching gem from a specific gemserver when explicit sources are given (to a server that doesn't respond to api requests)" do
+      dep = Gem::Dependency.new("rspec", ">= 0")
+      dep_installer = Gem::DependencyInstaller.new
+      allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
+      expect(dep_installer).not_to receive(:find_gems_with_sources).with(dep).and_call_original
+      expect(@gem_env.candidate_version_from_remote(dep, "http://production.cf.rubygems.org")).to be_kind_of(Gem::Version)
+    end
   end
 
-  it "gives the candidate version as nil if none is found" do
-    dep = Gem::Dependency.new("rspec", ">= 0")
-    latest = []
-    dep_installer = Gem::DependencyInstaller.new
-    allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
-    expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(latest)
-    expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">= 0"))).to be_nil
+  context "old rubygems caching behavior" do
+    before do
+      Chef::Config[:rubygems_cache_enabled] = true
+    end
+
+    it "finds a matching gem candidate version on rubygems 2.0.0+" do
+      dep = Gem::Dependency.new("rspec", ">= 0")
+      expect(@gem_env.candidate_version_from_remote(dep)).to be_kind_of(Gem::Version)
+    end
+
+    it "gives the candidate version as nil if none is found" do
+      dep = Gem::Dependency.new("lksdjflksdjflsdkfj", ">= 0")
+      expect(@gem_env.candidate_version_from_remote(dep)).to be_nil
+    end
+
+    it "finds a matching gem from a specific gemserver when explicit sources are given" do
+      dep = Gem::Dependency.new("rspec", ">= 0")
+      expect(@gem_env.candidate_version_from_remote(dep, "http://production.cf.rubygems.org")).to be_kind_of(Gem::Version)
+    end
   end
 
   it "finds a matching candidate version from a .gem file when the path to the gem is supplied" do
@@ -122,17 +142,6 @@ describe Chef::Provider::Package::Rubygems::CurrentGemEnvironment do
     expect(@gem_env.candidate_version_from_file(Gem::Dependency.new("chef-integration-test", ">= 0.2.0"), location)).to be_nil
   end
 
-  it "finds a matching gem from a specific gemserver when explicit sources are given" do
-    dep = Gem::Dependency.new("rspec", ">= 0")
-    latest = [[gemspec("rspec", Gem::Version.new("1.3.0")), "https://rubygems.org/"]]
-
-    expect(@gem_env).to receive(:with_gem_sources).with("http://gems.example.com").and_yield
-    dep_installer = Gem::DependencyInstaller.new
-    allow(@gem_env).to receive(:dependency_installer).and_return(dep_installer)
-    expect(dep_installer).to receive(:find_gems_with_sources).with(dep).and_return(latest)
-    expect(@gem_env.candidate_version_from_remote(Gem::Dependency.new("rspec", ">=0"), "http://gems.example.com")).to eq(Gem::Version.new("1.3.0"))
-  end
-
   it "installs a gem with a hash of options for the dependency installer" do
     dep_installer = Gem::DependencyInstaller.new
     expect(@gem_env).to receive(:dependency_installer).with(:install_dir => "/foo/bar").and_return(dep_installer)
@@ -545,7 +554,6 @@ describe Chef::Provider::Package::Rubygems do
           expect(provider.candidate_version).to eq("0.1.0")
         end
       end
-
     end
 
     describe "when installing a gem" do