Auto merge of #5782 - bundler:seg-global-gem-cache, r=indirect

[2.0] Add a global cache for downloaded .gem files

### What was the end-user problem that led to this PR?

The problem was that bundler would need to download `foo-1.0.gem` files from a RubyGems server for each different ruby version installed on a user's machine. It also meant that people installing into a per-app path would need to re-download every gem for that bundle an additional time. This adds up, and makes `bundle install` slower than it needs to be.

### Was was your diagnosis of the problem?

My diagnosis was that Bundler could keep a (per-source) cache of these `.gem` files, and pull from that cache instead of hitting the network whenever possible.

### What is your fix for the problem, implemented in this PR?

My fix implements said cache, in a very similar way to the compact index cache (same cache slug per remote strategy, etc). This largely comes from https://github.com/bundler/bundler/pull/3983.

### Why did you choose this fix out of the possible options?

I chose this fix because it is safe when used from multi-source gemfiles, it is easy to clear (`rm -rf bundle cache`), and it minimally interferes with the existing installation process.

11 files changed, 301 insertions, 16 deletions

diff --git a/lib/bundler.rb b/lib/bundler.rb
index c6d68c49de..c9d8846ce5 100644
--- a/lib/bundler.rb
+++ b/lib/bundler.rb
@@ -216,8 +216,8 @@ module Bundler
     end
 
     def app_config_path
-      if ENV["BUNDLE_APP_CONFIG"]
-        Pathname.new(ENV["BUNDLE_APP_CONFIG"]).expand_path(root)
+      if app_config = ENV["BUNDLE_APP_CONFIG"]
+        Pathname.new(app_config).expand_path(root)
       else
         root.join(".bundle")
       end
diff --git a/lib/bundler/feature_flag.rb b/lib/bundler/feature_flag.rb
index ee93773c2f..b2b1ef3f99 100644
--- a/lib/bundler/feature_flag.rb
+++ b/lib/bundler/feature_flag.rb
@@ -29,6 +29,7 @@ module Bundler
     settings_flag(:allow_bundler_dependency_conflicts) { bundler_2_mode? }
     settings_flag(:allow_offline_install) { bundler_2_mode? }
     settings_flag(:error_on_stderr) { bundler_2_mode? }
+    settings_flag(:global_gem_cache) { bundler_2_mode? }
     settings_flag(:init_gems_rb) { bundler_2_mode? }
     settings_flag(:only_update_to_newer_versions) { bundler_2_mode? }
     settings_flag(:plugins) { @bundler_version >= Gem::Version.new("1.14") }
diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb
index 97de88101b..ec20f18edd 100644
--- a/lib/bundler/fetcher/compact_index.rb
+++ b/lib/bundler/fetcher/compact_index.rb
@@ -61,7 +61,7 @@ module Bundler
       compact_index_request :fetch_spec
 
       def available?
-        return nil unless md5_available?
+        return nil unless SharedHelpers.md5_available?
         user_home = Bundler.user_home
         return nil unless user_home.directory? && user_home.writable?
         # Read info file checksums out of /versions, so we can know if gems are up to date
@@ -120,16 +120,6 @@ module Bundler
           Net::HTTPNotModified.new(nil, nil, nil)
         end
       end
-
-      def md5_available?
-        require "openssl"
-        OpenSSL::Digest::MD5.digest("")
-        true
-      rescue LoadError
-        true
-      rescue OpenSSL::Digest::DigestError
-        false
-      end
     end
   end
 end
diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb
index 078c517919..d91f697f70 100644
--- a/lib/bundler/settings.rb
+++ b/lib/bundler/settings.rb
@@ -22,6 +22,7 @@ module Bundler
       frozen
       gem.coc
       gem.mit
+      global_gem_cache
       ignore_messages
       init_gems_rb
       major_deprecations
diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb
index 01433a2096..339ddeb7de 100644
--- a/lib/bundler/shared_helpers.rb
+++ b/lib/bundler/shared_helpers.rb
@@ -187,6 +187,19 @@ module Bundler
       msg
     end
 
+    def md5_available?
+      return @md5_available if defined?(@md5_available)
+      @md5_available = begin
+        require "openssl"
+        OpenSSL::Digest::MD5.digest("")
+        true
+      rescue LoadError
+        true
+      rescue OpenSSL::Digest::DigestError
+        false
+      end
+    end
+
   private
 
     def find_gemfile(order_matters = false)
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index a1d4266960..70ec8c60f8 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -411,7 +411,7 @@ module Bundler
 
       def fetch_gem(spec)
         return false unless spec.remote
-        uri = spec.remote.uri
+
         spec.fetch_platform
         Bundler.ui.confirm("Fetching #{version_message(spec)}")
 
@@ -421,7 +421,7 @@ module Bundler
         SharedHelpers.filesystem_access("#{download_path}/cache") do |p|
           FileUtils.mkdir_p(p)
         end
-        Bundler.rubygems.download_gem(spec, uri, download_path)
+        download_gem(spec, download_path)
 
         if requires_sudo?
           SharedHelpers.filesystem_access("#{rubygems_dir}/cache") do |p|
@@ -458,6 +458,70 @@ module Bundler
       def cache_path
         Bundler.app_cache
       end
+
+    private
+
+      # Checks if the requested spec exists in the global cache. If it does,
+      # we copy it to the download path, and if it does not, we download it.
+      #
+      # @param  [Specification] spec
+      #         the spec we want to download or retrieve from the cache.
+      #
+      # @param  [String] download_path
+      #         the local directory the .gem will end up in.
+      #
+      def download_gem(spec, download_path)
+        local_path = File.join(download_path, "cache/#{spec.full_name}.gem")
+
+        if (cache_path = download_cache_path(spec)) && cache_path.file?
+          SharedHelpers.filesystem_access(local_path) do
+            FileUtils.cp(cache_path, local_path)
+          end
+        else
+          uri = spec.remote.uri
+          Bundler.rubygems.download_gem(spec, uri, download_path)
+          cache_globally(spec, local_path)
+        end
+      end
+
+      # Checks if the requested spec exists in the global cache. If it does
+      # not, we create the relevant global cache subdirectory if it does not
+      # exist and copy the spec from the local cache to the global cache.
+      #
+      # @param  [Specification] spec
+      #         the spec we want to copy to the global cache.
+      #
+      # @param  [String] local_cache_path
+      #         the local directory from which we want to copy the .gem.
+      #
+      def cache_globally(spec, local_cache_path)
+        return unless cache_path = download_cache_path(spec)
+        return if cache_path.exist?
+
+        SharedHelpers.filesystem_access(cache_path.dirname, &:mkpath)
+        SharedHelpers.filesystem_access(cache_path) do
+          FileUtils.cp(local_cache_path, cache_path)
+        end
+      end
+
+      # Returns the global cache path of the calling Rubygems::Source object.
+      #
+      # Note that the Source determines the path's subdirectory. We use this
+      # subdirectory in the global cache path so that gems with the same name
+      # -- and possibly different versions -- from different sources are saved
+      # to their respective subdirectories and do not override one another.
+      #
+      # @param  [Gem::Specification] specification
+      #
+      # @return [Pathname] The global cache path.
+      #
+      def download_cache_path(spec)
+        return unless Bundler.feature_flag.global_gem_cache?
+        return unless remote = spec.remote
+        return unless cache_slug = remote.cache_slug
+
+        Bundler.user_cache.join("gems", cache_slug, spec.file_name)
+      end
     end
   end
 end
diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb
index b49e645506..e60c1f9055 100644
--- a/lib/bundler/source/rubygems/remote.rb
+++ b/lib/bundler/source/rubygems/remote.rb
@@ -20,6 +20,8 @@ module Bundler
         #
         def cache_slug
           @cache_slug ||= begin
+            return nil unless SharedHelpers.md5_available?
+
             cache_uri = original_uri || uri
 
             uri_parts = [cache_uri.host, cache_uri.user, cache_uri.port, cache_uri.path]
diff --git a/man/bundle-config.ronn b/man/bundle-config.ronn
index 9fc62edae3..de041b7ab8 100644
--- a/man/bundle-config.ronn
+++ b/man/bundle-config.ronn
@@ -237,6 +237,9 @@ learn more about their operation in [bundle install(1)][bundle-install].
 * `allow_bundler_dependency_conflicts` (`BUNDLE_ALLOW_BUNDLER_DEPENDENCY_CONFLICTS`):
    Allow resolving to specifications that have dependencies on `bundler` that
    are incompatible with the running Bundler version.
+* `global_gem_cache` (`BUNDLE_GLOBAL_GEM_CACHE`):
+   Whether Bundler should cache all gems globally, rather than locally to the
+   installing Ruby installation.
 
 In general, you should set these settings per-application by using the applicable
 flag to the [bundle install(1)][bundle-install] or [bundle package(1)][bundle-package] command.
diff --git a/spec/bundler/fetcher/compact_index_spec.rb b/spec/bundler/fetcher/compact_index_spec.rb
index e624af73e1..e0f58766ea 100644
--- a/spec/bundler/fetcher/compact_index_spec.rb
+++ b/spec/bundler/fetcher/compact_index_spec.rb
@@ -45,7 +45,17 @@ RSpec.describe Bundler::Fetcher::CompactIndex do
       end
 
       context "when OpenSSL is FIPS-enabled", :ruby => ">= 2.0.0" do
-        before { stub_const("OpenSSL::OPENSSL_FIPS", true) }
+        def remove_cached_md5_availability
+          return unless Bundler::SharedHelpers.instance_variable_defined?(:@md5_available)
+          Bundler::SharedHelpers.remove_instance_variable(:@md5_available)
+        end
+
+        before do
+          remove_cached_md5_availability
+          stub_const("OpenSSL::OPENSSL_FIPS", true)
+        end
+
+        after { remove_cached_md5_availability }
 
         context "when FIPS-mode is active" do
           before do
diff --git a/spec/install/global_cache_spec.rb b/spec/install/global_cache_spec.rb
new file mode 100644
index 0000000000..c48bb1bd89
--- /dev/null
+++ b/spec/install/global_cache_spec.rb
@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+RSpec.describe "global gem caching" do
+  before { bundle! "config global_gem_cache true" }
+
+  describe "using the cross-application user cache" do
+    let(:source)  { "http://localgemserver.test" }
+    let(:source2) { "http://gemserver.example.org" }
+
+    def source_global_cache(*segments)
+      home(".bundle", "cache", "gems", "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", *segments)
+    end
+
+    def source2_global_cache(*segments)
+      home(".bundle", "cache", "gems", "gemserver.example.org.80.1ae1663619ffe0a3c9d97712f44c705b", *segments)
+    end
+
+    it "caches gems into the global cache on download" do
+      install_gemfile! <<-G, :artifice => "compact_index"
+        source "#{source}"
+        gem "rack"
+      G
+
+      expect(the_bundle).to include_gems "rack 1.0.0"
+      expect(source_global_cache("rack-1.0.0.gem")).to exist
+    end
+
+    it "uses globally cached gems if they exist" do
+      source_global_cache.mkpath
+      FileUtils.cp(gem_repo1("gems/rack-1.0.0.gem"), source_global_cache("rack-1.0.0.gem"))
+
+      install_gemfile! <<-G, :artifice => "compact_index_no_gem"
+        source "#{source}"
+        gem "rack"
+      G
+
+      expect(the_bundle).to include_gems "rack 1.0.0"
+    end
+
+    describe "when the same gem from different sources is installed" do
+      it "should use the appropriate one from the global cache" do
+        install_gemfile! <<-G, :artifice => "compact_index"
+          source "#{source}"
+          gem "rack"
+        G
+
+        FileUtils.rm_r(default_bundle_path)
+        expect(the_bundle).not_to include_gems "rack 1.0.0"
+        expect(source_global_cache("rack-1.0.0.gem")).to exist
+        # rack 1.0.0 is not installed and it is in the global cache
+
+        install_gemfile! <<-G, :artifice => "compact_index"
+          source "#{source2}"
+          gem "rack", "0.9.1"
+        G
+
+        FileUtils.rm_r(default_bundle_path)
+        expect(the_bundle).not_to include_gems "rack 0.9.1"
+        expect(source2_global_cache("rack-0.9.1.gem")).to exist
+        # rack 0.9.1 is not installed and it is in the global cache
+
+        gemfile <<-G
+          source "#{source}"
+          gem "rack", "1.0.0"
+        G
+
+        bundle! :install, :artifice => "compact_index_no_gem"
+        # rack 1.0.0 is installed and rack 0.9.1 is not
+        expect(the_bundle).to include_gems "rack 1.0.0"
+        expect(the_bundle).not_to include_gems "rack 0.9.1"
+        FileUtils.rm_r(default_bundle_path)
+
+        gemfile <<-G
+          source "#{source2}"
+          gem "rack", "0.9.1"
+        G
+
+        bundle! :install, :artifice => "compact_index_no_gem"
+        # rack 0.9.1 is installed and rack 1.0.0 is not
+        expect(the_bundle).to include_gems "rack 0.9.1"
+        expect(the_bundle).not_to include_gems "rack 1.0.0"
+      end
+
+      it "should not install if the wrong source is provided" do
+        gemfile <<-G
+          source "#{source}"
+          gem "rack"
+        G
+
+        bundle! :install, :artifice => "compact_index"
+        FileUtils.rm_r(default_bundle_path)
+        expect(the_bundle).not_to include_gems "rack 1.0.0"
+        expect(source_global_cache("rack-1.0.0.gem")).to exist
+        # rack 1.0.0 is not installed and it is in the global cache
+
+        gemfile <<-G
+          source "#{source2}"
+          gem "rack", "0.9.1"
+        G
+
+        bundle! :install, :artifice => "compact_index"
+        FileUtils.rm_r(default_bundle_path)
+        expect(the_bundle).not_to include_gems "rack 0.9.1"
+        expect(source2_global_cache("rack-0.9.1.gem")).to exist
+        # rack 0.9.1 is not installed and it is in the global cache
+
+        gemfile <<-G
+          source "#{source2}"
+          gem "rack", "1.0.0"
+        G
+
+        expect(source_global_cache("rack-1.0.0.gem")).to exist
+        expect(source2_global_cache("rack-0.9.1.gem")).to exist
+        bundle :install, :artifice => "compact_index_no_gem"
+        expect(out).to include("Internal Server Error 500")
+        # rack 1.0.0 is not installed and rack 0.9.1 is not
+        expect(the_bundle).not_to include_gems "rack 1.0.0"
+        expect(the_bundle).not_to include_gems "rack 0.9.1"
+
+        gemfile <<-G
+          source "#{source}"
+          gem "rack", "0.9.1"
+        G
+
+        expect(source_global_cache("rack-1.0.0.gem")).to exist
+        expect(source2_global_cache("rack-0.9.1.gem")).to exist
+        bundle :install, :artifice => "compact_index_no_gem"
+        expect(out).to include("Internal Server Error 500")
+        # rack 0.9.1 is not installed and rack 1.0.0 is not
+        expect(the_bundle).not_to include_gems "rack 0.9.1"
+        expect(the_bundle).not_to include_gems "rack 1.0.0"
+      end
+    end
+
+    describe "when installing gems from a different directory" do
+      it "uses the global cache as a source" do
+        install_gemfile! <<-G, :artifice => "compact_index"
+          source "#{source}"
+          gem "rack"
+          gem "activesupport"
+        G
+
+        # Both gems are installed and in the global cache
+        expect(the_bundle).to include_gems "rack 1.0.0"
+        expect(the_bundle).to include_gems "activesupport 2.3.5"
+        expect(source_global_cache("rack-1.0.0.gem")).to exist
+        expect(source_global_cache("activesupport-2.3.5.gem")).to exist
+        FileUtils.rm_r(default_bundle_path)
+        # Both gems are now only in the global cache
+        expect(the_bundle).not_to include_gems "rack 1.0.0"
+        expect(the_bundle).not_to include_gems "activesupport 2.3.5"
+
+        install_gemfile! <<-G, :artifice => "compact_index_no_gem"
+          source "#{source}"
+          gem "rack"
+        G
+
+        # rack is installed and both are in the global cache
+        expect(the_bundle).to include_gems "rack 1.0.0"
+        expect(the_bundle).not_to include_gems "activesupport 2.3.5"
+        expect(source_global_cache("rack-1.0.0.gem")).to exist
+        expect(source_global_cache("activesupport-2.3.5.gem")).to exist
+
+        Dir.chdir bundled_app2 do
+          create_file bundled_app2("gems.rb"), <<-G
+            source "#{source}"
+            gem "activesupport"
+          G
+
+          # Neither gem is installed and both are in the global cache
+          expect(the_bundle).not_to include_gems "rack 1.0.0"
+          expect(the_bundle).not_to include_gems "activesupport 2.3.5"
+          expect(source_global_cache("rack-1.0.0.gem")).to exist
+          expect(source_global_cache("activesupport-2.3.5.gem")).to exist
+
+          # Install using the global cache instead of by downloading the .gem
+          # from the server
+          bundle! :install, :artifice => "compact_index_no_gem"
+
+          # activesupport is installed and both are in the global cache
+          expect(the_bundle).not_to include_gems "rack 1.0.0"
+          expect(the_bundle).to include_gems "activesupport 2.3.5"
+          expect(source_global_cache("rack-1.0.0.gem")).to exist
+          expect(source_global_cache("activesupport-2.3.5.gem")).to exist
+        end
+      end
+    end
+  end
+end
diff --git a/spec/support/artifice/compact_index_no_gem.rb b/spec/support/artifice/compact_index_no_gem.rb
new file mode 100644
index 0000000000..0a59e498cd
--- /dev/null
+++ b/spec/support/artifice/compact_index_no_gem.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexNoGem < CompactIndexAPI
+  get "/gems/:id" do
+    halt 500
+  end
+end
+
+Artifice.activate_with(CompactIndexNoGem)