Auto merge of #3556 - bundler:seg-new-index-fetchers, r=segiddins

[Fetcher] Add Fetchers for the new index format!

Things that need to be tested:

- [x] CompactGemList sources
- [x] mixed compact and full index sources
- [x] mixed compact and api sources
- [x] mixed compact, full, and api sources
- [x] fallback from compact to api
- [x] fallback from compact to full

Things that need to be finished to ship this:

 - [x] install gems via compact index
 - [x] stop requesting /api/v1/dependencies if compact index request succeeds
 - [x] stop printing "Need to query more than 500 gems. Downloading full index instead..."
 - [x] stop printing "Fetching source index from https://bundler-api-staging.herokuapp.com/"
 - [ ] print useful info message, proposed:
    - [x] print "Fetching gem information from <server>..." before requesting `/versions`
    - [ ] print "." for each additional `/info/<gem>` request
 - [x] accept full files gzipped
 - [x] request and accept ranges using un-gzipped offsets
 - [ ] make sure this client works against fastly-cached text files
 - [x] share cache between mirrors and original source
 - [x] Add username to cache path

51 files changed, 1628 insertions, 196 deletions

diff --git a/.gitignore b/.gitignore
index 4f79ffe3ca..bd5d29dc7b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,5 +12,5 @@
 # output from ronn
 /lib/bundler/man/
 
-# output from ci_reporter
-/spec/reports/
+# rspec failure tracking
+.rspec_status
diff --git a/.rubocop.yml b/.rubocop.yml
index ea89254c7d..03f007416e 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -87,6 +87,9 @@ Style/SpecialGlobalVars:
 Style/TrailingComma:
   Enabled: false
 
+Performance/FlatMap:
+  Enabled: false
+
 # Metrics
 
 # We've chosen to use Rubocop only for style, and not for complexity or quality checks.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f9fb3fd15..4c9027e3d4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 1.12.0.pre (tbd)
+
+Features:
+
+  - speed up `install` and `update` by using the new gem index (@segiddins, @fotanus, @indirect)
+
 ## 1.11.2 (2015-12-15)
 
 Bugfixes:
diff --git a/Rakefile b/Rakefile
index 332e5e9da4..403cf262e1 100644
--- a/Rakefile
+++ b/Rakefile
@@ -281,6 +281,13 @@ end
 begin
   require "automatiek"
 
+  Automatiek::RakeTask.new("compact_index_client") do |lib|
+    lib.download = { :github => "https://github.com/bundler/compact_index_client" }
+    lib.namespace = "CompactIndexClient"
+    lib.prefix = "Bundler"
+    lib.vendor_lib = "lib/bundler/vendor/compact_index_client"
+  end
+
   Automatiek::RakeTask.new("molinillo") do |lib|
     lib.download = { :github => "https://github.com/CocoaPods/Molinillo" }
     lib.namespace = "Molinillo"
diff --git a/lib/bundler.rb b/lib/bundler.rb
index 146d343bd9..346310cd27 100644
--- a/lib/bundler.rb
+++ b/lib/bundler.rb
@@ -137,7 +137,7 @@ module Bundler
     end
 
     def user_bundle_path
-      Pathname.new(Bundler.rubygems.user_home).join(".bundler")
+      Pathname.new(Bundler.rubygems.user_home).join(".bundle")
     end
 
     def home
@@ -156,6 +156,10 @@ module Bundler
       bundle_path.join("cache/bundler")
     end
 
+    def user_cache
+      user_bundle_path.join("cache")
+    end
+
     def root
       @root ||= begin
                   default_gemfile.dirname.expand_path
diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb
index 477224e4e9..91b0569846 100644
--- a/lib/bundler/cli.rb
+++ b/lib/bundler/cli.rb
@@ -136,7 +136,7 @@ module Bundler
     method_option "frozen", :type => :boolean, :banner =>
       "Do not allow the Gemfile.lock to be updated after this install"
     method_option "full-index", :type => :boolean, :banner =>
-      "Use the rubygems modern index instead of the API endpoint"
+      "Fall back to using the single-file index of all gems"
     method_option "gemfile", :type => :string, :banner =>
       "Use the specified gemfile instead of Gemfile"
     method_option "jobs", :aliases => "-j", :type => :numeric, :banner =>
@@ -179,7 +179,7 @@ module Bundler
       possible versions of the gems in the bundle.
     D
     method_option "full-index", :type => :boolean, :banner =>
-      "Use the rubygems modern index instead of the API endpoint"
+      "Fall back to using the single-file index of all gems"
     method_option "group", :aliases => "-g", :type => :array, :banner =>
       "Update a specific group"
     method_option "jobs", :aliases => "-j", :type => :numeric, :banner =>
@@ -410,7 +410,7 @@ module Bundler
     method_option "lockfile", :type => :string, :default => nil, :banner =>
       "the path the lockfile should be written to"
     method_option "full-index", :type => :boolean, :default => false, :banner =>
-      "Use the rubygems modern index instead of the API endpoint"
+      "Fall back to using the single-file index of all gems"
     def lock
       require "bundler/cli/lock"
       Lock.new(options).run
diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index 98c6f66e72..636d643a94 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -55,6 +55,7 @@ module Bundler
 
       @lockfile_contents      = String.new
       @locked_bundler_version = nil
+      @locked_ruby_version    = nil
 
       if lockfile && File.exist?(lockfile)
         @lockfile_contents = Bundler.read_file(lockfile)
diff --git a/lib/bundler/endpoint_specification.rb b/lib/bundler/endpoint_specification.rb
index 21a6c85c5a..24f1637b9e 100644
--- a/lib/bundler/endpoint_specification.rb
+++ b/lib/bundler/endpoint_specification.rb
@@ -1,16 +1,19 @@
 module Bundler
   # used for Creating Specifications from the Gemcutter Endpoint
   class EndpointSpecification < Gem::Specification
+    ILLFORMED_MESSAGE = 'Ill-formed requirement ["#<YAML::Syck::DefaultKey'.freeze
     include MatchPlatform
 
-    attr_reader :name, :version, :platform, :dependencies
+    attr_reader :name, :version, :platform, :dependencies, :required_rubygems_version, :required_ruby_version, :checksum
     attr_accessor :source, :remote
 
-    def initialize(name, version, platform, dependencies)
+    def initialize(name, version, platform, dependencies, metadata = nil)
       @name         = name
-      @version      = version
+      @version      = Gem::Version.create version
       @platform     = platform
-      @dependencies = dependencies
+      @dependencies = dependencies.map {|dep, reqs| build_dependency(dep, reqs) }
+
+      parse_metadata(metadata)
     end
 
     def fetch_platform
@@ -96,5 +99,31 @@ module Bundler
     def local_specification_path
       "#{base_dir}/specifications/#{full_name}.gemspec"
     end
+
+    def parse_metadata(data)
+      return unless data
+      data.each do |k, v|
+        next unless v
+        case k.to_s
+        when "checksum"
+          @checksum = v.last
+        when "rubygems"
+          @required_rubygems_version = Gem::Requirement.new(v)
+        when "ruby"
+          @required_ruby_version = Gem::Requirement.new(v)
+        end
+      end
+    end
+
+    def build_dependency(name, *requirements)
+      Gem::Dependency.new(name, *requirements)
+    rescue ArgumentError => e
+      raise unless e.message.include?(ILLFORMED_MESSAGE)
+      puts # we shouldn't print the error message on the "fetching info" status line
+      raise GemspecError,
+        "Unfortunately, the gem #{name} (#{version}) has an invalid " \
+        "gemspec.\nPlease ask the gem author to yank the bad version to fix " \
+        "this issue. For more information, see http://bit.ly/syck-defaultkey."
+    end
   end
 end
diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb
index c8c21a92ff..2511b964d6 100644
--- a/lib/bundler/fetcher.rb
+++ b/lib/bundler/fetcher.rb
@@ -6,6 +6,7 @@ require "zlib"
 module Bundler
   # Handles all the fetching with the rubygems server
   class Fetcher
+    autoload :CompactIndex, "bundler/fetcher/compact_index"
     autoload :Downloader, "bundler/fetcher/downloader"
     autoload :Dependency, "bundler/fetcher/dependency"
     autoload :Index, "bundler/fetcher/index"
@@ -52,14 +53,15 @@ module Bundler
 
     # Exceptions classes that should bypass retry attempts. If your password didn't work the
     # first time, it's not going to the third time.
-    AUTH_ERRORS = [AuthenticationRequiredError, BadAuthenticationError]
+    FAIL_ERRORS = [AuthenticationRequiredError, BadAuthenticationError, FallbackError]
     NET_ERRORS = [:HTTPBadGateway, :HTTPBadRequest, :HTTPFailedDependency,
                   :HTTPForbidden, :HTTPInsufficientStorage, :HTTPMethodNotAllowed,
                   :HTTPMovedPermanently, :HTTPNoContent, :HTTPNotFound,
                   :HTTPNotImplemented, :HTTPPreconditionFailed, :HTTPRequestEntityTooLarge,
                   :HTTPRequestURITooLong, :HTTPUnauthorized, :HTTPUnprocessableEntity,
                   :HTTPUnsupportedMediaType, :HTTPVersionNotSupported]
-    AUTH_ERRORS.push(*NET_ERRORS.map {|e| SharedHelpers.const_get_safely(e, Net) }.compact)
+    FAIL_ERRORS << Gem::Requirement::BadRequirementError if defined?(Gem::Requirement::BadRequirementError)
+    FAIL_ERRORS.push(*NET_ERRORS.map {|e| SharedHelpers.const_get_safely(e, Net) }.compact)
 
     class << self
       attr_accessor :disable_endpoint, :api_timeout, :redirect_limit, :max_retries
@@ -91,7 +93,7 @@ module Bundler
       elsif cached_spec_path = gemspec_cached_path(spec_file_name)
         Bundler.load_gemspec(cached_spec_path)
       else
-        Bundler.load_marshal Gem.inflate(downloader.fetch uri)
+        Bundler.load_marshal Gem.inflate(downloader.fetch(uri).body)
       end
     rescue MarshalError
       raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
@@ -100,7 +102,7 @@ module Bundler
 
     # return the specs in the bundler format as an index with retries
     def specs_with_retry(gem_names, source)
-      Bundler::Retry.new("fetcher").attempts do
+      Bundler::Retry.new("fetcher", FAIL_ERRORS).attempts do
         specs(gem_names, source)
       end
     end
@@ -110,14 +112,20 @@ module Bundler
       old = Bundler.rubygems.sources
       index = Bundler::Index.new
 
-      specs = {}
-      fetchers.dup.each do |f|
-        break unless f.api_fetcher? && !gem_names || !specs = f.specs(gem_names)
-        fetchers.delete(f)
+      if Bundler::Fetcher.disable_endpoint
+        @use_api = false
+        specs = fetchers.last.specs(gem_names)
+      else
+        specs = []
+        fetchers.shift until fetchers.first.available? || fetchers.empty?
+        fetchers.dup.each do |f|
+          break unless f.api_fetcher? && !gem_names || !specs = f.specs(gem_names)
+          fetchers.delete(f)
+        end
+        @use_api = false if fetchers.none?(&:api_fetcher?)
       end
-      @use_api = false if fetchers.none?(&:api_fetcher?)
 
-      specs[remote_uri].each do |name, version, platform, dependencies|
+      specs.each do |name, version, platform, dependencies|
         next if name == "bundler"
         spec = nil
         if dependencies
@@ -141,11 +149,12 @@ module Bundler
     def use_api
       return @use_api if defined?(@use_api)
 
+      fetchers.shift until fetchers.first.available?
+
       if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
         @use_api = false
       else
-        fetchers.reject! {|f| f.api_fetcher? && !f.api_available? }
-        @use_api = fetchers.any?(&:api_fetcher?)
+        @use_api = fetchers.first.api_fetcher?
       end
     end
 
@@ -200,7 +209,7 @@ module Bundler
 
   private
 
-    FETCHERS = [Dependency, Index]
+    FETCHERS = [CompactIndex, Dependency, Index]
 
     def cis
       env_cis = {
diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb
index 5cc405cc8a..1d1adc73db 100644
--- a/lib/bundler/fetcher/base.rb
+++ b/lib/bundler/fetcher/base.rb
@@ -28,8 +28,8 @@ module Bundler
         end
       end
 
-      def api_available?
-        api_fetcher?
+      def available?
+        true
       end
 
       def api_fetcher?
diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb
new file mode 100644
index 0000000000..97b2e71dce
--- /dev/null
+++ b/lib/bundler/fetcher/compact_index.rb
@@ -0,0 +1,96 @@
+require "bundler/fetcher/base"
+require "bundler/worker"
+
+module Bundler
+  class Fetcher
+    class CompactIndex < Base
+      require "bundler/vendor/compact_index_client/lib/compact_index_client"
+
+      def self.compact_index_request(method_name)
+        method = instance_method(method_name)
+        undef_method(method_name)
+        define_method(method_name) do |*args, &blk|
+          begin
+            method.bind(self).call(*args, &blk)
+          rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e
+            raise HTTPError, e.message
+          rescue AuthenticationRequiredError
+            # We got a 401 from the server. Just fail.
+            raise
+          rescue HTTPError => e
+            Bundler.ui.trace(e)
+            nil
+          end
+        end
+      end
+
+      def specs(gem_names)
+        specs_for_names(gem_names)
+      end
+      compact_index_request :specs
+
+      def specs_for_names(gem_names)
+        gem_info = []
+        complete_gems = []
+        remaining_gems = gem_names.dup
+
+        until remaining_gems.empty?
+          Bundler.ui.debug "Looking up gems #{remaining_gems.inspect}"
+
+          deps = compact_index_client.dependencies(remaining_gems)
+          next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq
+          deps.each {|dep| gem_info << dep }
+          complete_gems.push(*deps.map(&:first)).uniq!
+          remaining_gems = next_gems - complete_gems
+        end
+
+        gem_info
+      end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        contents = compact_index_client.spec(*spec)
+        return nil if contents.nil?
+        contents.unshift(spec.first)
+        contents[3].map! {|d| Gem::Dependency.new(*d) }
+        EndpointSpecification.new(*contents)
+      end
+      compact_index_request :fetch_spec
+
+      def available?
+        # Read info file checksums out of /versions, so we can know if gems are up to date
+        fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums!
+      end
+      compact_index_request :available?
+
+      def api_fetcher?
+        true
+      end
+
+    private
+
+      def compact_index_client
+        @compact_index_client ||= begin
+          compact_fetcher = lambda do |path, headers|
+            downloader.fetch(fetch_uri + path, headers)
+          end
+
+          SharedHelpers.filesystem_access(cache_path) do
+            CompactIndexClient.new(cache_path, compact_fetcher)
+          end.tap do |client|
+            client.in_parallel = lambda do |inputs, &blk|
+              func = lambda {|object, _index| blk.call(object) }
+              worker = Bundler::Worker.new(25, func)
+              inputs.each {|input| worker.enq(input) }
+              inputs.map { worker.deq }
+            end
+          end
+        end
+      end
+
+      def cache_path
+        Bundler.user_cache.join("compact_index", remote.cache_slug)
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb
index 29af0dc318..fdf91c1710 100644
--- a/lib/bundler/fetcher/dependency.rb
+++ b/lib/bundler/fetcher/dependency.rb
@@ -4,14 +4,15 @@ require "cgi"
 module Bundler
   class Fetcher
     class Dependency < Base
-      def api_available?
-        downloader.fetch(dependency_api_uri)
+      def available?
+        fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri)
       rescue NetworkDownError => e
         raise HTTPError, e.message
       rescue AuthenticationRequiredError
         # We got a 401 from the server. Just fail.
         raise
       rescue HTTPError
+        false
       end
 
       def api_fetcher?
@@ -19,13 +20,13 @@ module Bundler
       end
 
       def specs(gem_names, full_dependency_list = [], last_spec_list = [])
-        query_list = gem_names - full_dependency_list
+        query_list = gem_names.uniq - full_dependency_list
 
         log_specs(query_list)
 
-        return { remote_uri => last_spec_list } if query_list.empty?
+        return last_spec_list if query_list.empty?
 
-        spec_list, deps_list = Bundler::Retry.new("dependency api", AUTH_ERRORS).attempts do
+        spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do
           dependency_specs(query_list)
         end
 
@@ -47,24 +48,22 @@ module Bundler
       def unmarshalled_dep_gems(gem_names)
         gem_list = []
         gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names|
-          marshalled_deps = downloader.fetch dependency_api_uri(names)
-          gem_list += Bundler.load_marshal(marshalled_deps)
+          marshalled_deps = downloader.fetch(dependency_api_uri(names)).body
+          gem_list.push(*Bundler.load_marshal(marshalled_deps))
         end
         gem_list
       end
 
       def get_formatted_specs_and_deps(gem_list)
         deps_list = []
-        spec_list = gem_list.map do |s|
-          dependencies = s[:dependencies].map do |name, requirement|
-            dep = well_formed_dependency(name, requirement.split(", "))
-            deps_list << dep.name
-            dep
-          end
+        spec_list = []
 
-          [s[:name], Gem::Version.new(s[:number]), s[:platform], dependencies]
+        gem_list.each do |s|
+          deps_list.push(*s[:dependencies].map(&:first))
+          deps = s[:dependencies].map {|n, d| [n, d.split(", ")] }
+          spec_list.push([s[:name], s[:number], s[:platform], deps])
         end
-        [spec_list, deps_list.uniq]
+        [spec_list, deps_list]
       end
 
       def dependency_api_uri(gem_names = [])
@@ -73,18 +72,6 @@ module Bundler
         uri
       end
 
-      def well_formed_dependency(name, *requirements)
-        Gem::Dependency.new(name, *requirements)
-      rescue ArgumentError => e
-        illformed = 'Ill-formed requirement ["#<YAML::Syck::DefaultKey'
-        raise e unless e.message.include?(illformed)
-        puts # we shouldn't print the error message on the "fetching info" status line
-        raise GemspecError,
-          "Unfortunately, the gem #{name} has an invalid " \
-          "gemspec. \nPlease ask the gem author to yank the bad version to fix " \
-          "this issue. For more information, see http://bit.ly/syck-defaultkey."
-      end
-
     private
 
       def log_specs(query_list)
diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb
index 3e05b77d5e..68f63ed54d 100644
--- a/lib/bundler/fetcher/downloader.rb
+++ b/lib/bundler/fetcher/downloader.rb
@@ -9,34 +9,36 @@ module Bundler
         @redirect_limit = redirect_limit
       end
 
-      def fetch(uri, counter = 0)
+      def fetch(uri, options = {}, counter = 0)
         raise HTTPError, "Too many redirects" if counter >= redirect_limit
 
-        response = request(uri)
+        response = request(uri, options)
         Bundler.ui.debug("HTTP #{response.code} #{response.message}")
 
         case response
+        when Net::HTTPSuccess, Net::HTTPNotModified
+          response
         when Net::HTTPRedirection
           new_uri = URI.parse(response["location"])
           if new_uri.host == uri.host
             new_uri.user = uri.user
             new_uri.password = uri.password
           end
-          fetch(new_uri, counter + 1)
-        when Net::HTTPSuccess
-          response.body
+          fetch(new_uri, options, counter + 1)
         when Net::HTTPRequestEntityTooLarge
           raise FallbackError, response.body
         when Net::HTTPUnauthorized
           raise AuthenticationRequiredError, uri.host
+        when Net::HTTPNotFound
+          raise FallbackError, "Net::HTTPNotFound"
         else
-          raise HTTPError, "#{response.class}: #{response.body}"
+          raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
         end
       end
 
-      def request(uri)
+      def request(uri, options)
         Bundler.ui.debug "HTTP GET #{uri}"
-        req = Net::HTTP::Get.new uri.request_uri
+        req = Net::HTTP::Get.new uri.request_uri, options
         if uri.user
           user = CGI.unescape(uri.user)
           password = uri.password ? CGI.unescape(uri.password) : nil
diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb
index b37f6a84c1..1d8aa657a8 100644
--- a/lib/bundler/fetcher/index.rb
+++ b/lib/bundler/fetcher/index.rb
@@ -23,6 +23,31 @@ module Bundler
           raise HTTPError, "Could not fetch specs from #{display_uri}"
         end
       end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        spec_file_name = "#{spec.join "-"}.gemspec"
+
+        uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
+        if uri.scheme == "file"
+          Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path))
+        elsif cached_spec_path = gemspec_cached_path(spec_file_name)
+          Bundler.load_gemspec(cached_spec_path)
+        else
+          Bundler.load_marshal Gem.inflate(downloader.fetch(uri).body)
+        end
+      rescue MarshalError
+        raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
+          "Your network or your gem server is probably having issues right now."
+      end
+
+    private
+
+      # cached gem specification path, if one exists
+      def gemspec_cached_path(spec_file_name)
+        paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
+        paths.find {|path| File.file? path }
+      end
     end
   end
 end
diff --git a/lib/bundler/remote_specification.rb b/lib/bundler/remote_specification.rb
index c8247c4c95..2ed4fadadc 100644
--- a/lib/bundler/remote_specification.rb
+++ b/lib/bundler/remote_specification.rb
@@ -15,7 +15,7 @@ module Bundler
 
     def initialize(name, version, platform, spec_fetcher)
       @name         = name
-      @version      = version
+      @version      = Gem::Version.create version
       @platform     = platform
       @spec_fetcher = spec_fetcher
     end
@@ -69,6 +69,8 @@ module Bundler
 
     def _remote_specification
       @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, @platform])
+      @_remote_specification || raise(GemspecError, "Gemspec data for #{full_name} was" \
+        " missing from the server! Try installing with `--full-index` as a workaround.")
     end
 
     def method_missing(method, *args, &blk)
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index f40b40b1de..5818f5b16a 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -64,7 +64,7 @@ module Bundler
     def configuration
       require "bundler/psyched_yaml"
       Gem.configuration
-    rescue Gem::SystemExitException => e
+    rescue Gem::SystemExitException, LoadError => e
       Bundler.ui.error "#{e.class}: #{e.message}"
       Bundler.ui.trace e
       raise
@@ -181,7 +181,7 @@ module Bundler
     def fetch_prerelease_specs
       fetch_specs(false, true)
     rescue Gem::RemoteFetcher::FetchError
-      [] # if we can't download them, there aren't any
+      {} # if we can't download them, there aren't any
     end
 
     # TODO: This is for older versions of Rubygems... should we support the
@@ -194,9 +194,9 @@ module Bundler
       # Fetch all specs, minus prerelease specs
       spec_list = fetch_specs(true, false)
       # Then fetch the prerelease specs
-      fetch_prerelease_specs.each {|k, v| spec_list[k] += v }
+      fetch_prerelease_specs.each {|k, v| spec_list[k].push(*v) }
 
-      spec_list
+      spec_list.values.first
     ensure
       Bundler.rubygems.sources = old_sources
     end
@@ -578,18 +578,12 @@ module Bundler
       end
 
       def fetch_all_remote_specs(remote)
-        # Since SpecFetcher now returns NameTuples, we just fetch directly
-        # and unmarshal the array ourselves.
-        hash = {}
+        source = remote.uri.is_a?(URI) ? remote.uri : URI.parse(source.to_s)
 
-        source = remote.uri
-        source = URI.parse(source.to_s) unless source.is_a?(URI)
-        hash[source] = fetch_specs(source, remote, "specs")
+        specs = fetch_specs(source, remote, "specs")
+        pres = fetch_specs(source, remote, "prerelease_specs") || []
 
-        pres = fetch_specs(source, remote, "prerelease_specs")
-        hash[source].push(*pres) if pres && !pres.empty?
-
-        hash
+        specs.push(*pres)
       end
 
       def download_gem(spec, uri, path)
diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb
index 05208379c6..31f1fdd7f2 100644
--- a/lib/bundler/settings.rb
+++ b/lib/bundler/settings.rb
@@ -209,8 +209,11 @@ module Bundler
     end
 
     def global_config_file
-      file = ENV["BUNDLE_CONFIG"] || File.join(Bundler.rubygems.user_home, ".bundle/config")
-      Pathname.new(file)
+      if ENV["BUNDLE_CONFIG"] && !ENV["BUNDLE_CONFIG"].empty?
+        Pathname.new(ENV["BUNDLE_CONFIG"])
+      else
+        Bundler.user_bundle_path.join("config")
+      end
     end
 
     def local_config_file
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index effd4680f6..1315677a52 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -327,7 +327,7 @@ module Bundler
       end
 
       def api_fetchers
-        fetchers.select(&:use_api)
+        fetchers.select {|f| f.use_api && f.fetchers.first.api_fetcher? }
       end
 
       def remote_specs
diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb
index 561bde194e..4cd8ea23b8 100644
--- a/lib/bundler/source/rubygems/remote.rb
+++ b/lib/bundler/source/rubygems/remote.rb
@@ -14,6 +14,21 @@ module Bundler
           @anonymized_uri = remove_auth(@uri).freeze
         end
 
+        # @return [String] A slug suitable for use as a cache key for this
+        #         remote.
+        #
+        def cache_slug
+          @cache_slug ||= begin
+            cache_uri = original_uri || uri
+
+            uri_parts = [cache_uri.host, cache_uri.user, cache_uri.port, cache_uri.path]
+            uri_digest = Digest::MD5.hexdigest(uri_parts.compact.join("."))
+
+            uri_parts[-1] = uri_digest
+            uri_parts.compact.join(".")
+          end
+        end
+
       private
 
         def apply_auth(uri, auth)
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb
new file mode 100644
index 0000000000..e7d67d9926
--- /dev/null
+++ b/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb
@@ -0,0 +1,78 @@
+require "pathname"
+require "set"
+
+class Bundler::CompactIndexClient
+  class Error < StandardError; end
+
+  require "bundler/vendor/compact_index_client/lib/compact_index_client/cache"
+  require "bundler/vendor/compact_index_client/lib/compact_index_client/updater"
+  require "bundler/vendor/compact_index_client/lib/compact_index_client/version"
+
+  attr_reader :directory
+
+  # @return [Lambda] A lambda that takes an array of inputs and a block, and
+  #         maps the inputs with the block in parallel.
+  #
+  attr_accessor :in_parallel
+
+  def initialize(directory, fetcher)
+    @directory = Pathname.new(directory)
+    @updater = Updater.new(fetcher)
+    @cache = Cache.new(@directory)
+    @endpoints = Set.new
+    @info_checksums_by_name = {}
+    @in_parallel = lambda do |inputs, &blk|
+      inputs.map(&blk)
+    end
+  end
+
+  def names
+    update(@cache.names_path, "names")
+    @cache.names
+  end
+
+  def versions
+    update(@cache.versions_path, "versions")
+    versions, @info_checksums_by_name = @cache.versions
+    versions
+  end
+
+  def dependencies(names)
+    in_parallel.call(names) do |name|
+      update_info(name)
+      @cache.dependencies(name).map {|d| d.unshift(name) }
+    end.flatten(1)
+  end
+
+  def spec(name, version, platform = nil)
+    update_info(name)
+    @cache.specific_dependency(name, version, platform)
+  end
+
+  def update_and_parse_checksums!
+    return @info_checksums_by_name if @parsed_checksums
+    update(@cache.versions_path, "versions")
+    @info_checksums_by_name = @cache.checksums
+    @parsed_checksums = true
+  end
+
+private
+
+  def update(local_path, remote_path)
+    return if @endpoints.include?(remote_path)
+    @updater.update(local_path, url(remote_path))
+    @endpoints << remote_path
+  end
+
+  def update_info(name)
+    path = @cache.info_path(name)
+    checksum = @updater.checksum_for_file(path)
+    return unless existing = @info_checksums_by_name[name]
+    return if checksum == existing
+    update(path, "info/#{name}")
+  end
+
+  def url(path)
+    path
+  end
+end
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb
new file mode 100644
index 0000000000..57ef551849
--- /dev/null
+++ b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb
@@ -0,0 +1,97 @@
+class Bundler::CompactIndexClient
+  class Cache
+    attr_reader :directory
+
+    def initialize(directory)
+      @directory = Pathname.new(directory).expand_path
+      FileUtils.mkdir_p info_path(nil)
+    end
+
+    def names
+      lines(names_path)
+    end
+
+    def names_path
+      directory.join("names")
+    end
+
+    def versions
+      versions_by_name = Hash.new {|hash, key| hash[key] = [] }
+      info_checksums_by_name = {}
+
+      lines(versions_path).each do |line|
+        name, versions_string, info_checksum = line.split(" ", 3)
+        info_checksums_by_name[name] = info_checksum || ""
+        versions_string.split(",").each do |version|
+          if version.start_with?("-")
+            version = version[1..-1].split("-", 2).unshift(name)
+            versions_by_name[name].delete(version)
+          else
+            version = version.split("-", 2).unshift(name)
+            versions_by_name[name] << version
+          end
+        end
+      end
+
+      [versions_by_name, info_checksums_by_name]
+    end
+
+    def versions_path
+      directory.join("versions")
+    end
+
+    def checksums
+      checksums = {}
+
+      lines(versions_path).each do |line|
+        name, _, checksum = line.split(" ", 3)
+        checksums[name] = checksum
+      end
+
+      checksums
+    end
+
+    def dependencies(name)
+      lines(info_path(name)).map do |line|
+        parse_gem(line)
+      end
+    end
+
+    def info_path(name)
+      directory.join("info", name.to_s)
+    end
+
+    def specific_dependency(name, version, platform)
+      pattern = [version, platform].compact.join("-")
+      return nil if pattern.empty?
+
+      gem_lines = info_path(name).read
+      gem_line = gem_lines[/^#{Regexp.escape(pattern)}\b.*/, 0]
+      gem_line ? parse_gem(gem_line) : nil
+    end
+
+  private
+
+    def lines(path)
+      return [] unless path.file?
+      lines = path.read.split("\n")
+      header = lines.index("---")
+      lines = header ? lines[header + 1..-1] : lines
+    end
+
+    def parse_gem(string)
+      version_and_platform, rest = string.split(" ", 2)
+      version, platform = version_and_platform.split("-", 2)
+      dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
+      dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
+      requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
+      [version, platform, dependencies, requirements]
+    end
+
+    def parse_dependency(string)
+      dependency = string.split(":")
+      dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+      dependency
+    end
+  end
+end
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb
new file mode 100644
index 0000000000..a3cdc55140
--- /dev/null
+++ b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb
@@ -0,0 +1,55 @@
+require "stringio"
+require "zlib"
+
+class Bundler::CompactIndexClient
+  class Updater
+    class MisMatchedChecksumError < Error; end
+
+    def initialize(fetcher)
+      @fetcher = fetcher
+    end
+
+    def update(local_path, remote_path, retrying = nil)
+      headers = {}
+
+      if local_path.file?
+        headers["If-None-Match"] = etag_for(local_path)
+        headers["Range"] = "bytes=#{local_path.size}-"
+      else
+        # Fastly ignores Range when Accept-Encoding: gzip is set
+        headers["Accept-Encoding"] = "gzip"
+      end
+
+      response = @fetcher.call(remote_path, headers)
+      return if response.is_a?(Net::HTTPNotModified)
+
+      content = response.body
+      if response["Content-Encoding"] == "gzip"
+        content = Zlib::GzipReader.new(StringIO.new(content)).read
+      end
+
+      mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w"
+      local_path.open(mode) {|f| f << content }
+
+      return if etag_for(local_path) == response["ETag"]
+
+      if retrying.nil?
+        local_path.delete
+        update(local_path, remote_path, :retrying)
+      else
+        raise MisMatchedChecksumError, "Checksum of /#{remote_path} " \
+          "does not match the checksum provided by server! Something is wrong."
+      end
+    end
+
+    def etag_for(path)
+      sum = checksum_for_file(path)
+      sum ? '"' << sum << '"' : nil
+    end
+
+    def checksum_for_file(path)
+      return nil unless path.file?
+      Digest::MD5.file(path).hexdigest
+    end
+  end
+end
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb
new file mode 100644
index 0000000000..f18c522ae8
--- /dev/null
+++ b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb
@@ -0,0 +1,3 @@
+class Bundler::CompactIndexClient
+  VERSION = "0.1.0"
+end
diff --git a/lib/bundler/version.rb b/lib/bundler/version.rb
index a3311288ef..9a3c7dde22 100644
--- a/lib/bundler/version.rb
+++ b/lib/bundler/version.rb
@@ -2,5 +2,5 @@ module Bundler
   # We're doing this because we might write tests that deal
   # with other versions of bundler and we are unsure how to
   # handle this better.
-  VERSION = "1.11.2" unless defined?(::Bundler::VERSION)
+  VERSION = "1.12.0.pre" unless defined?(::Bundler::VERSION)
 end
diff --git a/spec/bundler/endpoint_specification_spec.rb b/spec/bundler/endpoint_specification_spec.rb
new file mode 100644
index 0000000000..0340c36261
--- /dev/null
+++ b/spec/bundler/endpoint_specification_spec.rb
@@ -0,0 +1,49 @@
+require "spec_helper"
+
+describe Bundler::EndpointSpecification do
+  let(:name)         { "foo" }
+  let(:version)      { "1.0.0" }
+  let(:platform)     { Gem::Platform::RUBY }
+  let(:dependencies) { [] }
+  let(:metadata)     { nil }
+
+  subject { described_class.new(name, version, platform, dependencies, metadata) }
+
+  describe "#build_dependency" do
+    let(:name)           { "foo" }
+    let(:requirement1)   { "~> 1.1" }
+    let(:requirement2)   { ">= 1.1.7" }
+
+    it "should return a Gem::Dependency" do
+      expect(subject.send(:build_dependency, name, requirement1, requirement2)).to be_instance_of(Gem::Dependency)
+    end
+
+    context "when an ArgumentError occurs" do
+      before do
+        allow(Gem::Dependency).to receive(:new).with(name, requirement1, requirement2) {
+          raise ArgumentError.new("Some error occurred")
+        }
+      end
+
+      it "should raise the original error" do
+        expect { subject.send(:build_dependency, name, requirement1, requirement2) }.to raise_error(
+          ArgumentError, "Some error occurred")
+      end
+    end
+
+    context "when there is an ill formed requirement" do
+      before do
+        allow(Gem::Dependency).to receive(:new).with(name, requirement1, requirement2) {
+          raise ArgumentError.new("Ill-formed requirement [\"#<YAML::Syck::DefaultKey")
+        }
+        # Eliminate extra line break in rspec output due to `puts` in `#build_dependency`
+        allow(subject).to receive(:puts) {}
+      end
+
+      it "should raise a Bundler::GemspecError with invalid gemspec message" do
+        expect { subject.send(:build_dependency, name, requirement1, requirement2) }.to raise_error(
+          Bundler::GemspecError, /Unfortunately, the gem foo \(1\.0\.0\) has an invalid gemspec/)
+      end
+    end
+  end
+end
diff --git a/spec/bundler/fetcher/base_spec.rb b/spec/bundler/fetcher/base_spec.rb
index 539d73b207..4ca47e4723 100644
--- a/spec/bundler/fetcher/base_spec.rb
+++ b/spec/bundler/fetcher/base_spec.rb
@@ -62,11 +62,9 @@ describe Bundler::Fetcher::Base do
     end
   end
 
-  describe "#api_available?" do
-    before { allow(subject).to receive(:api_fetcher?).and_return(false) }
-
+  describe "#available?" do
     it "should return whether the api is available" do
-      expect(subject.api_available?).to eq(false)
+      expect(subject.available?).to be_truthy
     end
   end
 
diff --git a/spec/bundler/fetcher/dependency_spec.rb b/spec/bundler/fetcher/dependency_spec.rb
index 64c3902cfa..b78d82b98b 100644
--- a/spec/bundler/fetcher/dependency_spec.rb
+++ b/spec/bundler/fetcher/dependency_spec.rb
@@ -2,12 +2,12 @@ require "spec_helper"
 
 describe Bundler::Fetcher::Dependency do
   let(:downloader)  { double(:downloader) }
-  let(:remote)      { nil }
+  let(:remote)      { double(:remote, :uri => URI("http://localhost:5000")) }
   let(:display_uri) { "http://sample_uri.com" }
 
   subject { described_class.new(downloader, remote, display_uri) }
 
-  describe "#api_available?" do
+  describe "#available?" do
     let(:dependency_api_uri) { double(:dependency_api_uri) }
     let(:fetched_spec)       { double(:fetched_spec) }
 
@@ -17,7 +17,7 @@ describe Bundler::Fetcher::Dependency do
     end
 
     it "should be truthy" do
-      expect(subject.api_available?).to be_truthy
+      expect(subject.available?).to be_truthy
     end
 
     context "when there is no network access" do
@@ -28,7 +28,7 @@ describe Bundler::Fetcher::Dependency do
       end
 
       it "should raise an HTTPError with the original message" do
-        expect { subject.api_available? }.to raise_error(Bundler::HTTPError, "Network Down Message")
+        expect { subject.available? }.to raise_error(Bundler::HTTPError, "Network Down Message")
       end
     end
 
@@ -42,7 +42,7 @@ describe Bundler::Fetcher::Dependency do
       end
 
       it "should raise the original error" do
-        expect { subject.api_available? }.to raise_error(Bundler::Fetcher::AuthenticationRequiredError,
+        expect { subject.available? }.to raise_error(Bundler::Fetcher::AuthenticationRequiredError,
           %r{Authentication is required for http://remote_uri.org})
       end
     end
@@ -51,7 +51,7 @@ describe Bundler::Fetcher::Dependency do
       before { allow(downloader).to receive(:fetch).with(dependency_api_uri) { raise Bundler::HTTPError.new } }
 
       it "should be falsey" do
-        expect(subject.api_available?).to be_falsey
+        expect(subject.available?).to be_falsey
       end
     end
   end
@@ -66,15 +66,15 @@ describe Bundler::Fetcher::Dependency do
     let(:gem_names)            { %w(foo bar) }
     let(:full_dependency_list) { ["bar"] }
     let(:last_spec_list)       { [["boulder", gem_version1, "ruby", resque]] }
-    let(:auth_errors)          { double(:auth_errors) }
+    let(:fail_errors)          { double(:fail_errors) }
     let(:bundler_retry)        { double(:bundler_retry) }
     let(:gem_version1)         { double(:gem_version1) }
     let(:resque)               { double(:resque) }
     let(:remote_uri)           { "http://remote-uri.org" }
 
     before do
-      stub_const("Bundler::Fetcher::AUTH_ERRORS", auth_errors)
-      allow(Bundler::Retry).to receive(:new).with("dependency api", auth_errors).and_return(bundler_retry)
+      stub_const("Bundler::Fetcher::FAIL_ERRORS", fail_errors)
+      allow(Bundler::Retry).to receive(:new).with("dependency api", fail_errors).and_return(bundler_retry)
       allow(bundler_retry).to receive(:attempts) {|&block| block.call }
       allow(subject).to receive(:log_specs) {}
       allow(subject).to receive(:remote_uri).and_return(remote_uri)
@@ -90,7 +90,7 @@ describe Bundler::Fetcher::Dependency do
       before { allow(subject).to receive(:dependency_specs).with(["foo"]).and_return(dependency_specs) }
 
       it "should return a hash with the remote_uri and the list of specs" do
-        expect(subject.specs(gem_names, full_dependency_list, last_spec_list)).to eq("http://remote-uri.org" => [
+        expect(subject.specs(gem_names, full_dependency_list, last_spec_list)).to eq([
           ["top", gem_version2, "ruby", faraday],
           ["boulder", gem_version1, "ruby", resque]
         ])
@@ -103,7 +103,7 @@ describe Bundler::Fetcher::Dependency do
       let(:last_spec_list)       { ["boulder"] }
 
       it "should return a hash with the remote_uri and the last spec list" do
-        expect(subject.specs(gem_names, full_dependency_list, last_spec_list)).to eq("http://remote-uri.org" => ["boulder"])
+        expect(subject.specs(gem_names, full_dependency_list, last_spec_list)).to eq(["boulder"])
       end
     end
 
@@ -206,16 +206,16 @@ describe Bundler::Fetcher::Dependency do
   describe "#unmarshalled_dep_gems" do
     let(:gem_names)         { [%w(foo bar), %w(bundler rubocop)] }
     let(:dep_api_uri)       { double(:dep_api_uri) }
-    let(:marshalled_deps)   { double(:marshalled_deps) }
     let(:unmarshalled_gems) { double(:unmarshalled_gems) }
+    let(:fetch_response)    { double(:fetch_response, :body => double(:body)) }
     let(:rubygems_limit)    { 50 }
 
     before { allow(subject).to receive(:dependency_api_uri).with(gem_names).and_return(dep_api_uri) }
 
     it "should fetch dependencies from Rubygems and unmarshal them" do
       expect(gem_names).to receive(:each_slice).with(rubygems_limit).and_call_original
-      expect(downloader).to receive(:fetch).with(dep_api_uri).and_return([marshalled_deps])
-      expect(Bundler).to receive(:load_marshal).with([marshalled_deps]).and_return([unmarshalled_gems])
+      expect(downloader).to receive(:fetch).with(dep_api_uri).and_return(fetch_response)
+      expect(Bundler).to receive(:load_marshal).with(fetch_response.body).and_return([unmarshalled_gems])
       expect(subject.unmarshalled_dep_gems(gem_names)).to eq([unmarshalled_gems])
     end
   end
@@ -241,26 +241,11 @@ describe Bundler::Fetcher::Dependency do
         }
       ]
     end
-    let(:gem_version1) { double(:gem_version1) }
-    let(:gem_version2) { double(:gem_version2) }
-
-    before do
-      %w(faraday resque).each do |dep_name|
-        instance_variable_set(:"@#{dep_name}", double(dep_name.to_sym))
-        dep_double = instance_variable_get(:"@#{dep_name}")
-        allow(subject).to receive(:well_formed_dependency).with(dep_name, anything).and_return(dep_double)
-        allow(dep_double).to receive(:name).and_return(dep_name)
-      end
-      allow(Gem::Version).to receive(:new).with("1.0.1").and_return(gem_version1)
-      allow(Gem::Version).to receive(:new).with("2.0.2").and_return(gem_version2)
-    end
 
     it "should return formatted specs and a unique list of dependencies" do
       spec_list, deps_list = subject.get_formatted_specs_and_deps(gem_list)
-      expect(spec_list).to eq([
-        ["typhoeus", gem_version1, "ruby", [@resque]],
-        ["grape", gem_version2, "jruby", [@faraday]]
-      ])
+      expect(spec_list).to eq([["typhoeus", "1.0.1", "ruby", [["resque", ["req3,req4"]]]],
+                               ["grape", "2.0.2", "jruby", [["faraday", ["req1,req2"]]]]])
       expect(deps_list).to eq(%w(resque faraday))
     end
   end
@@ -290,46 +275,4 @@ describe Bundler::Fetcher::Dependency do
       end
     end
   end
-
-  describe "#well_formed_dependency" do
-    let(:name)           { "foo" }
-    let(:requirement1)   { "req1" }
-    let(:requirement2)   { "req2" }
-    let(:gem_dependency) { double(:gem_dependency) }
-
-    before { allow(Gem::Dependency).to receive(:new).and_return(gem_dependency) }
-
-    it "should return a Gem::Dependency" do
-      expect(Gem::Dependency).to receive(:new).with("foo", "req1", "req2")
-      subject.well_formed_dependency(name, requirement1, requirement2)
-    end
-
-    context "when an ArgumentError occurs" do
-      before do
-        allow(Gem::Dependency).to receive(:new).with("foo", "req1", "req2") {
-          raise ArgumentError.new("Some error occurred")
-        }
-      end
-
-      it "should raise the original error" do
-        expect { subject.well_formed_dependency(name, requirement1, requirement2) }.to raise_error(
-          ArgumentError, "Some error occurred")
-      end
-    end
-
-    context "when there is an ill formed requirement" do
-      before do
-        allow(Gem::Dependency).to receive(:new).with("foo", "req1", "req2") {
-          raise ArgumentError.new("Ill-formed requirement [\"#<YAML::Syck::DefaultKey")
-        }
-        # Eliminate extra line break in rspec output due to `puts` in `#well_formed_dependency`
-        allow(subject).to receive(:puts) {}
-      end
-
-      it "should raise a Bundler::GemspecError with invalid gemspec message" do
-        expect { subject.well_formed_dependency(name, requirement1, requirement2) }.to raise_error(
-          Bundler::GemspecError, /Unfortunately, the gem foo has an invalid gemspec/)
-      end
-    end
-  end
 end
diff --git a/spec/bundler/rubygems_integration_spec.rb b/spec/bundler/rubygems_integration_spec.rb
index dc34d3d1e6..84ce709913 100644
--- a/spec/bundler/rubygems_integration_spec.rb
+++ b/spec/bundler/rubygems_integration_spec.rb
@@ -42,7 +42,7 @@ describe Bundler::RubygemsIntegration do
         expect(fetcher).to receive(:fetch_path).with(uri + "specs.4.8.gz").and_return(specs_response)
         expect(fetcher).to receive(:fetch_path).with(uri + "prerelease_specs.4.8.gz").and_return(prerelease_specs_response)
         result = Bundler.rubygems.fetch_all_remote_specs(remote_with_mirror)
-        expect(result).to eq(uri => %w(specs prerelease_specs))
+        expect(result).to eq(%w(specs prerelease_specs))
       end
     end
 
@@ -55,7 +55,7 @@ describe Bundler::RubygemsIntegration do
         expect(fetcher).to receive(:fetch_path).with(uri + "specs.4.8.gz").and_return(specs_response)
         expect(fetcher).to receive(:fetch_path).with(uri + "prerelease_specs.4.8.gz").and_return(prerelease_specs_response)
         result = Bundler.rubygems.fetch_all_remote_specs(remote_no_mirror)
-        expect(result).to eq(uri => %w(specs prerelease_specs))
+        expect(result).to eq(%w(specs prerelease_specs))
       end
     end
   end
diff --git a/spec/bundler/source/rubygems/remote_spec.rb b/spec/bundler/source/rubygems/remote_spec.rb
index 2842077498..599887d47a 100644
--- a/spec/bundler/source/rubygems/remote_spec.rb
+++ b/spec/bundler/source/rubygems/remote_spec.rb
@@ -6,6 +6,10 @@ describe Bundler::Source::Rubygems::Remote do
     Bundler::Source::Rubygems::Remote.new(uri)
   end
 
+  before do
+    allow(Digest::MD5).to receive(:hexdigest).with(duck_type(:to_s)) {|string| "MD5HEX(#{string})" }
+  end
+
   let(:uri_no_auth) { URI("https://gems.example.com") }
   let(:uri_with_auth) { URI("https://#{credentials}@gems.example.com") }
   let(:credentials) { "username:password" }
@@ -32,6 +36,17 @@ describe Bundler::Source::Rubygems::Remote do
         expect(remote(uri_no_auth).anonymized_uri).to eq(uri_no_auth)
       end
     end
+
+    describe "#cache_slug" do
+      it "returns the correct slug" do
+        expect(remote(uri_no_auth).cache_slug).to eq("gems.example.com.443.MD5HEX(gems.example.com.443./)")
+      end
+
+      it "only applies the given user" do
+        Bundler.settings[uri_no_auth.to_s] = credentials
+        expect(remote(uri_no_auth).cache_slug).to eq("gems.example.com.username.443.MD5HEX(gems.example.com.username.443./)")
+      end
+    end
   end
 
   context "when the original URI has a username and password" do
@@ -56,6 +71,17 @@ describe Bundler::Source::Rubygems::Remote do
         expect(remote(uri_with_auth).anonymized_uri).to eq(uri_no_auth)
       end
     end
+
+    describe "#cache_slug" do
+      it "returns the correct slug" do
+        expect(remote(uri_with_auth).cache_slug).to eq("gems.example.com.username.443.MD5HEX(gems.example.com.username.443./)")
+      end
+
+      it "does not apply given credentials" do
+        Bundler.settings[uri_with_auth.to_s] = credentials
+        expect(remote(uri_with_auth).cache_slug).to eq("gems.example.com.username.443.MD5HEX(gems.example.com.username.443./)")
+      end
+    end
   end
 
   context "when the original URI has only a username" do
@@ -66,6 +92,12 @@ describe Bundler::Source::Rubygems::Remote do
         expect(remote(uri).anonymized_uri).to eq(URI("https://gem.fury.io/me/"))
       end
     end
+
+    describe "#cache_slug" do
+      it "returns the correct slug" do
+        expect(remote(uri).cache_slug).to eq("gem.fury.io.SeCrEt-ToKeN.443.MD5HEX(gem.fury.io.SeCrEt-ToKeN.443./me/)")
+      end
+    end
   end
 
   context "when a mirror with inline credentials is configured for the URI" do
@@ -86,6 +118,10 @@ describe Bundler::Source::Rubygems::Remote do
     specify "#original_uri returns the original source" do
       expect(remote(uri).original_uri).to eq(uri)
     end
+
+    specify "#cache_slug returns the correct slug" do
+      expect(remote(uri).cache_slug).to eq("rubygems.org.443.MD5HEX(rubygems.org.443./)")
+    end
   end
 
   context "when a mirror with configured credentials is configured for the URI" do
@@ -109,6 +145,10 @@ describe Bundler::Source::Rubygems::Remote do
     specify "#original_uri returns the original source" do
       expect(remote(uri).original_uri).to eq(uri)
     end
+
+    specify "#cache_slug returns the original source" do
+      expect(remote(uri).cache_slug).to eq("rubygems.org.443.MD5HEX(rubygems.org.443./)")
+    end
   end
 
   context "when there is no mirror set" do
diff --git a/spec/commands/update_spec.rb b/spec/commands/update_spec.rb
index a23407dbd6..4eadd956f1 100644
--- a/spec/commands/update_spec.rb
+++ b/spec/commands/update_spec.rb
@@ -18,6 +18,7 @@ describe "bundle update" do
       end
 
       bundle "update"
+      expect(out).to include("Bundle updated!")
       should_be_installed "rack 1.2", "rack-obama 1.0", "activesupport 3.0"
     end
 
@@ -34,14 +35,9 @@ describe "bundle update" do
   end
 
   describe "--quiet argument" do
-    it "shows UI messages without --quiet argument" do
-      bundle "update"
-      expect(out).to include("Fetching source")
-    end
-
-    it "does not show UI messages with --quiet argument" do
+    it "hides UI messages" do
       bundle "update --quiet"
-      expect(out).not_to include("Fetching source")
+      expect(out).not_to include("Bundle updated!")
     end
   end
 
diff --git a/spec/install/gems/compact_index_spec.rb b/spec/install/gems/compact_index_spec.rb
new file mode 100644
index 0000000000..1069027a9d
--- /dev/null
+++ b/spec/install/gems/compact_index_spec.rb
@@ -0,0 +1,631 @@
+require "spec_helper"
+
+describe "compact index api" do
+  let(:source_hostname) { "localgemserver.test" }
+  let(:source_uri) { "http://#{source_hostname}" }
+
+  it "should use the API" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle! :install, :artifice => "compact_index"
+    expect(out).to include("Fetching gem metadata from #{source_uri}")
+    should_be_installed "rack 1.0.0"
+  end
+
+  it "should URI encode gem names" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem " sinatra"
+    G
+
+    bundle :install, :artifice => "compact_index"
+    expect(out).to include("' sinatra' is not a valid gem name because it contains whitespace.")
+  end
+
+  it "should handle nested dependencies" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rails"
+    G
+
+    bundle! :install, :artifice => "compact_index"
+    expect(out).to include("Fetching gem metadata from #{source_uri}")
+    should_be_installed(
+      "rails 2.3.2",
+      "actionpack 2.3.2",
+      "activerecord 2.3.2",
+      "actionmailer 2.3.2",
+      "activeresource 2.3.2",
+      "activesupport 2.3.2")
+  end
+
+  it "should handle multiple gem dependencies on the same gem" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "net-sftp"
+    G
+
+    bundle! :install, :artifice => "compact_index"
+    should_be_installed "net-sftp 1.1.1"
+  end
+
+  it "should use the endpoint when using --deployment" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+    bundle! :install, :artifice => "compact_index"
+
+    bundle "install --deployment", :artifice => "compact_index"
+    expect(out).to include("Fetching gem metadata from #{source_uri}")
+    should_be_installed "rack 1.0.0"
+  end
+
+  it "handles git dependencies that are in rubygems" do
+    build_git "foo" do |s|
+      s.executables = "foobar"
+      s.add_dependency "rails", "2.3.2"
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      git "file:///#{lib_path("foo-1.0")}" do
+        gem 'foo'
+      end
+    G
+
+    bundle! :install, :artifice => "compact_index"
+
+    should_be_installed("rails 2.3.2")
+  end
+
+  it "handles git dependencies that are in rubygems using --deployment" do
+    build_git "foo" do |s|
+      s.executables = "foobar"
+      s.add_dependency "rails", "2.3.2"
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      gem 'foo', :git => "file:///#{lib_path("foo-1.0")}"
+    G
+
+    bundle! :install, :artifice => "compact_index"
+
+    bundle "install --deployment", :artifice => "compact_index"
+
+    should_be_installed("rails 2.3.2")
+  end
+
+  it "doesn't fail if you only have a git gem with no deps when using --deployment" do
+    build_git "foo"
+    gemfile <<-G
+      source "#{source_uri}"
+      gem 'foo', :git => "file:///#{lib_path("foo-1.0")}"
+    G
+
+    bundle "install", :artifice => "compact_index"
+    bundle "install --deployment", :artifice => "compact_index"
+
+    expect(exitstatus).to eq(0) if exitstatus
+    should_be_installed("foo 1.0")
+  end
+
+  it "falls back when the API errors out" do
+    simulate_platform mswin
+
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rcov"
+    G
+
+    bundle! :install, :fakeweb => "windows"
+    expect(out).to include("Fetching source index from #{source_uri}")
+    should_be_installed "rcov 1.0.0"
+  end
+
+  it "falls back when the API URL returns 403 Forbidden" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle! :install, :verbose => true, :artifice => "compact_index_forbidden"
+    expect(out).to include("Fetching gem metadata from #{source_uri}")
+    should_be_installed "rack 1.0.0"
+  end
+
+  it "handles host redirects" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle! :install, :artifice => "compact_index_host_redirect"
+    should_be_installed "rack 1.0.0"
+  end
+
+  it "handles host redirects without Net::HTTP::Persistent" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    FileUtils.mkdir_p lib_path
+    File.open(lib_path("disable_net_http_persistent.rb"), "w") do |h|
+      h.write <<-H
+        module Kernel
+          alias require_without_disabled_net_http require
+          def require(*args)
+            raise LoadError, 'simulated' if args.first == 'openssl' && !caller.grep(/vendored_persistent/).empty?
+            require_without_disabled_net_http(*args)
+          end
+        end
+      H
+    end
+
+    bundle! :install, :artifice => "compact_index_host_redirect", :requires => [lib_path("disable_net_http_persistent.rb")]
+    expect(out).to_not match(/Too many redirects/)
+    should_be_installed "rack 1.0.0"
+  end
+
+  it "times out when Bundler::Fetcher redirects too much" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle :install, :artifice => "compact_index_redirects"
+    expect(out).to match(/Too many redirects/)
+  end
+
+  context "when --full-index is specified" do
+    it "should use the modern index for install" do
+      gemfile <<-G
+        source "#{source_uri}"
+        gem "rack"
+      G
+
+      bundle "install --full-index", :artifice => "compact_index"
+      expect(out).to include("Fetching source index from #{source_uri}")
+      should_be_installed "rack 1.0.0"
+    end
+
+    it "should use the modern index for update" do
+      gemfile <<-G
+        source "#{source_uri}"
+        gem "rack"
+      G
+
+      bundle "update --full-index", :artifice => "compact_index"
+      expect(out).to include("Fetching source index from #{source_uri}")
+      should_be_installed "rack 1.0.0"
+    end
+  end
+
+  it "fetches again when more dependencies are found in subsequent sources" do
+    build_repo2 do
+      build_gem "back_deps" do |s|
+        s.add_dependency "foo"
+      end
+      FileUtils.rm_rf Dir[gem_repo2("gems/foo-*.gem")]
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      source "#{source_uri}/extra"
+      gem "back_deps"
+    G
+
+    bundle! :install, :artifice => "compact_index_extra"
+    should_be_installed "back_deps 1.0"
+  end
+
+  it "fetches gem versions even when those gems are already installed" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack", "1.0.0"
+    G
+    bundle! :install, :artifice => "compact_index_extra_api"
+    should_be_installed "rack 1.0.0"
+
+    build_repo4 do
+      build_gem "rack", "1.2" do |s|
+        s.executables = "rackup"
+      end
+    end
+
+    gemfile <<-G
+      source "#{source_uri}" do; end
+      source "#{source_uri}/extra"
+      gem "rack", "1.2"
+    G
+    bundle! :install, :artifice => "compact_index_extra_api"
+    should_be_installed "rack 1.2"
+  end
+
+  it "considers all possible versions of dependencies from all api gem sources" do
+    # In this scenario, the gem "somegem" only exists in repo4.  It depends on specific version of activesupport that
+    # exists only in repo1.  There happens also be a version of activesupport in repo4, but not the one that version 1.0.0
+    # of somegem wants. This test makes sure that bundler actually finds version 1.2.3 of active support in the other
+    # repo and installs it.
+    build_repo4 do
+      build_gem "activesupport", "1.2.0"
+      build_gem "somegem", "1.0.0" do |s|
+        s.add_dependency "activesupport", "1.2.3" # This version exists only in repo1
+      end
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      source "#{source_uri}/extra"
+      gem 'somegem', '1.0.0'
+    G
+
+    bundle! :install, :artifice => "compact_index_extra_api"
+
+    should_be_installed "somegem 1.0.0"
+    should_be_installed "activesupport 1.2.3"
+  end
+
+  it "prints API output properly with back deps" do
+    build_repo2 do
+      build_gem "back_deps" do |s|
+        s.add_dependency "foo"
+      end
+      FileUtils.rm_rf Dir[gem_repo2("gems/foo-*.gem")]
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      source "#{source_uri}/extra"
+      gem "back_deps"
+    G
+
+    bundle! :install, :artifice => "compact_index_extra"
+
+    expect(out).to include("Fetching gem metadata from http://localgemserver.test/")
+    expect(out).to include("Fetching source index from http://localgemserver.test/extra")
+  end
+
+  it "does not fetch every spec if the index of gems is large when doing back deps" do
+    build_repo2 do
+      build_gem "back_deps" do |s|
+        s.add_dependency "foo"
+      end
+      build_gem "missing"
+      # need to hit the limit
+      1.upto(Bundler::Source::Rubygems::API_REQUEST_LIMIT) do |i|
+        build_gem "gem#{i}"
+      end
+
+      FileUtils.rm_rf Dir[gem_repo2("gems/foo-*.gem")]
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      source "#{source_uri}/extra"
+      gem "back_deps"
+    G
+
+    bundle! :install, :artifice => "compact_index_extra_missing"
+    should_be_installed "back_deps 1.0"
+  end
+
+  it "uses the endpoint if all sources support it" do
+    gemfile <<-G
+      source "#{source_uri}"
+
+      gem 'foo'
+    G
+
+    bundle! :install, :artifice => "compact_index_api_missing"
+    should_be_installed "foo 1.0"
+  end
+
+  it "fetches again when more dependencies are found in subsequent sources using --deployment" do
+    build_repo2 do
+      build_gem "back_deps" do |s|
+        s.add_dependency "foo"
+      end
+      FileUtils.rm_rf Dir[gem_repo2("gems/foo-*.gem")]
+    end
+
+    gemfile <<-G
+      source "#{source_uri}"
+      source "#{source_uri}/extra"
+      gem "back_deps"
+    G
+
+    bundle! :install, :artifice => "compact_index_extra"
+
+    bundle "install --deployment", :artifice => "compact_index_extra"
+    should_be_installed "back_deps 1.0"
+  end
+
+  it "does not refetch if the only unmet dependency is bundler" do
+    gemfile <<-G
+      source "#{source_uri}"
+
+      gem "bundler_dep"
+    G
+
+    bundle! :install, :artifice => "compact_index"
+    expect(out).to include("Fetching gem metadata from #{source_uri}")
+  end
+
+  it "should install when EndpointSpecification has a bin dir owned by root", :sudo => true do
+    sudo "mkdir -p #{system_gem_path("bin")}"
+    sudo "chown -R root #{system_gem_path("bin")}"
+
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rails"
+    G
+    bundle! :install, :artifice => "compact_index"
+    should_be_installed "rails 2.3.2"
+  end
+
+  it "installs the binstubs" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle "install --binstubs", :artifice => "compact_index"
+
+    gembin "rackup"
+    expect(out).to eq("1.0.0")
+  end
+
+  it "installs the bins when using --path and uses autoclean" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle "install --path vendor/bundle", :artifice => "compact_index"
+
+    expect(vendored_gems("bin/rackup")).to exist
+  end
+
+  it "installs the bins when using --path and uses bundle clean" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem "rack"
+    G
+
+    bundle "install --path vendor/bundle --no-clean", :artifice => "compact_index"
+
+    expect(vendored_gems("bin/rackup")).to exist
+  end
+
+  it "prints post_install_messages" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem 'rack-obama'
+    G
+
+    bundle! :install, :artifice => "compact_index"
+    expect(out).to include("Post-install message from rack:")
+  end
+
+  it "should display the post install message for a dependency" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem 'rack_middleware'
+    G
+
+    bundle! :install, :artifice => "compact_index"
+    expect(out).to include("Post-install message from rack:")
+    expect(out).to include("Rack's post install message")
+  end
+
+  context "when using basic authentication" do
+    let(:user)     { "user" }
+    let(:password) { "pass" }
+    let(:basic_auth_source_uri) do
+      uri          = URI.parse(source_uri)
+      uri.user     = user
+      uri.password = password
+
+      uri
+    end
+
+    it "passes basic authentication details and strips out creds" do
+      gemfile <<-G
+        source "#{basic_auth_source_uri}"
+        gem "rack"
+      G
+
+      bundle! :install, :artifice => "compact_index_basic_authentication"
+      expect(out).not_to include("#{user}:#{password}")
+      should_be_installed "rack 1.0.0"
+    end
+
+    it "strips http basic authentication creds for modern index" do
+      gemfile <<-G
+        source "#{basic_auth_source_uri}"
+        gem "rack"
+      G
+
+      bundle! :install, :artifice => "endopint_marshal_fail_basic_authentication"
+      expect(out).not_to include("#{user}:#{password}")
+      should_be_installed "rack 1.0.0"
+    end
+
+    it "strips http basic auth creds when it can't reach the server" do
+      gemfile <<-G
+        source "#{basic_auth_source_uri}"
+        gem "rack"
+      G
+
+      bundle :install, :artifice => "endpoint_500"
+      expect(out).not_to include("#{user}:#{password}")
+    end
+
+    it "strips http basic auth creds when warning about ambiguous sources" do
+      gemfile <<-G
+        source "#{basic_auth_source_uri}"
+        source "file://#{gem_repo1}"
+        gem "rack"
+      G
+
+      bundle! :install, :artifice => "compact_index_basic_authentication"
+      expect(out).to include("Warning: the gem 'rack' was found in multiple sources.")
+      expect(out).not_to include("#{user}:#{password}")
+      should_be_installed "rack 1.0.0"
+    end
+
+    it "does not pass the user / password to different hosts on redirect" do
+      gemfile <<-G
+        source "#{basic_auth_source_uri}"
+        gem "rack"
+      G
+
+      bundle! :install, :artifice => "compact_index_creds_diff_host"
+      should_be_installed "rack 1.0.0"
+    end
+
+    describe "with authentication details in bundle config" do
+      before do
+        gemfile <<-G
+          source "#{source_uri}"
+          gem "rack"
+        G
+      end
+
+      it "reads authentication details by host name from bundle config" do
+        bundle "config #{source_hostname} #{user}:#{password}"
+
+        bundle! :install, :artifice => "compact_index_strict_basic_authentication"
+
+        expect(out).to include("Fetching gem metadata from #{source_uri}")
+        should_be_installed "rack 1.0.0"
+      end
+
+      it "reads authentication details by full url from bundle config" do
+        # The trailing slash is necessary here; Fetcher canonicalizes the URI.
+        bundle "config #{source_uri}/ #{user}:#{password}"
+
+        bundle! :install, :artifice => "compact_index_strict_basic_authentication"
+
+        expect(out).to include("Fetching gem metadata from #{source_uri}")
+        should_be_installed "rack 1.0.0"
+      end
+
+      it "should use the API" do
+        bundle "config #{source_hostname} #{user}:#{password}"
+        bundle! :install, :artifice => "compact_index_strict_basic_authentication"
+        expect(out).to include("Fetching gem metadata from #{source_uri}")
+        should_be_installed "rack 1.0.0"
+      end
+
+      it "prefers auth supplied in the source uri" do
+        gemfile <<-G
+          source "#{basic_auth_source_uri}"
+          gem "rack"
+        G
+
+        bundle "config #{source_hostname} otheruser:wrong"
+
+        bundle! :install, :artifice => "compact_index_strict_basic_authentication"
+        should_be_installed "rack 1.0.0"
+      end
+
+      it "shows instructions if auth is not provided for the source" do
+        bundle :install, :artifice => "compact_index_strict_basic_authentication"
+        expect(out).to include("bundle config #{source_hostname} username:password")
+      end
+
+      it "fails if authentication has already been provided, but failed" do
+        bundle "config #{source_hostname} #{user}:wrong"
+
+        bundle :install, :artifice => "compact_index_strict_basic_authentication"
+        expect(out).to include("Bad username or password")
+      end
+    end
+
+    describe "with no password" do
+      let(:password) { nil }
+
+      it "passes basic authentication details" do
+        gemfile <<-G
+          source "#{basic_auth_source_uri}"
+          gem "rack"
+        G
+
+        bundle! :install, :artifice => "compact_index_basic_authentication"
+        should_be_installed "rack 1.0.0"
+      end
+    end
+  end
+
+  context "when ruby is compiled without openssl" do
+    before do
+      # Install a monkeypatch that reproduces the effects of openssl being
+      # missing when the fetcher runs, as happens in real life. The reason
+      # we can't just overwrite openssl.rb is that Artifice uses it.
+      bundled_app("broken_ssl").mkpath
+      bundled_app("broken_ssl/openssl.rb").open("w") do |f|
+        f.write <<-RUBY
+          raise LoadError, "cannot load such file -- openssl"
+        RUBY
+      end
+    end
+
+    it "explains what to do to get it" do
+      gemfile <<-G
+        source "#{source_uri.gsub(/http/, "https")}"
+        gem "rack"
+      G
+
+      bundle :install, :env => { "RUBYOPT" => "-I#{bundled_app("broken_ssl")}" }
+      expect(out).to include("OpenSSL")
+    end
+  end
+
+  context "when SSL certificate verification fails" do
+    it "explains what happened" do
+      # Install a monkeypatch that reproduces the effects of openssl raising
+      # a certificate validation error when Rubygems tries to connect.
+      gemfile <<-G
+        class Net::HTTP
+          def start
+            raise OpenSSL::SSL::SSLError, "certificate verify failed"
+          end
+        end
+
+        source "#{source_uri.gsub(/http/, "https")}"
+        gem "rack"
+      G
+
+      bundle :install
+      expect(out).to match(/could not verify the SSL certificate/i)
+    end
+  end
+
+  context ".gemrc with sources is present" do
+    before do
+      File.open(home(".gemrc"), "w") do |file|
+        file.puts({ :sources => ["https://rubygems.org"] }.to_yaml)
+      end
+    end
+
+    after do
+      home(".gemrc").rmtree
+    end
+
+    it "uses other sources declared in the Gemfile" do
+      gemfile <<-G
+        source "#{source_uri}"
+        gem 'rack'
+      G
+
+      bundle! :install, :artifice => "compact_index_forbidden"
+    end
+  end
+end
diff --git a/spec/quality_spec.rb b/spec/quality_spec.rb
index c9c7e7d8e7..bbe78a4079 100644
--- a/spec/quality_spec.rb
+++ b/spec/quality_spec.rb
@@ -86,16 +86,23 @@ describe "The library itself" do
   end
 
   it "does not contain any warnings" do
-    Dir.chdir(root.join("lib"))
-    exclusions = %r{bundler/capistrano\.rb|bundler/vlad\.rb|bundler/gem_tasks\.rb|tmp/rubygems}
-    lib_files = `git ls-files -z -- **/*.rb`.split("\x0").reject {|f| f =~ exclusions }
-    sys_exec("ruby -w -I. ", :expect_err) do |input|
-      lib_files.each do |f|
-        input.puts "require '#{f.gsub(/\.rb$/, "")}'"
+    Dir.chdir(root.join("lib")) do
+      exclusions = %w(
+        bundler/capistrano.rb
+        bundler/gem_tasks.rb
+        bundler/vlad.rb
+      )
+      lib_files = `git ls-files -z`.split("\x0").grep(/\.rb$/) - exclusions
+      lib_files.reject! {|f| f.start_with?("bundler/vendor") }
+      lib_files.map! {|f| f.chomp(".rb") }
+      sys_exec("ruby -w -I. ", :expect_err) do |input|
+        lib_files.each do |f|
+          input.puts "require '#{f}'"
+        end
       end
-    end
 
-    expect(@err.split("\n")).to eq([])
-    expect(@out).to eq("")
+      expect(@err.split("\n")).to be_well_formed
+      expect(@out.split("\n")).to be_well_formed
+    end
   end
 end
diff --git a/spec/realworld/gemfile_source_header_spec.rb b/spec/realworld/gemfile_source_header_spec.rb
index 70039fa640..6c2c05eaa7 100644
--- a/spec/realworld/gemfile_source_header_spec.rb
+++ b/spec/realworld/gemfile_source_header_spec.rb
@@ -1,7 +1,7 @@
 require "spec_helper"
 require "thread"
 
-describe "fetching dependencies with a mirrored source", :rubygems => ">= 2.0" do
+describe "fetching dependencies with a mirrored source", :realworld => true, :rubygems => ">= 2.0" do
   let(:mirror) { "https://server.example.org" }
   let(:original) { "http://127.0.0.1:#{@port}" }
 
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index c392b4f1bb..cd2dc39469 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -57,6 +57,9 @@ RSpec.configure do |config|
   config.include Spec::Sudo
   config.include Spec::Permissions
 
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
   if ENV["BUNDLER_SUDO_TESTS"] && Spec::Sudo.present?
     config.filter_run :sudo => true
   else
diff --git a/spec/support/artifice/compact_index.rb b/spec/support/artifice/compact_index.rb
new file mode 100644
index 0000000000..e7dfda3bea
--- /dev/null
+++ b/spec/support/artifice/compact_index.rb
@@ -0,0 +1,106 @@
+require File.expand_path("../endpoint", __FILE__)
+
+$LOAD_PATH.unshift "#{Dir[base_system_gems.join("gems/compact_index*/lib")].first}"
+require "compact_index"
+
+class CompactIndexAPI < Endpoint
+  helpers do
+    def load_spec(name, version, platform, gem_repo)
+      full_name = "#{name}-#{version}"
+      full_name += "-#{platform}" if platform != "ruby"
+      Marshal.load(Gem.inflate(File.open(gem_repo.join("quick/Marshal.4.8/#{full_name}.gemspec.rz")).read))
+    end
+
+    def etag_response
+      response_body = yield
+      checksum = Digest::MD5.hexdigest(response_body)
+      return if not_modified?(checksum)
+      headers "ETag" => quote(checksum)
+      headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60"
+      content_type "text/plain"
+      requested_range_for(response_body)
+    rescue => e
+      puts e
+      puts e.backtrace
+      raise
+    end
+
+    def not_modified?(checksum)
+      etags = parse_etags(request.env["HTTP_IF_NONE_MATCH"])
+
+      if etags.include?(checksum)
+        headers "ETag" => quote(checksum)
+        status 304
+        body ""
+      end
+    end
+
+    def requested_range_for(response_body)
+      ranges = Rack::Utils.byte_ranges(env, response_body.bytesize)
+
+      if ranges
+        status 206
+        body ranges.map! {|range| response_body.byteslice(range) }.join
+      else
+        status 200
+        body response_body
+      end
+    end
+
+    def quote(string)
+      '"' << string << '"'
+    end
+
+    def parse_etags(value)
+      value ? value.split(/, ?/).select {|s| s.sub!(/"(.*)"/, '\1') } : []
+    end
+
+    def gems(gem_repo = gem_repo1)
+      @gems ||= {}
+      @gems[gem_repo] ||= begin
+        specs = Bundler::Deprecate.skip_during do
+          Marshal.load(File.open(gem_repo.join("specs.4.8")).read).map do |name, version, platform|
+            load_spec(name, version, platform, gem_repo)
+          end
+        end
+
+        specs.group_by(&:name).map do |name, versions|
+          gem_versions = versions.map do |spec|
+            deps = spec.dependencies.select {|d| d.type == :runtime }.map do |d|
+              reqs = d.requirement.requirements.map {|r| r.join(" ") }.join(", ")
+              CompactIndex::Dependency.new(d.name, reqs)
+            end
+            CompactIndex::GemVersion.new(spec.version.version, spec.platform.to_s, nil, nil,
+              deps, spec.required_ruby_version, spec.required_rubygems_version)
+          end
+          CompactIndex::Gem.new(name, gem_versions)
+        end
+      end
+    end
+  end
+
+  get "/names" do
+    etag_response do
+      CompactIndex.names(gems.map(&:name))
+    end
+  end
+
+  get "/versions" do
+    etag_response do
+      file = tmp("versions.list")
+      file.delete if file.file?
+      file = CompactIndex::VersionsFile.new(file.to_s)
+      file.update_with(gems)
+      CompactIndex.versions(file, nil, {})
+    end
+  end
+
+  get "/info/:name" do
+    etag_response do
+      gem = gems.find {|g| g.name == params[:name] }
+      CompactIndex.info(gem ? gem.versions : [])
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexAPI)
diff --git a/spec/support/artifice/compact_index_api_missing.rb b/spec/support/artifice/compact_index_api_missing.rb
new file mode 100644
index 0000000000..db6528b878
--- /dev/null
+++ b/spec/support/artifice/compact_index_api_missing.rb
@@ -0,0 +1,16 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexApiMissing < CompactIndexAPI
+  get "/fetch/actual/gem/:id" do
+    $stderr.puts params[:id]
+    if params[:id] == "rack-1.0.gemspec.rz"
+      halt 404
+    else
+      File.read("#{gem_repo2}/quick/Marshal.4.8/#{params[:id]}")
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexApiMissing)
diff --git a/spec/support/artifice/compact_index_basic_authentication.rb b/spec/support/artifice/compact_index_basic_authentication.rb
new file mode 100644
index 0000000000..5ddcdf7fa7
--- /dev/null
+++ b/spec/support/artifice/compact_index_basic_authentication.rb
@@ -0,0 +1,13 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexBasicAuthentication < CompactIndexAPI
+  before do
+    unless env["HTTP_AUTHORIZATION"]
+      halt 401, "Authentication info not supplied"
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexBasicAuthentication)
diff --git a/spec/support/artifice/compact_index_creds_diff_host.rb b/spec/support/artifice/compact_index_creds_diff_host.rb
new file mode 100644
index 0000000000..6a56c508a2
--- /dev/null
+++ b/spec/support/artifice/compact_index_creds_diff_host.rb
@@ -0,0 +1,38 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexCredsDiffHost < CompactIndexAPI
+  helpers do
+    def auth
+      @auth ||= Rack::Auth::Basic::Request.new(request.env)
+    end
+
+    def authorized?
+      auth.provided? && auth.basic? && auth.credentials && auth.credentials == %w(user pass)
+    end
+
+    def protected!
+      unless authorized?
+        response["WWW-Authenticate"] = %(Basic realm="Testing HTTP Auth")
+        throw(:halt, [401, "Not authorized\n"])
+      end
+    end
+  end
+
+  before do
+    protected! unless request.path_info.include?("/no/creds/")
+  end
+
+  get "/gems/:id" do
+    redirect "http://diffhost.com/no/creds/#{params[:id]}"
+  end
+
+  get "/no/creds/:id" do
+    if request.host.include?("diffhost") && !auth.provided?
+      File.read("#{gem_repo1}/gems/#{params[:id]}")
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexCredsDiffHost)
diff --git a/spec/support/artifice/compact_index_extra.rb b/spec/support/artifice/compact_index_extra.rb
new file mode 100644
index 0000000000..3300b70610
--- /dev/null
+++ b/spec/support/artifice/compact_index_extra.rb
@@ -0,0 +1,35 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexExtra < CompactIndexAPI
+  get "/extra/versions" do
+    halt 404
+  end
+
+  get "/extra/api/v1/dependencies" do
+    halt 404
+  end
+
+  get "/extra/specs.4.8.gz" do
+    File.read("#{gem_repo2}/specs.4.8.gz")
+  end
+
+  get "/extra/prerelease_specs.4.8.gz" do
+    File.read("#{gem_repo2}/prerelease_specs.4.8.gz")
+  end
+
+  get "/extra/quick/Marshal.4.8/:id" do
+    redirect "/extra/fetch/actual/gem/#{params[:id]}"
+  end
+
+  get "/extra/fetch/actual/gem/:id" do
+    File.read("#{gem_repo2}/quick/Marshal.4.8/#{params[:id]}")
+  end
+
+  get "/extra/gems/:id" do
+    File.read("#{gem_repo2}/gems/#{params[:id]}")
+  end
+end
+
+Artifice.activate_with(CompactIndexExtra)
diff --git a/spec/support/artifice/compact_index_extra_api.rb b/spec/support/artifice/compact_index_extra_api.rb
new file mode 100644
index 0000000000..df4674396a
--- /dev/null
+++ b/spec/support/artifice/compact_index_extra_api.rb
@@ -0,0 +1,50 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexExtraApi < CompactIndexAPI
+  get "/extra/names" do
+    etag_response do
+      CompactIndex.names(gems(gem_repo4).map(&:name))
+    end
+  end
+
+  get "/extra/versions" do
+    etag_response do
+      file = tmp("versions.list")
+      file.delete if file.file?
+      file = CompactIndex::VersionsFile.new(file.to_s)
+      file.update_with(gems(gem_repo4))
+      CompactIndex.versions(file, nil, {})
+    end
+  end
+
+  get "/extra/info/:name" do
+    etag_response do
+      gem = gems(gem_repo4).find {|g| g.name == params[:name] }
+      CompactIndex.info(gem ? gem.versions : [])
+    end
+  end
+
+  get "/extra/specs.4.8.gz" do
+    File.read("#{gem_repo4}/specs.4.8.gz")
+  end
+
+  get "/extra/prerelease_specs.4.8.gz" do
+    File.read("#{gem_repo4}/prerelease_specs.4.8.gz")
+  end
+
+  get "/extra/quick/Marshal.4.8/:id" do
+    redirect "/extra/fetch/actual/gem/#{params[:id]}"
+  end
+
+  get "/extra/fetch/actual/gem/:id" do
+    File.read("#{gem_repo4}/quick/Marshal.4.8/#{params[:id]}")
+  end
+
+  get "/extra/gems/:id" do
+    File.read("#{gem_repo4}/gems/#{params[:id]}")
+  end
+end
+
+Artifice.activate_with(CompactIndexExtraApi)
diff --git a/spec/support/artifice/compact_index_extra_missing.rb b/spec/support/artifice/compact_index_extra_missing.rb
new file mode 100644
index 0000000000..48fffd51a8
--- /dev/null
+++ b/spec/support/artifice/compact_index_extra_missing.rb
@@ -0,0 +1,15 @@
+require File.expand_path("../compact_index_extra", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexExtraMissing < CompactIndexExtra
+  get "/extra/fetch/actual/gem/:id" do
+    if params[:id] == "missing-1.0.gemspec.rz"
+      halt 404
+    else
+      File.read("#{gem_repo2}/quick/Marshal.4.8/#{params[:id]}")
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexExtraMissing)
diff --git a/spec/support/artifice/compact_index_forbidden.rb b/spec/support/artifice/compact_index_forbidden.rb
new file mode 100644
index 0000000000..09d03bc88d
--- /dev/null
+++ b/spec/support/artifice/compact_index_forbidden.rb
@@ -0,0 +1,11 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexForbidden < CompactIndexAPI
+  get "/versions" do
+    halt 403
+  end
+end
+
+Artifice.activate_with(CompactIndexForbidden)
diff --git a/spec/support/artifice/compact_index_host_redirect.rb b/spec/support/artifice/compact_index_host_redirect.rb
new file mode 100644
index 0000000000..2e53cc7814
--- /dev/null
+++ b/spec/support/artifice/compact_index_host_redirect.rb
@@ -0,0 +1,19 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexHostRedirect < CompactIndexAPI
+  get "/fetch/actual/gem/:id", :host_name => "localgemserver.test" do
+    redirect "http://bundler.localgemserver.test#{request.path_info}"
+  end
+
+  get "/versions" do
+    status 404
+  end
+
+  get "/api/v1/dependencies" do
+    status 404
+  end
+end
+
+Artifice.activate_with(CompactIndexHostRedirect)
diff --git a/spec/support/artifice/compact_index_redirects.rb b/spec/support/artifice/compact_index_redirects.rb
new file mode 100644
index 0000000000..256e73d7f4
--- /dev/null
+++ b/spec/support/artifice/compact_index_redirects.rb
@@ -0,0 +1,19 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexRedirect < CompactIndexAPI
+  get "/fetch/actual/gem/:id" do
+    redirect "/fetch/actual/gem/#{params[:id]}"
+  end
+
+  get "/versions" do
+    status 404
+  end
+
+  get "/api/v1/dependencies" do
+    status 404
+  end
+end
+
+Artifice.activate_with(CompactIndexRedirect)
diff --git a/spec/support/artifice/compact_index_strict_basic_authentication.rb b/spec/support/artifice/compact_index_strict_basic_authentication.rb
new file mode 100644
index 0000000000..bb1e794b4e
--- /dev/null
+++ b/spec/support/artifice/compact_index_strict_basic_authentication.rb
@@ -0,0 +1,18 @@
+require File.expand_path("../compact_index", __FILE__)
+
+Artifice.deactivate
+
+class CompactIndexStrictBasicAuthentication < CompactIndexAPI
+  before do
+    unless env["HTTP_AUTHORIZATION"]
+      halt 401, "Authentication info not supplied"
+    end
+
+    # Only accepts password == "password"
+    unless env["HTTP_AUTHORIZATION"] == "Basic dXNlcjpwYXNz"
+      halt 403, "Authentication failed"
+    end
+  end
+end
+
+Artifice.activate_with(CompactIndexStrictBasicAuthentication)
diff --git a/spec/support/artifice/endpoint.rb b/spec/support/artifice/endpoint.rb
index 931bbc48e8..847f1d4b23 100644
--- a/spec/support/artifice/endpoint.rb
+++ b/spec/support/artifice/endpoint.rb
@@ -12,6 +12,9 @@ require "artifice"
 require "sinatra/base"
 
 class Endpoint < Sinatra::Base
+  set :raise_errors, true
+  set :show_exceptions, false
+
   helpers do
     def dependencies_for(gem_names, gem_repo = gem_repo1)
       return [] if gem_names.nil? || gem_names.empty?
diff --git a/spec/support/artifice/endpoint_500.rb b/spec/support/artifice/endpoint_500.rb
index 41076bf2c7..2812a0d208 100644
--- a/spec/support/artifice/endpoint_500.rb
+++ b/spec/support/artifice/endpoint_500.rb
@@ -13,23 +13,7 @@ require "sinatra/base"
 Artifice.deactivate
 
 class Endpoint500 < Sinatra::Base
-  get "/quick/Marshal.4.8/:id" do
-    halt 500
-  end
-
-  get "/fetch/actual/gem/:id" do
-    halt 500
-  end
-
-  get "/gems/:id" do
-    halt 500
-  end
-
-  get "/api/v1/dependencies" do
-    halt 500
-  end
-
-  get "/specs.4.8.gz" do
+  before do
     halt 500
   end
 end
diff --git a/spec/support/fakeweb/windows.rb b/spec/support/fakeweb/windows.rb
index 6ff9560d4a..6c674b31b6 100644
--- a/spec/support/fakeweb/windows.rb
+++ b/spec/support/fakeweb/windows.rb
@@ -21,3 +21,6 @@ FakeWeb.register_uri(:get, "http://localgemserver.test/gems/rcov-1.0-x86-mswin32
 
 FakeWeb.register_uri(:get, "http://localgemserver.test/api/v1/dependencies",
   :status => ["404", "Not Found"])
+
+FakeWeb.register_uri(:get, "http://localgemserver.test/versions",
+  :status => ["500", "Internal Server Error"])
diff --git a/spec/support/helpers.rb b/spec/support/helpers.rb
index 09ef6e2945..182f7b9bad 100644
--- a/spec/support/helpers.rb
+++ b/spec/support/helpers.rb
@@ -17,6 +17,20 @@ module Spec
       Bundler.send(:remove_instance_variable, :@settings) if Bundler.send(:instance_variable_defined?, :@settings)
     end
 
+    def self.bang(method)
+      define_method("#{method}!") do |*args, &blk|
+        send(method, *args, &blk).tap do
+          if exitstatus && exitstatus != 0
+            error = out + "\n" + err
+            error.strip!
+            raise RuntimeError,
+              "Invoking #{method}!(#{args.map(&:inspect).join(", ")}) failed:\n#{error}",
+              caller.drop_while {|bt| bt.start_with?(__FILE__) }
+          end
+        end
+      end
+    end
+
     attr_reader :out, :err, :exitstatus
 
     def in_app_root(&blk)
@@ -39,6 +53,7 @@ module Spec
       setup = "require 'rubygems' ; require 'bundler' ; Bundler.setup(#{groups})\n"
       @out = ruby(setup + cmd, :expect_err => expect_err, :env => env)
     end
+    bang :run
 
     def load_error_run(ruby, name, *args)
       cmd = <<-RUBY
@@ -85,6 +100,7 @@ module Spec
       cmd = "#{env} #{sudo} #{Gem.ruby} -I#{lib}:#{spec} #{requires_str} #{bundle_bin} #{cmd}#{args}"
       sys_exec(cmd, expect_err) {|i| yield i if block_given? }
     end
+    bang :bundle
 
     def bundle_ruby(options = {})
       expect_err = options.delete(:expect_err)
@@ -110,6 +126,7 @@ module Spec
       lib_option = options[:no_lib] ? "" : " -I#{lib}"
       sys_exec(%(#{env}#{Gem.ruby}#{lib_option} -e "#{ruby}"), expect_err)
     end
+    bang :ruby
 
     def load_error_ruby(ruby, name, opts = {})
       cmd = <<-R
@@ -149,6 +166,7 @@ module Spec
       puts @err unless expect_err || @err.empty? || !$show_err
       @out
     end
+    bang :sys_exec
 
     def config(config = nil, path = bundled_app(".bundle/config"))
       return YAML.load_file(path) unless config
diff --git a/spec/support/matchers.rb b/spec/support/matchers.rb
index 6c585ddae4..58babf2ceb 100644
--- a/spec/support/matchers.rb
+++ b/spec/support/matchers.rb
@@ -31,8 +31,9 @@ module Spec
       names.each do |name|
         name, version, platform = name.split(/\s+/)
         version_const = name == "bundler" ? "Bundler::VERSION" : Spec::Builders.constantize(name)
-        run "require '#{name}.rb'; puts #{version_const}", *groups
-        actual_version, actual_platform = out.split(/\s+/)
+        run! "require '#{name}.rb'; puts #{version_const}", *groups
+        expect(out).not_to be_empty, "#{name} is not installed"
+        actual_version, actual_platform = out.split(/\s+/, 2)
         expect(Gem::Version.new(actual_version)).to eq(Gem::Version.new(version))
         expect(actual_platform).to eq(platform)
       end
@@ -44,7 +45,7 @@ module Spec
       opts = names.last.is_a?(Hash) ? names.pop : {}
       groups = Array(opts[:groups]) || []
       names.each do |name|
-        name, version = name.split(/\s+/)
+        name, version = name.split(/\s+/, 2)
         run <<-R, *(groups + [opts])
           begin
             require '#{name}'
diff --git a/spec/support/rubygems_ext.rb b/spec/support/rubygems_ext.rb
index 8ccb48ae69..2889f44040 100644
--- a/spec/support/rubygems_ext.rb
+++ b/spec/support/rubygems_ext.rb
@@ -5,7 +5,7 @@ module Spec
   module Rubygems
     DEPS = begin
       deps = {
-        "fakeweb artifice rack" => nil,
+        "fakeweb artifice rack compact_index" => nil,
         "sinatra" => "1.2.7",
         # Rake version has to be consistent for tests to pass
         "rake" => "10.0.2",