Auto merge of #5007 - bundler:seg-unvendor-compact-index-client, r=indirect

Un-vendor the compact index client

Step one to https://github.com/bundler/bundler/issues/4935

9 files changed, 310 insertions, 305 deletions

diff --git a/Rakefile b/Rakefile
index cc7162a6ef..178d481655 100644
--- a/Rakefile
+++ b/Rakefile
@@ -285,13 +285,6 @@ end
 begin
   require "automatiek"
 
-  Automatiek::RakeTask.new("compact_index_client") do |lib|
-    lib.download = { :github => "https://github.com/bundler/compact_index_client" }
-    lib.namespace = "CompactIndexClient"
-    lib.prefix = "Bundler"
-    lib.vendor_lib = "lib/bundler/vendor/compact_index_client"
-  end
-
   Automatiek::RakeTask.new("molinillo") do |lib|
     lib.download = { :github => "https://github.com/CocoaPods/Molinillo" }
     lib.namespace = "Molinillo"
diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb
new file mode 100644
index 0000000000..1ff3deda01
--- /dev/null
+++ b/lib/bundler/compact_index_client.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+require "pathname"
+require "set"
+
+module Bundler
+  class CompactIndexClient
+    DEBUG_MUTEX = Mutex.new
+    def self.debug
+      return unless ENV["DEBUG_COMPACT_INDEX"]
+      DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }
+    end
+
+    class Error < StandardError; end
+
+    require "bundler/compact_index_client/cache"
+    require "bundler/compact_index_client/updater"
+
+    attr_reader :directory
+
+    # @return [Lambda] A lambda that takes an array of inputs and a block, and
+    #         maps the inputs with the block in parallel.
+    #
+    attr_accessor :in_parallel
+
+    def initialize(directory, fetcher)
+      @directory = Pathname.new(directory)
+      @updater = Updater.new(fetcher)
+      @cache = Cache.new(@directory)
+      @endpoints = Set.new
+      @info_checksums_by_name = {}
+      @in_parallel = lambda do |inputs, &blk|
+        inputs.map(&blk)
+      end
+    end
+
+    def names
+      Bundler::CompactIndexClient.debug { "/names" }
+      update(@cache.names_path, "names")
+      @cache.names
+    end
+
+    def versions
+      Bundler::CompactIndexClient.debug { "/versions" }
+      update(@cache.versions_path, "versions")
+      versions, @info_checksums_by_name = @cache.versions
+      versions
+    end
+
+    def dependencies(names)
+      Bundler::CompactIndexClient.debug { "dependencies(#{names})" }
+      in_parallel.call(names) do |name|
+        update_info(name)
+        @cache.dependencies(name).map {|d| d.unshift(name) }
+      end.flatten(1)
+    end
+
+    def spec(name, version, platform = nil)
+      Bundler::CompactIndexClient.debug { "spec(name = #{name}, version = #{version}, platform = #{platform})" }
+      update_info(name)
+      @cache.specific_dependency(name, version, platform)
+    end
+
+    def update_and_parse_checksums!
+      Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" }
+      return @info_checksums_by_name if @parsed_checksums
+      update(@cache.versions_path, "versions")
+      @info_checksums_by_name = @cache.checksums
+      @parsed_checksums = true
+    end
+
+  private
+
+    def update(local_path, remote_path)
+      Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" }
+      unless @endpoints.add?(remote_path)
+        Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" }
+        return
+      end
+      @updater.update(local_path, url(remote_path))
+    end
+
+    def update_info(name)
+      Bundler::CompactIndexClient.debug { "update_info(#{name})" }
+      path = @cache.info_path(name)
+      checksum = @updater.checksum_for_file(path)
+      unless existing = @info_checksums_by_name[name]
+        Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" }
+        return
+      end
+      if checksum == existing
+        Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" }
+        return
+      end
+      Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" }
+      update(path, "info/#{name}")
+    end
+
+    def url(path)
+      path
+    end
+  end
+end
diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb
new file mode 100644
index 0000000000..e44f05dc7e
--- /dev/null
+++ b/lib/bundler/compact_index_client/cache.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+require "digest/md5"
+
+module Bundler
+  class CompactIndexClient
+    class Cache
+      attr_reader :directory
+
+      def initialize(directory)
+        @directory = Pathname.new(directory).expand_path
+        info_roots.each do |dir|
+          SharedHelpers.filesystem_access(dir) do
+            FileUtils.mkdir_p(dir)
+          end
+        end
+      end
+
+      def names
+        lines(names_path)
+      end
+
+      def names_path
+        directory.join("names")
+      end
+
+      def versions
+        versions_by_name = Hash.new {|hash, key| hash[key] = [] }
+        info_checksums_by_name = {}
+
+        lines(versions_path).each do |line|
+          name, versions_string, info_checksum = line.split(" ", 3)
+          info_checksums_by_name[name] = info_checksum || ""
+          versions_string.split(",").each do |version|
+            if version.start_with?("-")
+              version = version[1..-1].split("-", 2).unshift(name)
+              versions_by_name[name].delete(version)
+            else
+              version = version.split("-", 2).unshift(name)
+              versions_by_name[name] << version
+            end
+          end
+        end
+
+        [versions_by_name, info_checksums_by_name]
+      end
+
+      def versions_path
+        directory.join("versions")
+      end
+
+      def checksums
+        checksums = {}
+
+        lines(versions_path).each do |line|
+          name, _, checksum = line.split(" ", 3)
+          checksums[name] = checksum
+        end
+
+        checksums
+      end
+
+      def dependencies(name)
+        lines(info_path(name)).map do |line|
+          parse_gem(line)
+        end
+      end
+
+      def info_path(name)
+        name = name.to_s
+        if name =~ /[^a-z0-9_-]/
+          name += "-#{Digest::MD5.hexdigest(name).downcase}"
+          info_roots.last.join(name)
+        else
+          info_roots.first.join(name)
+        end
+      end
+
+      def specific_dependency(name, version, platform)
+        pattern = [version, platform].compact.join("-")
+        return nil if pattern.empty?
+
+        gem_lines = info_path(name).read
+        gem_line = gem_lines[/^#{Regexp.escape(pattern)}\b.*/, 0]
+        gem_line ? parse_gem(gem_line) : nil
+      end
+
+    private
+
+      def lines(path)
+        return [] unless path.file?
+        lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n")
+        header = lines.index("---")
+        header ? lines[header + 1..-1] : lines
+      end
+
+      def parse_gem(string)
+        version_and_platform, rest = string.split(" ", 2)
+        version, platform = version_and_platform.split("-", 2)
+        dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
+        dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
+        requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
+        [version, platform, dependencies, requirements]
+      end
+
+      def parse_dependency(string)
+        dependency = string.split(":")
+        dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+        dependency
+      end
+
+      def info_roots
+        [
+          directory.join("info"),
+          directory.join("info-special-characters"),
+        ]
+      end
+    end
+  end
+end
diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb
new file mode 100644
index 0000000000..03a063beb5
--- /dev/null
+++ b/lib/bundler/compact_index_client/updater.rb
@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+require "fileutils"
+require "stringio"
+require "tmpdir"
+require "zlib"
+
+module Bundler
+  class CompactIndexClient
+    class Updater
+      class MisMatchedChecksumError < Error
+        def initialize(path, server_checksum, local_checksum)
+          @path = path
+          @server_checksum = server_checksum
+          @local_checksum = local_checksum
+        end
+
+        def message
+          "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \
+            "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})."
+        end
+      end
+
+      def initialize(fetcher)
+        @fetcher = fetcher
+      end
+
+      def update(local_path, remote_path, retrying = nil)
+        headers = {}
+
+        Dir.mktmpdir(local_path.basename.to_s, local_path.dirname) do |local_temp_dir|
+          local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename)
+
+          # download new file if retrying
+          if retrying.nil? && local_path.file?
+            FileUtils.cp local_path, local_temp_path
+            headers["If-None-Match"] = etag_for(local_temp_path)
+            headers["Range"] = "bytes=#{local_temp_path.size}-"
+          else
+            # Fastly ignores Range when Accept-Encoding: gzip is set
+            headers["Accept-Encoding"] = "gzip"
+          end
+
+          response = @fetcher.call(remote_path, headers)
+          return nil if response.is_a?(Net::HTTPNotModified)
+
+          content = response.body
+          if response["Content-Encoding"] == "gzip"
+            content = Zlib::GzipReader.new(StringIO.new(content)).read
+          end
+
+          mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w"
+          SharedHelpers.filesystem_access(local_temp_path) do
+            local_temp_path.open(mode) {|f| f << content }
+          end
+
+          response_etag = response["ETag"].gsub(%r{\AW/}, "")
+          if etag_for(local_temp_path) == response_etag
+            SharedHelpers.filesystem_access(local_path) do
+              FileUtils.mv(local_temp_path, local_path)
+            end
+            return nil
+          end
+
+          unless retrying.nil?
+            raise MisMatchedChecksumError.new(remote_path, response_etag, etag_for(local_temp_path))
+          end
+
+          update(local_path, remote_path, :retrying)
+        end
+      end
+
+      def etag_for(path)
+        sum = checksum_for_file(path)
+        sum ? %("#{sum}") : nil
+      end
+
+      def checksum_for_file(path)
+        return nil unless path.file?
+        # This must use IO.read instead of Digest.file().hexdigest
+        # because we need to preserve \n line endings on windows when calculating
+        # the checksum
+        SharedHelpers.filesystem_access(path, :read) do
+          Digest::MD5.hexdigest(IO.read(path))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb
index 63b5c8371d..76f7bc3094 100644
--- a/lib/bundler/fetcher/compact_index.rb
+++ b/lib/bundler/fetcher/compact_index.rb
@@ -5,7 +5,7 @@ require "bundler/worker"
 module Bundler
   class Fetcher
     class CompactIndex < Base
-      require "bundler/vendor/compact_index_client/lib/compact_index_client"
+      require "bundler/compact_index_client"
 
       def self.compact_index_request(method_name)
         method = instance_method(method_name)
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb
deleted file mode 100644
index c063c6b4dc..0000000000
--- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb
+++ /dev/null
@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-require "pathname"
-require "set"
-
-class Bundler::CompactIndexClient
-  DEBUG_MUTEX = Mutex.new
-  def self.debug
-    return unless ENV["DEBUG_COMPACT_INDEX"]
-    DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") }
-  end
-
-  class Error < StandardError; end
-
-  require "bundler/vendor/compact_index_client/lib/compact_index_client/cache"
-  require "bundler/vendor/compact_index_client/lib/compact_index_client/updater"
-  require "bundler/vendor/compact_index_client/lib/compact_index_client/version"
-
-  attr_reader :directory
-
-  # @return [Lambda] A lambda that takes an array of inputs and a block, and
-  #         maps the inputs with the block in parallel.
-  #
-  attr_accessor :in_parallel
-
-  def initialize(directory, fetcher)
-    @directory = Pathname.new(directory)
-    @updater = Updater.new(fetcher)
-    @cache = Cache.new(@directory)
-    @endpoints = Set.new
-    @info_checksums_by_name = {}
-    @in_parallel = lambda do |inputs, &blk|
-      inputs.map(&blk)
-    end
-  end
-
-  def names
-    Bundler::CompactIndexClient.debug { "/names" }
-    update(@cache.names_path, "names")
-    @cache.names
-  end
-
-  def versions
-    Bundler::CompactIndexClient.debug { "/versions" }
-    update(@cache.versions_path, "versions")
-    versions, @info_checksums_by_name = @cache.versions
-    versions
-  end
-
-  def dependencies(names)
-    Bundler::CompactIndexClient.debug { "dependencies(#{names})" }
-    in_parallel.call(names) do |name|
-      update_info(name)
-      @cache.dependencies(name).map {|d| d.unshift(name) }
-    end.flatten(1)
-  end
-
-  def spec(name, version, platform = nil)
-    Bundler::CompactIndexClient.debug { "spec(name = #{name}, version = #{version}, platform = #{platform})" }
-    update_info(name)
-    @cache.specific_dependency(name, version, platform)
-  end
-
-  def update_and_parse_checksums!
-    Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" }
-    return @info_checksums_by_name if @parsed_checksums
-    update(@cache.versions_path, "versions")
-    @info_checksums_by_name = @cache.checksums
-    @parsed_checksums = true
-  end
-
-private
-
-  def update(local_path, remote_path)
-    Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" }
-    unless @endpoints.add?(remote_path)
-      Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" }
-      return
-    end
-    @updater.update(local_path, url(remote_path))
-  end
-
-  def update_info(name)
-    Bundler::CompactIndexClient.debug { "update_info(#{name})" }
-    path = @cache.info_path(name)
-    checksum = @updater.checksum_for_file(path)
-    unless existing = @info_checksums_by_name[name]
-      Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" }
-      return
-    end
-    if checksum == existing
-      Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" }
-      return
-    end
-    Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" }
-    update(path, "info/#{name}")
-  end
-
-  def url(path)
-    path
-  end
-end
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb
deleted file mode 100644
index d2639ee717..0000000000
--- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb
+++ /dev/null
@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-require "digest/md5"
-class Bundler::CompactIndexClient
-  class Cache
-    attr_reader :directory
-
-    def initialize(directory)
-      @directory = Pathname.new(directory).expand_path
-      info_roots.each {|dir| FileUtils.mkdir_p(dir) }
-    end
-
-    def names
-      lines(names_path)
-    end
-
-    def names_path
-      directory.join("names")
-    end
-
-    def versions
-      versions_by_name = Hash.new {|hash, key| hash[key] = [] }
-      info_checksums_by_name = {}
-
-      lines(versions_path).each do |line|
-        name, versions_string, info_checksum = line.split(" ", 3)
-        info_checksums_by_name[name] = info_checksum || ""
-        versions_string.split(",").each do |version|
-          if version.start_with?("-")
-            version = version[1..-1].split("-", 2).unshift(name)
-            versions_by_name[name].delete(version)
-          else
-            version = version.split("-", 2).unshift(name)
-            versions_by_name[name] << version
-          end
-        end
-      end
-
-      [versions_by_name, info_checksums_by_name]
-    end
-
-    def versions_path
-      directory.join("versions")
-    end
-
-    def checksums
-      checksums = {}
-
-      lines(versions_path).each do |line|
-        name, _, checksum = line.split(" ", 3)
-        checksums[name] = checksum
-      end
-
-      checksums
-    end
-
-    def dependencies(name)
-      lines(info_path(name)).map do |line|
-        parse_gem(line)
-      end
-    end
-
-    def info_path(name)
-      name = name.to_s
-      if name =~ /[^a-z0-9_-]/
-        name += "-#{Digest::MD5.hexdigest(name).downcase}"
-        info_roots.last.join(name)
-      else
-        info_roots.first.join(name)
-      end
-    end
-
-    def specific_dependency(name, version, platform)
-      pattern = [version, platform].compact.join("-")
-      return nil if pattern.empty?
-
-      gem_lines = info_path(name).read
-      gem_line = gem_lines[/^#{Regexp.escape(pattern)}\b.*/, 0]
-      gem_line ? parse_gem(gem_line) : nil
-    end
-
-  private
-
-    def lines(path)
-      return [] unless path.file?
-      lines = path.read.split("\n")
-      header = lines.index("---")
-      lines = header ? lines[header + 1..-1] : lines
-    end
-
-    def parse_gem(string)
-      version_and_platform, rest = string.split(" ", 2)
-      version, platform = version_and_platform.split("-", 2)
-      dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
-      dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
-      requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
-      [version, platform, dependencies, requirements]
-    end
-
-    def parse_dependency(string)
-      dependency = string.split(":")
-      dependency[-1] = dependency[-1].split("&") if dependency.size > 1
-      dependency
-    end
-
-    def info_roots
-      [
-        directory.join("info"),
-        directory.join("info-special-characters"),
-      ]
-    end
-  end
-end
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb
deleted file mode 100644
index a410dd423c..0000000000
--- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb
+++ /dev/null
@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-require "fileutils"
-require "stringio"
-require "tmpdir"
-require "zlib"
-
-class Bundler::CompactIndexClient
-  class Updater
-    class MisMatchedChecksumError < Error
-      def initialize(path, server_checksum, local_checksum)
-        @path = path
-        @server_checksum = server_checksum
-        @local_checksum = local_checksum
-      end
-
-      def message
-        "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \
-          "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})."
-      end
-    end
-
-    def initialize(fetcher)
-      @fetcher = fetcher
-    end
-
-    def update(local_path, remote_path, retrying = nil)
-      headers = {}
-
-      Dir.mktmpdir(local_path.basename.to_s, local_path.dirname) do |local_temp_dir|
-        local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename)
-
-        # download new file if retrying
-        if retrying.nil? && local_path.file?
-          FileUtils.cp local_path, local_temp_path
-          headers["If-None-Match"] = etag_for(local_temp_path)
-          headers["Range"] = "bytes=#{local_temp_path.size}-"
-        else
-          # Fastly ignores Range when Accept-Encoding: gzip is set
-          headers["Accept-Encoding"] = "gzip"
-        end
-
-        response = @fetcher.call(remote_path, headers)
-        return if response.is_a?(Net::HTTPNotModified)
-
-        content = response.body
-        if response["Content-Encoding"] == "gzip"
-          content = Zlib::GzipReader.new(StringIO.new(content)).read
-        end
-
-        mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w"
-        local_temp_path.open(mode) {|f| f << content }
-
-        response_etag = response["ETag"].gsub(%r{\AW/}, "")
-        if etag_for(local_temp_path) == response_etag
-          FileUtils.mv(local_temp_path, local_path)
-          return
-        end
-
-        if retrying.nil?
-          update(local_path, remote_path, :retrying)
-        else
-          raise MisMatchedChecksumError.new(remote_path, response_etag, etag_for(local_temp_path))
-        end
-      end
-    end
-
-    def etag_for(path)
-      sum = checksum_for_file(path)
-      sum ? %("#{sum}") : nil
-    end
-
-    def checksum_for_file(path)
-      return nil unless path.file?
-      # This must use IO.read instead of Digest.file().hexdigest
-      # because we need to preserve \n line endings on windows when calculating
-      # the checksum
-      Digest::MD5.hexdigest(IO.read(path))
-    end
-  end
-end
diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb
deleted file mode 100644
index 64520daead..0000000000
--- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb
+++ /dev/null
@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-class Bundler::CompactIndexClient
-  VERSION = "0.1.0".freeze
-end