diff options
author | Homu <homu@barosl.com> | 2016-09-30 20:13:15 +0900 |
---|---|---|
committer | Homu <homu@barosl.com> | 2016-09-30 20:13:15 +0900 |
commit | fd4ba9fed87cfd94a89057034bb69cecec27d150 (patch) | |
tree | afad8a51ee43c97d8da4c363b0fd6d3d1b4f7300 | |
parent | 5ee76130518832b12bb3dae22eae6cfd408309e3 (diff) | |
parent | 5f5e350017a1c7feb7315e45fd9a8298dee69efc (diff) | |
download | bundler-fd4ba9fed87cfd94a89057034bb69cecec27d150.tar.gz |
Auto merge of #5007 - bundler:seg-unvendor-compact-index-client, r=indirect
Un-vendor the compact index client
Step one to https://github.com/bundler/bundler/issues/4935
-rw-r--r-- | Rakefile | 7 | ||||
-rw-r--r-- | lib/bundler/compact_index_client.rb | 102 | ||||
-rw-r--r-- | lib/bundler/compact_index_client/cache.rb | 119 | ||||
-rw-r--r-- | lib/bundler/compact_index_client/updater.rb | 88 | ||||
-rw-r--r-- | lib/bundler/fetcher/compact_index.rb | 2 | ||||
-rw-r--r-- | lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb | 101 | ||||
-rw-r--r-- | lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb | 112 | ||||
-rw-r--r-- | lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb | 80 | ||||
-rw-r--r-- | lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb | 4 |
9 files changed, 310 insertions, 305 deletions
@@ -285,13 +285,6 @@ end begin require "automatiek" - Automatiek::RakeTask.new("compact_index_client") do |lib| - lib.download = { :github => "https://github.com/bundler/compact_index_client" } - lib.namespace = "CompactIndexClient" - lib.prefix = "Bundler" - lib.vendor_lib = "lib/bundler/vendor/compact_index_client" - end - Automatiek::RakeTask.new("molinillo") do |lib| lib.download = { :github => "https://github.com/CocoaPods/Molinillo" } lib.namespace = "Molinillo" diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb new file mode 100644 index 0000000000..1ff3deda01 --- /dev/null +++ b/lib/bundler/compact_index_client.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true +require "pathname" +require "set" + +module Bundler + class CompactIndexClient + DEBUG_MUTEX = Mutex.new + def self.debug + return unless ENV["DEBUG_COMPACT_INDEX"] + DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } + end + + class Error < StandardError; end + + require "bundler/compact_index_client/cache" + require "bundler/compact_index_client/updater" + + attr_reader :directory + + # @return [Lambda] A lambda that takes an array of inputs and a block, and + # maps the inputs with the block in parallel. + # + attr_accessor :in_parallel + + def initialize(directory, fetcher) + @directory = Pathname.new(directory) + @updater = Updater.new(fetcher) + @cache = Cache.new(@directory) + @endpoints = Set.new + @info_checksums_by_name = {} + @in_parallel = lambda do |inputs, &blk| + inputs.map(&blk) + end + end + + def names + Bundler::CompactIndexClient.debug { "/names" } + update(@cache.names_path, "names") + @cache.names + end + + def versions + Bundler::CompactIndexClient.debug { "/versions" } + update(@cache.versions_path, "versions") + versions, @info_checksums_by_name = @cache.versions + versions + end + + def dependencies(names) + Bundler::CompactIndexClient.debug { "dependencies(#{names})" } + in_parallel.call(names) do |name| + update_info(name) + @cache.dependencies(name).map {|d| d.unshift(name) } + end.flatten(1) + end + + def spec(name, version, platform = nil) + Bundler::CompactIndexClient.debug { "spec(name = #{name}, version = #{version}, platform = #{platform})" } + update_info(name) + @cache.specific_dependency(name, version, platform) + end + + def update_and_parse_checksums! + Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } + return @info_checksums_by_name if @parsed_checksums + update(@cache.versions_path, "versions") + @info_checksums_by_name = @cache.checksums + @parsed_checksums = true + end + + private + + def update(local_path, remote_path) + Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } + unless @endpoints.add?(remote_path) + Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } + return + end + @updater.update(local_path, url(remote_path)) + end + + def update_info(name) + Bundler::CompactIndexClient.debug { "update_info(#{name})" } + path = @cache.info_path(name) + checksum = @updater.checksum_for_file(path) + unless existing = @info_checksums_by_name[name] + Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } + return + end + if checksum == existing + Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } + return + end + Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } + update(path, "info/#{name}") + end + + def url(path) + path + end + end +end diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb new file mode 100644 index 0000000000..e44f05dc7e --- /dev/null +++ b/lib/bundler/compact_index_client/cache.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true +require "digest/md5" + +module Bundler + class CompactIndexClient + class Cache + attr_reader :directory + + def initialize(directory) + @directory = Pathname.new(directory).expand_path + info_roots.each do |dir| + SharedHelpers.filesystem_access(dir) do + FileUtils.mkdir_p(dir) + end + end + end + + def names + lines(names_path) + end + + def names_path + directory.join("names") + end + + def versions + versions_by_name = Hash.new {|hash, key| hash[key] = [] } + info_checksums_by_name = {} + + lines(versions_path).each do |line| + name, versions_string, info_checksum = line.split(" ", 3) + info_checksums_by_name[name] = info_checksum || "" + versions_string.split(",").each do |version| + if version.start_with?("-") + version = version[1..-1].split("-", 2).unshift(name) + versions_by_name[name].delete(version) + else + version = version.split("-", 2).unshift(name) + versions_by_name[name] << version + end + end + end + + [versions_by_name, info_checksums_by_name] + end + + def versions_path + directory.join("versions") + end + + def checksums + checksums = {} + + lines(versions_path).each do |line| + name, _, checksum = line.split(" ", 3) + checksums[name] = checksum + end + + checksums + end + + def dependencies(name) + lines(info_path(name)).map do |line| + parse_gem(line) + end + end + + def info_path(name) + name = name.to_s + if name =~ /[^a-z0-9_-]/ + name += "-#{Digest::MD5.hexdigest(name).downcase}" + info_roots.last.join(name) + else + info_roots.first.join(name) + end + end + + def specific_dependency(name, version, platform) + pattern = [version, platform].compact.join("-") + return nil if pattern.empty? + + gem_lines = info_path(name).read + gem_line = gem_lines[/^#{Regexp.escape(pattern)}\b.*/, 0] + gem_line ? parse_gem(gem_line) : nil + end + + private + + def lines(path) + return [] unless path.file? + lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n") + header = lines.index("---") + header ? lines[header + 1..-1] : lines + end + + def parse_gem(string) + version_and_platform, rest = string.split(" ", 2) + version, platform = version_and_platform.split("-", 2) + dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest + dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : [] + requirements = requirements ? requirements.map {|r| parse_dependency(r) } : [] + [version, platform, dependencies, requirements] + end + + def parse_dependency(string) + dependency = string.split(":") + dependency[-1] = dependency[-1].split("&") if dependency.size > 1 + dependency + end + + def info_roots + [ + directory.join("info"), + directory.join("info-special-characters"), + ] + end + end + end +end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb new file mode 100644 index 0000000000..03a063beb5 --- /dev/null +++ b/lib/bundler/compact_index_client/updater.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true +require "fileutils" +require "stringio" +require "tmpdir" +require "zlib" + +module Bundler + class CompactIndexClient + class Updater + class MisMatchedChecksumError < Error + def initialize(path, server_checksum, local_checksum) + @path = path + @server_checksum = server_checksum + @local_checksum = local_checksum + end + + def message + "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \ + "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})." + end + end + + def initialize(fetcher) + @fetcher = fetcher + end + + def update(local_path, remote_path, retrying = nil) + headers = {} + + Dir.mktmpdir(local_path.basename.to_s, local_path.dirname) do |local_temp_dir| + local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename) + + # download new file if retrying + if retrying.nil? && local_path.file? + FileUtils.cp local_path, local_temp_path + headers["If-None-Match"] = etag_for(local_temp_path) + headers["Range"] = "bytes=#{local_temp_path.size}-" + else + # Fastly ignores Range when Accept-Encoding: gzip is set + headers["Accept-Encoding"] = "gzip" + end + + response = @fetcher.call(remote_path, headers) + return nil if response.is_a?(Net::HTTPNotModified) + + content = response.body + if response["Content-Encoding"] == "gzip" + content = Zlib::GzipReader.new(StringIO.new(content)).read + end + + mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w" + SharedHelpers.filesystem_access(local_temp_path) do + local_temp_path.open(mode) {|f| f << content } + end + + response_etag = response["ETag"].gsub(%r{\AW/}, "") + if etag_for(local_temp_path) == response_etag + SharedHelpers.filesystem_access(local_path) do + FileUtils.mv(local_temp_path, local_path) + end + return nil + end + + unless retrying.nil? + raise MisMatchedChecksumError.new(remote_path, response_etag, etag_for(local_temp_path)) + end + + update(local_path, remote_path, :retrying) + end + end + + def etag_for(path) + sum = checksum_for_file(path) + sum ? %("#{sum}") : nil + end + + def checksum_for_file(path) + return nil unless path.file? + # This must use IO.read instead of Digest.file().hexdigest + # because we need to preserve \n line endings on windows when calculating + # the checksum + SharedHelpers.filesystem_access(path, :read) do + Digest::MD5.hexdigest(IO.read(path)) + end + end + end + end +end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index 63b5c8371d..76f7bc3094 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -5,7 +5,7 @@ require "bundler/worker" module Bundler class Fetcher class CompactIndex < Base - require "bundler/vendor/compact_index_client/lib/compact_index_client" + require "bundler/compact_index_client" def self.compact_index_request(method_name) method = instance_method(method_name) diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb deleted file mode 100644 index c063c6b4dc..0000000000 --- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true -require "pathname" -require "set" - -class Bundler::CompactIndexClient - DEBUG_MUTEX = Mutex.new - def self.debug - return unless ENV["DEBUG_COMPACT_INDEX"] - DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } - end - - class Error < StandardError; end - - require "bundler/vendor/compact_index_client/lib/compact_index_client/cache" - require "bundler/vendor/compact_index_client/lib/compact_index_client/updater" - require "bundler/vendor/compact_index_client/lib/compact_index_client/version" - - attr_reader :directory - - # @return [Lambda] A lambda that takes an array of inputs and a block, and - # maps the inputs with the block in parallel. - # - attr_accessor :in_parallel - - def initialize(directory, fetcher) - @directory = Pathname.new(directory) - @updater = Updater.new(fetcher) - @cache = Cache.new(@directory) - @endpoints = Set.new - @info_checksums_by_name = {} - @in_parallel = lambda do |inputs, &blk| - inputs.map(&blk) - end - end - - def names - Bundler::CompactIndexClient.debug { "/names" } - update(@cache.names_path, "names") - @cache.names - end - - def versions - Bundler::CompactIndexClient.debug { "/versions" } - update(@cache.versions_path, "versions") - versions, @info_checksums_by_name = @cache.versions - versions - end - - def dependencies(names) - Bundler::CompactIndexClient.debug { "dependencies(#{names})" } - in_parallel.call(names) do |name| - update_info(name) - @cache.dependencies(name).map {|d| d.unshift(name) } - end.flatten(1) - end - - def spec(name, version, platform = nil) - Bundler::CompactIndexClient.debug { "spec(name = #{name}, version = #{version}, platform = #{platform})" } - update_info(name) - @cache.specific_dependency(name, version, platform) - end - - def update_and_parse_checksums! - Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } - return @info_checksums_by_name if @parsed_checksums - update(@cache.versions_path, "versions") - @info_checksums_by_name = @cache.checksums - @parsed_checksums = true - end - -private - - def update(local_path, remote_path) - Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } - unless @endpoints.add?(remote_path) - Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } - return - end - @updater.update(local_path, url(remote_path)) - end - - def update_info(name) - Bundler::CompactIndexClient.debug { "update_info(#{name})" } - path = @cache.info_path(name) - checksum = @updater.checksum_for_file(path) - unless existing = @info_checksums_by_name[name] - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } - return - end - if checksum == existing - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } - return - end - Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } - update(path, "info/#{name}") - end - - def url(path) - path - end -end diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb deleted file mode 100644 index d2639ee717..0000000000 --- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/cache.rb +++ /dev/null @@ -1,112 +0,0 @@ -# frozen_string_literal: true -require "digest/md5" -class Bundler::CompactIndexClient - class Cache - attr_reader :directory - - def initialize(directory) - @directory = Pathname.new(directory).expand_path - info_roots.each {|dir| FileUtils.mkdir_p(dir) } - end - - def names - lines(names_path) - end - - def names_path - directory.join("names") - end - - def versions - versions_by_name = Hash.new {|hash, key| hash[key] = [] } - info_checksums_by_name = {} - - lines(versions_path).each do |line| - name, versions_string, info_checksum = line.split(" ", 3) - info_checksums_by_name[name] = info_checksum || "" - versions_string.split(",").each do |version| - if version.start_with?("-") - version = version[1..-1].split("-", 2).unshift(name) - versions_by_name[name].delete(version) - else - version = version.split("-", 2).unshift(name) - versions_by_name[name] << version - end - end - end - - [versions_by_name, info_checksums_by_name] - end - - def versions_path - directory.join("versions") - end - - def checksums - checksums = {} - - lines(versions_path).each do |line| - name, _, checksum = line.split(" ", 3) - checksums[name] = checksum - end - - checksums - end - - def dependencies(name) - lines(info_path(name)).map do |line| - parse_gem(line) - end - end - - def info_path(name) - name = name.to_s - if name =~ /[^a-z0-9_-]/ - name += "-#{Digest::MD5.hexdigest(name).downcase}" - info_roots.last.join(name) - else - info_roots.first.join(name) - end - end - - def specific_dependency(name, version, platform) - pattern = [version, platform].compact.join("-") - return nil if pattern.empty? - - gem_lines = info_path(name).read - gem_line = gem_lines[/^#{Regexp.escape(pattern)}\b.*/, 0] - gem_line ? parse_gem(gem_line) : nil - end - - private - - def lines(path) - return [] unless path.file? - lines = path.read.split("\n") - header = lines.index("---") - lines = header ? lines[header + 1..-1] : lines - end - - def parse_gem(string) - version_and_platform, rest = string.split(" ", 2) - version, platform = version_and_platform.split("-", 2) - dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest - dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : [] - requirements = requirements ? requirements.map {|r| parse_dependency(r) } : [] - [version, platform, dependencies, requirements] - end - - def parse_dependency(string) - dependency = string.split(":") - dependency[-1] = dependency[-1].split("&") if dependency.size > 1 - dependency - end - - def info_roots - [ - directory.join("info"), - directory.join("info-special-characters"), - ] - end - end -end diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb deleted file mode 100644 index a410dd423c..0000000000 --- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/updater.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true -require "fileutils" -require "stringio" -require "tmpdir" -require "zlib" - -class Bundler::CompactIndexClient - class Updater - class MisMatchedChecksumError < Error - def initialize(path, server_checksum, local_checksum) - @path = path - @server_checksum = server_checksum - @local_checksum = local_checksum - end - - def message - "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \ - "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})." - end - end - - def initialize(fetcher) - @fetcher = fetcher - end - - def update(local_path, remote_path, retrying = nil) - headers = {} - - Dir.mktmpdir(local_path.basename.to_s, local_path.dirname) do |local_temp_dir| - local_temp_path = Pathname.new(local_temp_dir).join(local_path.basename) - - # download new file if retrying - if retrying.nil? && local_path.file? - FileUtils.cp local_path, local_temp_path - headers["If-None-Match"] = etag_for(local_temp_path) - headers["Range"] = "bytes=#{local_temp_path.size}-" - else - # Fastly ignores Range when Accept-Encoding: gzip is set - headers["Accept-Encoding"] = "gzip" - end - - response = @fetcher.call(remote_path, headers) - return if response.is_a?(Net::HTTPNotModified) - - content = response.body - if response["Content-Encoding"] == "gzip" - content = Zlib::GzipReader.new(StringIO.new(content)).read - end - - mode = response.is_a?(Net::HTTPPartialContent) ? "a" : "w" - local_temp_path.open(mode) {|f| f << content } - - response_etag = response["ETag"].gsub(%r{\AW/}, "") - if etag_for(local_temp_path) == response_etag - FileUtils.mv(local_temp_path, local_path) - return - end - - if retrying.nil? - update(local_path, remote_path, :retrying) - else - raise MisMatchedChecksumError.new(remote_path, response_etag, etag_for(local_temp_path)) - end - end - end - - def etag_for(path) - sum = checksum_for_file(path) - sum ? %("#{sum}") : nil - end - - def checksum_for_file(path) - return nil unless path.file? - # This must use IO.read instead of Digest.file().hexdigest - # because we need to preserve \n line endings on windows when calculating - # the checksum - Digest::MD5.hexdigest(IO.read(path)) - end - end -end diff --git a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb b/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb deleted file mode 100644 index 64520daead..0000000000 --- a/lib/bundler/vendor/compact_index_client/lib/compact_index_client/version.rb +++ /dev/null @@ -1,4 +0,0 @@ -# frozen_string_literal: true -class Bundler::CompactIndexClient - VERSION = "0.1.0".freeze -end |