From 9c7042fbed34141bffa0ddf8cdd4da1a9222a684 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 5 Oct 2022 17:35:09 +0900 Subject: Removed the related files for downloading with extlibs --- ext/psych/extconf.rb | 17 --- tool/downloader.rb | 415 --------------------------------------------------- tool/extlibs.rb | 285 ----------------------------------- 3 files changed, 717 deletions(-) delete mode 100644 tool/downloader.rb delete mode 100755 tool/extlibs.rb diff --git a/ext/psych/extconf.rb b/ext/psych/extconf.rb index 6d03870..16c76e4 100644 --- a/ext/psych/extconf.rb +++ b/ext/psych/extconf.rb @@ -18,23 +18,6 @@ end if yaml_source == true # search the latest libyaml source under $srcdir yaml_source = Dir.glob("#{$srcdir}/yaml{,-*}/").max_by {|n| File.basename(n).scan(/\d+/).map(&:to_i)} - unless yaml_source - download_failure = "failed to download libyaml source. Try manually installing libyaml?" - begin - require_relative '../../tool/extlibs.rb' - rescue LoadError - # When running in ruby/ruby, we use miniruby and don't have stdlib. - # Avoid LoadError because it aborts the whole build. Usually when - # stdlib extension fail to configure we skip it and continue. - raise download_failure - end - extlibs = ExtLibs.new(cache_dir: File.expand_path("../../tmp/download_cache", $srcdir)) - unless extlibs.process_under($srcdir) - raise download_failure - end - yaml_source, = Dir.glob("#{$srcdir}/yaml-*/") - raise "libyaml not found" unless yaml_source - end elsif yaml_source yaml_source = yaml_source.gsub(/\$\((\w+)\)|\$\{(\w+)\}/) {ENV[$1||$2]} end diff --git a/tool/downloader.rb b/tool/downloader.rb deleted file mode 100644 index d3a9f75..0000000 --- a/tool/downloader.rb +++ /dev/null @@ -1,415 +0,0 @@ -# Used by configure and make to download or update mirrored Ruby and GCC -# files. This will use HTTPS if possible, falling back to HTTP. - -require 'fileutils' -require 'open-uri' -require 'pathname' -begin - require 'net/https' -rescue LoadError - https = 'http' -else - https = 'https' - - # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old - # versions do not. so, patching OpenSSL::X509::Store#add_file instead. - class OpenSSL::X509::Store - alias orig_add_file add_file - def add_file(pems) - Array(pems).each do |pem| - if File.directory?(pem) - add_path pem - else - orig_add_file pem - end - end - end - end - # since open-uri internally checks ssl_ca_cert using File.directory?, - # allow to accept an array. - class < e - STDERR.puts "Download failed (#{e.message}), try another URL" - super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest) - end - else - super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest) - end - end - end - - class RubyGems < self - def self.download(name, dir = nil, since = true, options = {}) - require 'rubygems' - options = options.dup - options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) - super("https://rubygems.org/downloads/#{name}", name, dir, since, options) - end - end - - Gems = RubyGems - - class Unicode < self - INDEX = {} # cache index file information across files in the same directory - UNICODE_PUBLIC = "https://www.unicode.org/Public/" - - def self.download(name, dir = nil, since = true, options = {}) - options = options.dup - unicode_beta = options.delete(:unicode_beta) - name_dir_part = name.sub(/[^\/]+$/, '') - if unicode_beta == 'YES' - if INDEX.size == 0 - index_options = options.dup - index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file - index_data = File.read(under(dir, "index.html")) rescue nil - index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options) - INDEX[:index] = File.read(index_file) - since = true unless INDEX[:index] == index_data - end - file_base = File.basename(name, '.txt') - return if file_base == '.' # Use pre-generated headers and tables - beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/] - # make sure we always check for new versions of files, - # because they can easily change in the beta period - super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options) - else - index_file = Pathname.new(under(dir, name_dir_part+'index.html')) - if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/ - raise "Although Unicode is not in beta, file #{index_file} exists. " + - "Remove all files in this directory and in .downloaded-cache/ " + - "because they may be leftovers from the beta period." - end - super(UNICODE_PUBLIC+name, name, dir, since, options) - end - end - end - - def self.mode_for(data) - /\A#!/ =~ data ? 0755 : 0644 - end - - def self.http_options(file, since) - options = {} - if since - case since - when true - since = (File.mtime(file).httpdate rescue nil) - when Time - since = since.httpdate - end - if since - options['If-Modified-Since'] = since - end - end - options['Accept-Encoding'] = 'identity' # to disable Net::HTTP::GenericRequest#decode_content - options - end - - def self.httpdate(date) - Time.httpdate(date) - rescue ArgumentError => e - # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4 - # digit year, sometimes. - /\A\s* - (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20 - (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20 - (\d\d):(\d\d):(\d\d)\x20 - GMT - \s*\z/ix =~ date or raise - warn e.message - Time.utc($3, $2, $1, $4, $5, $6) - end - - # Downloader.download(url, name, [dir, [since]]) - # - # Update a file from url if newer version is available. - # Creates the file if the file doesn't yet exist; however, the - # directory where the file is being created has to exist already. - # The +since+ parameter can take the following values, with associated meanings: - # true :: - # Take the last-modified time of the current file on disk, and only download - # if the server has a file that was modified later. Download unconditionally - # if we don't have the file yet. Default. - # +some time value+ :: - # Use this time value instead of the time of modification of the file on disk. - # nil :: - # Only download the file if it doesn't exist yet. - # false :: - # always download url regardless of whether we already have a file, - # and regardless of modification times. (This is essentially just a waste of - # network resources, except in the case that the file we have is somehow damaged. - # Please note that using this recurringly might create or be seen as a - # denial of service attack.) - # - # Example usage: - # download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt', - # 'UnicodeData.txt', 'enc/unicode/data' - def self.download(url, name, dir = nil, since = true, options = {}) - options = options.dup - url = URI(url) - dryrun = options.delete(:dryrun) - options.delete(:unicode_beta) # just to be on the safe side for gems and gcc - - if name - file = Pathname.new(under(dir, name)) - else - name = File.basename(url.path) - end - cache_save = options.delete(:cache_save) { - ENV["CACHE_SAVE"] != "no" - } - cache = cache_file(url, name, options.delete(:cache_dir)) - file ||= cache - if since.nil? and file.exist? - if $VERBOSE - $stdout.puts "#{file} already exists" - $stdout.flush - end - if cache_save - save_cache(cache, file, name) - end - return file.to_path - end - if dryrun - puts "Download #{url} into #{file}" - return - end - if link_cache(cache, file, name, $VERBOSE) - return file.to_path - end - if !https? and URI::HTTPS === url - warn "*** using http instead of https ***" - url.scheme = 'http' - url = URI(url.to_s) - end - if $VERBOSE - $stdout.print "downloading #{name} ... " - $stdout.flush - end - mtime = nil - options = options.merge(http_options(file, since.nil? ? true : since)) - begin - data = with_retry(10) do - data = url.read(options) - if mtime = data.meta["last-modified"] - mtime = Time.httpdate(mtime) - end - data - end - rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified - if $VERBOSE - $stdout.puts "#{name} not modified" - $stdout.flush - end - return file.to_path - end - raise - rescue Timeout::Error - if since.nil? and file.exist? - puts "Request for #{url} timed out, using old version." - return file.to_path - end - raise - rescue SocketError - if since.nil? and file.exist? - puts "No network connection, unable to download #{url}, using old version." - return file.to_path - end - raise - end - dest = (cache_save && cache && !cache.exist? ? cache : file) - dest.parent.mkpath - dest.open("wb", 0600) do |f| - f.write(data) - f.chmod(mode_for(data)) - end - if mtime - dest.utime(mtime, mtime) - end - if $VERBOSE - $stdout.puts "done" - $stdout.flush - end - if dest.eql?(cache) - link_cache(cache, file, name) - elsif cache_save - save_cache(cache, file, name) - end - return file.to_path - rescue => e - raise "failed to download #{name}\n#{e.class}: #{e.message}: #{url}" - end - - def self.under(dir, name) - dir ? File.join(dir, File.basename(name)) : name - end - - def self.cache_file(url, name, cache_dir = nil) - case cache_dir - when false - return nil - when nil - cache_dir = ENV['CACHE_DIR'] - if !cache_dir or cache_dir.empty? - cache_dir = ".downloaded-cache" - end - end - Pathname.new(cache_dir) + (name || File.basename(URI(url).path)) - end - - def self.link_cache(cache, file, name, verbose = false) - return false unless cache and cache.exist? - return true if cache.eql?(file) - if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN'] - begin - file.make_symlink(cache.relative_path_from(file.parent)) - rescue SystemCallError - else - if verbose - $stdout.puts "made symlink #{name} to #{cache}" - $stdout.flush - end - return true - end - end - begin - file.make_link(cache) - rescue SystemCallError - else - if verbose - $stdout.puts "made link #{name} to #{cache}" - $stdout.flush - end - return true - end - end - - def self.save_cache(cache, file, name) - return unless cache or cache.eql?(file) - begin - st = cache.stat - rescue - begin - file.rename(cache) - rescue - return - end - else - return unless st.mtime > file.lstat.mtime - file.unlink - end - link_cache(cache, file, name) - end - - def self.with_retry(max_times, &block) - times = 0 - begin - block.call - rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e - raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error - times += 1 - if times <= max_times - $stderr.puts "retrying #{e.class} (#{e.message}) after #{times ** 2} seconds..." - sleep(times ** 2) - retry - else - raise - end - end - end - private_class_method :with_retry -end - -Downloader.https = https.freeze - -if $0 == __FILE__ - since = true - options = {} - until ARGV.empty? - case ARGV[0] - when '-d' - destdir = ARGV[1] - ARGV.shift - when '-p' - # strip directory names from the name to download, and add the - # prefix instead. - prefix = ARGV[1] - ARGV.shift - when '-e' - since = nil - when '-a' - since = false - when '-n', '--dryrun' - options[:dryrun] = true - when '--cache-dir' - options[:cache_dir] = ARGV[1] - ARGV.shift - when '--unicode-beta' - options[:unicode_beta] = ARGV[1] - ARGV.shift - when /\A--cache-dir=(.*)/m - options[:cache_dir] = $1 - when /\A-/ - abort "#{$0}: unknown option #{ARGV[0]}" - else - break - end - ARGV.shift - end - dl = Downloader.constants.find do |name| - ARGV[0].casecmp(name.to_s) == 0 - end unless ARGV.empty? - $VERBOSE = true - if dl - dl = Downloader.const_get(dl) - ARGV.shift - ARGV.each do |name| - dir = destdir - if prefix - name = name.sub(/\A\.\//, '') - destdir2 = destdir.sub(/\A\.\//, '') - if name.start_with?(destdir2+"/") - name = name[(destdir2.size+1)..-1] - if (dir = File.dirname(name)) == '.' - dir = destdir - else - dir = File.join(destdir, dir) - end - else - name = File.basename(name) - end - name = "#{prefix}/#{name}" - end - dl.download(name, dir, since, options) - end - else - abort "usage: #{$0} url name" unless ARGV.size == 2 - Downloader.download(ARGV[0], ARGV[1], destdir, since, options) - end -end diff --git a/tool/extlibs.rb b/tool/extlibs.rb deleted file mode 100755 index 5d1a2db..0000000 --- a/tool/extlibs.rb +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/ruby - -# Used to download, extract and patch extension libraries (extlibs) -# for Ruby. See common.mk for Ruby's usage. - -require 'digest' -require_relative 'downloader' -begin - require_relative 'lib/colorize' -rescue LoadError -end - -class ExtLibs - unless defined?(Colorize) - class Colorize - def pass(str) str; end - def fail(str) str; end - end - end - - class Vars < Hash - def pattern - /\$\((#{Regexp.union(keys)})\)/ - end - - def expand(str) - if empty? - str - else - str.gsub(pattern) {self[$1]} - end - end - end - - def initialize(mode = :all, cache_dir: nil) - @mode = mode - @cache_dir = cache_dir - @colorize = Colorize.new - end - - def cache_file(url, cache_dir) - Downloader.cache_file(url, nil, cache_dir).to_path - end - - def do_download(url, cache_dir) - Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir) - end - - def do_checksum(cache, chksums) - chksums.each do |sum| - name, sum = sum.split(/:/) - if $VERBOSE - $stdout.print "checking #{name} of #{cache} ..." - $stdout.flush - end - hd = Digest(name.upcase).file(cache).hexdigest - if $VERBOSE - $stdout.print " " - $stdout.puts hd == sum ? @colorize.pass("OK") : @colorize.fail("NG") - $stdout.flush - end - unless hd == sum - raise "checksum mismatch: #{cache}, #{name}:#{hd}, expected #{sum}" - end - end - end - - def do_extract(cache, dir) - if $VERBOSE - $stdout.puts "extracting #{cache} into #{dir}" - $stdout.flush - end - ext = File.extname(cache) - case ext - when '.gz', '.tgz' - f = IO.popen(["gzip", "-dc", cache]) - cache = cache.chomp('.gz') - when '.bz2', '.tbz' - f = IO.popen(["bzip2", "-dc", cache]) - cache = cache.chomp('.bz2') - when '.xz', '.txz' - f = IO.popen(["xz", "-dc", cache]) - cache = cache.chomp('.xz') - else - inp = cache - end - inp ||= f.binmode - ext = File.extname(cache) - case ext - when '.tar', /\A\.t[gbx]z\z/ - pid = Process.spawn("tar", "xpf", "-", in: inp, chdir: dir) - when '.zip' - pid = Process.spawn("unzip", inp, "-d", dir) - end - f.close if f - Process.wait(pid) - $?.success? or raise "failed to extract #{cache}" - end - - def do_patch(dest, patch, args) - if $VERBOSE - $stdout.puts "applying #{patch} under #{dest}" - $stdout.flush - end - Process.wait(Process.spawn(ENV.fetch("PATCH", "patch"), "-d", dest, "-i", patch, *args)) - $?.success? or raise "failed to patch #{patch}" - end - - def do_link(file, src, dest) - file = File.join(dest, file) - if (target = src).start_with?("/") - target = File.join([".."] * file.count("/"), src) - end - return unless File.exist?(File.expand_path(target, File.dirname(file))) - File.unlink(file) rescue nil - begin - File.symlink(target, file) - rescue - else - if $VERBOSE - $stdout.puts "linked #{target} to #{file}" - $stdout.flush - end - return - end - begin - src = src.sub(/\A\//, '') - File.copy_stream(src, file) - rescue - if $VERBOSE - $stdout.puts "failed to link #{src} to #{file}: #{$!.message}" - end - else - if $VERBOSE - $stdout.puts "copied #{src} to #{file}" - end - end - end - - def do_exec(command, dir, dest) - dir = dir ? File.join(dest, dir) : dest - if $VERBOSE - $stdout.puts "running #{command.dump} under #{dir}" - $stdout.flush - end - system(command, chdir: dir) or raise "failed #{command.dump}" - end - - def do_command(mode, dest, url, cache_dir, chksums) - extracted = false - base = /.*(?=\.tar(?:\.\w+)?\z)/ - - case mode - when :download - cache = do_download(url, cache_dir) - do_checksum(cache, chksums) - when :extract - cache = cache_file(url, cache_dir) - target = File.join(dest, File.basename(cache)[base]) - unless File.directory?(target) - do_checksum(cache, chksums) - extracted = do_extract(cache, dest) - end - when :all - cache = do_download(url, cache_dir) - target = File.join(dest, File.basename(cache)[base]) - unless File.directory?(target) - do_checksum(cache, chksums) - extracted = do_extract(cache, dest) - end - end - extracted - end - - def process(list) - mode = @mode - cache_dir = @cache_dir - after_extract = (mode == :all or mode == :patch) - success = true - if $VERBOSE - $stdout.puts "downloading for #{list}" - $stdout.flush - end - vars = Vars.new - extracted = false - dest = File.dirname(list) - url = chksums = nil - IO.foreach(list) do |line| - line.sub!(/\s*#.*/, '') - if /^(\w+)\s*=\s*(.*)/ =~ line - vars[$1] = vars.expand($2) - next - end - if chksums - chksums.concat(line.split) - elsif /^\t/ =~ line - if extracted and after_extract - patch, *args = line.split.map {|s| vars.expand(s)} - do_patch(dest, patch, args) - end - next - elsif /^!\s*(?:chdir:\s*([^|\s]+)\|\s*)?(.*)/ =~ line - if extracted and after_extract - command = vars.expand($2.strip) - chdir = $1 and chdir = vars.expand(chdir) - do_exec(command, chdir, dest) - end - next - elsif /->/ =~ line - if extracted and after_extract - link, file = $`.strip, $'.strip - do_link(vars.expand(link), vars.expand(file), dest) - end - next - else - url, *chksums = line.split(' ') - end - if chksums.last == '\\' - chksums.pop - next - end - unless url - chksums = nil - next - end - url = vars.expand(url) - begin - extracted = do_command(mode, dest, url, cache_dir, chksums) - rescue => e - warn e.full_message - success = false - end - url = chksums = nil - end - success - end - - def process_under(dir) - success = true - Dir.glob("#{dir}/**/extlibs") do |list| - success &= process(list) - end - success - end - - def self.run(argv) - cache_dir = nil - mode = :all - until argv.empty? - case argv[0] - when '--download' - mode = :download - when '--extract' - mode = :extract - when '--patch' - mode = :patch - when '--all' - mode = :all - when '--cache' - argv.shift - cache_dir = argv[0] - when /\A--cache=/ - cache_dir = $' - when '--' - argv.shift - break - when /\A-/ - warn "unknown option: #{argv[0]}" - return false - else - break - end - argv.shift - end - - extlibs = new(mode, cache_dir: cache_dir) - argv.inject(true) do |success, dir| - success & extlibs.process_under(dir) - end - end -end - -if $0 == __FILE__ - exit ExtLibs.run(ARGV) -end -- cgit v1.2.1 From 74b5bffa8644c9c3ca1063e61bd47704a6676216 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 5 Oct 2022 18:01:54 +0900 Subject: Split libyaml step --- .github/workflows/libyaml.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/libyaml.yml b/.github/workflows/libyaml.yml index b3fab15..8d87fa1 100644 --- a/.github/workflows/libyaml.yml +++ b/.github/workflows/libyaml.yml @@ -31,12 +31,12 @@ jobs: if: ${{ matrix.libyaml-prefix != '' }} - name: Install dependencies run: bundle install - - name: Compile - run: >- - rake compile -- - ${{ matrix.libyaml-prefix != '' && '--with-libyaml-dir=' - || '--with-libyaml-source-dir' - }}${{ matrix.libyaml-prefix }} + - name: Compile with libyaml + run: rake compile -- --with-libyaml-dir=${{ matrix.libyaml-prefix }} + if: ${{ matrix.libyaml-prefix != '' }} + - name: Compile with libyaml source + run: rake compile -- --with-libyaml-source-dir=yaml-${{ matrix.libyaml }} + if: ${{ matrix.libyaml-prefix == '' }} - name: Run test run: rake - name: Install gem -- cgit v1.2.1 From 6fe3861fee727d5ab21e06e944ad71ec6800ac4d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 5 Oct 2022 18:04:57 +0900 Subject: Always extract libyaml --- .github/workflows/libyaml.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/libyaml.yml b/.github/workflows/libyaml.yml index 8d87fa1..aea00eb 100644 --- a/.github/workflows/libyaml.yml +++ b/.github/workflows/libyaml.yml @@ -24,6 +24,8 @@ jobs: run: | wget https://pyyaml.org/download/libyaml/yaml-${{ matrix.libyaml }}.tar.gz tar xzf yaml-${{ matrix.libyaml }}.tar.gz + - name: Compile libyaml ${{ matrix.libyaml }} + run: | cd yaml-${{ matrix.libyaml }} ./configure --prefix=${{ matrix.libyaml-prefix }} make -- cgit v1.2.1 From c7612200daae976097a50ec815ae451f8372a71a Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 5 Oct 2022 18:12:49 +0900 Subject: keep src location for libyaml source --- .github/workflows/libyaml.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/libyaml.yml b/.github/workflows/libyaml.yml index aea00eb..ea25f57 100644 --- a/.github/workflows/libyaml.yml +++ b/.github/workflows/libyaml.yml @@ -37,7 +37,7 @@ jobs: run: rake compile -- --with-libyaml-dir=${{ matrix.libyaml-prefix }} if: ${{ matrix.libyaml-prefix != '' }} - name: Compile with libyaml source - run: rake compile -- --with-libyaml-source-dir=yaml-${{ matrix.libyaml }} + run: rake compile -- --with-libyaml-source-dir=$(pwd)/yaml-${{ matrix.libyaml }} if: ${{ matrix.libyaml-prefix == '' }} - name: Run test run: rake -- cgit v1.2.1