Merge #7017

7017: Development environment cleanup r=deivid-rodriguez a=deivid-rodriguez

### What was the end-user problem that led to this PR?

The problem was making changes in the development setup was brittle and led to hard to debug errors. For example, the ones in #6980. Also, it was hard to reproduce exactly what's going on in the current CI environment. Which is useful to setup other CI, or run the specs under a docker image.

### What was your diagnosis of the problem?

My diagnosis was that we can simplify a lot of this stuff.

### What is your fix for the problem, implemented in this PR?

My fix is... a lot of simplifications, but the most important ones being:

* Now each binstub activates only its own dependency, not every development dependency. That means a contributor can perfectly work on manual pages installing only `rake` and `ronn` without having to install all development dependencies. Same with styling (`rubocop`), or with specs (`rspec`).
* Error messages are better now.
* The rake tasks for each main `Rakefile` section (man / spec / vendor) have been extracted to their own files.
* Rake tasks now shell out to the proper binstub, so gem activation is only needed there.

### Why did you choose this fix out of the possible options?

I chose this fix because it simplifies a lot the development environment in my opinion. Also, specs now pass under a docker image!


Co-authored-by: David Rodríguez <deivid.rodriguez@riseup.net>

5 files changed, 154 insertions, 172 deletions

diff --git a/Rakefile b/Rakefile
index bfaf1d964d..364ba3ed1f 100644
--- a/Rakefile
+++ b/Rakefile
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 $:.unshift File.expand_path("../lib", __FILE__)
-require "shellwords"
 require "benchmark"
 
 NULL_DEVICE = (Gem.win_platform? ? "NUL" : "/dev/null")
@@ -15,13 +14,6 @@ def bundler_spec
   @bundler_spec ||= Gem::Specification.load("bundler.gemspec")
 end
 
-def safe_task(&block)
-  yield
-  true
-rescue StandardError
-  false
-end
-
 # Benchmark task execution
 module Rake
   class Task
@@ -36,7 +28,19 @@ module Rake
   end
 end
 
+desc "Run specs"
+task :spec do
+  sh("bin/rspec")
+end
+
 namespace :spec do
+  def safe_task(&block)
+    yield
+    true
+  rescue StandardError
+    false
+  end
+
   desc "Ensure spec dependencies are installed"
   task :deps do
     deps = Hash[bundler_spec.development_dependencies.map do |d|
@@ -88,167 +92,156 @@ namespace :spec do
       Rake::Task["spec:deps"].invoke
     end
   end
-end
 
-begin
-  rspec = bundler_spec.development_dependencies.find {|d| d.name == "rspec" }
-  gem "rspec", rspec.requirement.to_s
-  require "rspec/core/rake_task"
+  task :clean do
+    rm_rf "tmp"
+  end
 
-  desc "Run specs"
-  RSpec::Core::RakeTask.new
-  task :spec => "man:build"
+  desc "Run the real-world spec suite"
+  task :realworld => %w[set_realworld spec]
 
-  require "rubocop/rake_task"
-  rubocop = RuboCop::RakeTask.new
-  rubocop.options = ["--parallel"]
+  namespace :realworld do
+    desc "Re-record cassettes for the realworld specs"
+    task :record => %w[set_record realworld]
 
-  namespace :spec do
-    task :clean do
-      rm_rf "tmp"
+    task :set_record do
+      ENV["BUNDLER_SPEC_FORCE_RECORD"] = "TRUE"
     end
+  end
 
-    desc "Run the real-world spec suite"
-    task :realworld => %w[set_realworld spec]
-
-    namespace :realworld do
-      desc "Re-record cassettes for the realworld specs"
-      task :record => %w[set_record realworld]
-
-      task :set_record do
-        ENV["BUNDLER_SPEC_FORCE_RECORD"] = "TRUE"
-      end
-    end
+  task :set_realworld do
+    ENV["BUNDLER_REALWORLD_TESTS"] = "1"
+  end
 
-    task :set_realworld do
-      ENV["BUNDLER_REALWORLD_TESTS"] = "1"
-    end
+  desc "Run the spec suite with the sudo tests"
+  task :sudo => %w[set_sudo spec clean_sudo]
 
-    desc "Run the spec suite with the sudo tests"
-    task :sudo => %w[set_sudo spec clean_sudo]
+  task :set_sudo do
+    ENV["BUNDLER_SUDO_TESTS"] = "1"
+  end
 
-    task :set_sudo do
-      ENV["BUNDLER_SUDO_TESTS"] = "1"
-    end
+  task :clean_sudo do
+    puts "Cleaning up sudo test files..."
+    system "sudo rm -rf #{File.expand_path("../tmp/sudo_gem_home", __FILE__)}"
+  end
 
-    task :clean_sudo do
-      puts "Cleaning up sudo test files..."
-      system "sudo rm -rf #{File.expand_path("../tmp/sudo_gem_home", __FILE__)}"
-    end
+  # RubyGems specs by version
+  namespace :rubygems do
+    rubyopt = ENV["RUBYOPT"]
+    # When editing this list, also edit .travis.yml!
+    branches = %w[master]
+    releases = %w[v2.5.2 v2.6.14 v2.7.9 v3.0.3]
+    (branches + releases).each do |rg|
+      desc "Run specs with RubyGems #{rg}"
+      task rg do
+        sh("bin/rspec")
+      end
 
-    # RubyGems specs by version
-    namespace :rubygems do
-      rubyopt = ENV["RUBYOPT"]
-      # When editing this list, also edit .travis.yml!
-      branches = %w[master]
-      releases = %w[v2.5.2 v2.6.14 v2.7.9 v3.0.3]
-      (branches + releases).each do |rg|
-        desc "Run specs with RubyGems #{rg}"
-        RSpec::Core::RakeTask.new(rg) do |t|
-          t.rspec_opts = %w[--format progress --color]
-          t.ruby_opts  = %w[-w]
-        end
+      # Create tasks like spec:rubygems:v1.8.3:sudo to run the sudo specs
+      namespace rg do
+        task :sudo => ["set_sudo", rg, "clean_sudo"]
+        task :realworld => ["set_realworld", rg]
+      end
 
-        # Create tasks like spec:rubygems:v1.8.3:sudo to run the sudo specs
-        namespace rg do
-          task :sudo => ["set_sudo", rg, "clean_sudo"]
-          task :realworld => ["set_realworld", rg]
+      task "clone_rubygems_#{rg}" do
+        unless File.directory?(RUBYGEMS_REPO)
+          system("git clone https://github.com/rubygems/rubygems.git tmp/rubygems")
         end
-
-        task "clone_rubygems_#{rg}" do
-          unless File.directory?(RUBYGEMS_REPO)
-            system("git clone https://github.com/rubygems/rubygems.git tmp/rubygems")
-          end
-          hash = nil
-
-          if RUBYGEMS_REPO.start_with?(Dir.pwd)
-            Dir.chdir(RUBYGEMS_REPO) do
-              system("git remote update")
-              if rg == "master"
-                system("git checkout origin/master")
-              else
-                system("git checkout #{rg}") || raise("Unknown RubyGems ref #{rg}")
-              end
-              hash = `git rev-parse HEAD`.chomp
+        hash = nil
+
+        if RUBYGEMS_REPO.start_with?(Dir.pwd)
+          Dir.chdir(RUBYGEMS_REPO) do
+            system("git remote update")
+            if rg == "master"
+              system("git checkout origin/master")
+            else
+              system("git checkout #{rg}") || raise("Unknown RubyGems ref #{rg}")
             end
-          elsif rg != "master"
-            raise "need to be running against master with bundler as a submodule"
+            hash = `git rev-parse HEAD`.chomp
           end
-
-          puts "Checked out rubygems '#{rg}' at #{hash}"
-          ENV["RUBYOPT"] = "-I#{File.join(RUBYGEMS_REPO, "lib")} #{rubyopt}"
-          puts "RUBYOPT=#{ENV["RUBYOPT"]}"
+        elsif rg != "master"
+          raise "need to be running against master with bundler as a submodule"
         end
 
-        task rg => ["man:build", "clone_rubygems_#{rg}"]
-        task "rubygems:all" => rg
+        puts "Checked out rubygems '#{rg}' at #{hash}"
+        ENV["RGV"] = rg
       end
 
-      desc "Run specs under a RubyGems checkout (set RG=path)"
-      RSpec::Core::RakeTask.new("co") do |t|
-        t.rspec_opts = %w[--format documentation --color]
-        t.ruby_opts  = %w[-w]
-      end
+      task rg => ["clone_rubygems_#{rg}"]
+      task "rubygems:all" => rg
+    end
 
-      task "setup_co" do
-        rg = File.expand_path ENV["RG"]
-        puts "Running specs against RubyGems in #{rg}..."
-        ENV["RUBYOPT"] = "-I#{rg} #{rubyopt}"
-      end
+    desc "Run specs under a RubyGems checkout (set RG=path)"
+    task "co" do
+      sh("bin/rspec")
+    end
 
-      task "co" => "setup_co"
-      task "rubygems:all" => "co"
+    task "setup_co" do
+      rg = File.expand_path ENV["RG"]
+      puts "Running specs against RubyGems in #{rg}..."
+      ENV["RUBYOPT"] = "-I#{rg} #{rubyopt}"
     end
 
-    desc "Run the tests on Travis CI against a RubyGem version (using ENV['RGV'])"
-    task :travis do
-      rg = ENV["RGV"] || raise("RubyGems version is required on Travis!")
+    task "co" => "setup_co"
+    task "rubygems:all" => "co"
+  end
 
-      # disallow making network requests on CI
-      ENV["BUNDLER_SPEC_PRE_RECORDED"] = "TRUE"
+  desc "Run the tests on Travis CI against a RubyGem version (using ENV['RGV'])"
+  task :travis do
+    rg = ENV["RGV"] || raise("RubyGems version is required on Travis!")
 
-      puts "\n\e[1;33m[Travis CI] Running bundler specs against RubyGems #{rg}\e[m\n\n"
-      specs = safe_task { Rake::Task["spec:rubygems:#{rg}"].invoke }
+    # disallow making network requests on CI
+    ENV["BUNDLER_SPEC_PRE_RECORDED"] = "TRUE"
 
-      Rake::Task["spec:rubygems:#{rg}"].reenable
+    puts "\n\e[1;33m[Travis CI] Running bundler specs against RubyGems #{rg}\e[m\n\n"
+    specs = safe_task { Rake::Task["spec:rubygems:#{rg}"].invoke }
 
-      puts "\n\e[1;33m[Travis CI] Running bundler sudo specs against RubyGems #{rg}\e[m\n\n"
-      sudos = system("sudo -E rake spec:rubygems:#{rg}:sudo")
-      # clean up by chowning the newly root-owned tmp directory back to the travis user
-      system("sudo chown -R #{ENV["USER"]} #{File.join(File.dirname(__FILE__), "tmp")}")
+    Rake::Task["spec:rubygems:#{rg}"].reenable
 
-      Rake::Task["spec:rubygems:#{rg}"].reenable
+    puts "\n\e[1;33m[Travis CI] Running bundler sudo specs against RubyGems #{rg}\e[m\n\n"
+    sudos = system("sudo -E rake spec:rubygems:#{rg}:sudo")
+    # clean up by chowning the newly root-owned tmp directory back to the travis user
+    system("sudo chown -R #{ENV["USER"]} #{File.join(File.dirname(__FILE__), "tmp")}")
 
-      puts "\n\e[1;33m[Travis CI] Running bundler real world specs against RubyGems #{rg}\e[m\n\n"
-      realworld = safe_task { Rake::Task["spec:rubygems:#{rg}:realworld"].invoke }
+    Rake::Task["spec:rubygems:#{rg}"].reenable
 
-      { "specs" => specs, "sudo" => sudos, "realworld" => realworld }.each do |name, passed|
-        if passed
-          puts "\e[0;32m[Travis CI] #{name} passed\e[m"
-        else
-          puts "\e[0;31m[Travis CI] #{name} failed\e[m"
-        end
-      end
+    puts "\n\e[1;33m[Travis CI] Running bundler real world specs against RubyGems #{rg}\e[m\n\n"
+    realworld = safe_task { Rake::Task["spec:rubygems:#{rg}:realworld"].invoke }
 
-      unless specs && sudos && realworld
-        raise "Spec run failed, please review the log for more information"
+    { "specs" => specs, "sudo" => sudos, "realworld" => realworld }.each do |name, passed|
+      if passed
+        puts "\e[0;32m[Travis CI] #{name} passed\e[m"
+      else
+        puts "\e[0;31m[Travis CI] #{name} failed\e[m"
       end
     end
-  end
-rescue LoadError
-  task :spec do
-    abort "Run `rake spec:deps` to be able to run the specs"
-  end
 
-  task :rubocop do
-    abort "Run `rake spec:deps` to be able to run rubocop"
+    unless specs && sudos && realworld
+      raise "Spec run failed, please review the log for more information"
+    end
   end
 end
 
-begin
-  require "ronn"
+desc "Run RuboCop"
+task :rubocop do
+  sh("bin/rubocop --parallel")
+end
 
-  namespace :man do
+namespace :man do
+  ronn_dep = bundler_spec.development_dependencies.find do |dep|
+    dep.name == "ronn"
+  end
+
+  ronn_requirement = ronn_dep.requirement.to_s
+
+  begin
+    gem "ronn", ronn_requirement
+
+    require "ronn"
+  rescue LoadError
+    task(:require) { abort "We couln't activate ronn (#{ronn_requirement}). Try `gem install ronn:'#{ronn_requirement}'` to be able to release!" }
+    task(:build) { abort "We couln't activate ronn (#{ronn_requirement}). Try `gem install ronn:'#{ronn_requirement}'` to be able to build the help pages" }
+  else
     directory "man"
 
     index = []
@@ -302,11 +295,6 @@ begin
 
     task(:require) {}
   end
-rescue LoadError
-  namespace :man do
-    task(:require) { abort "Install the ronn gem to be able to release!" }
-    task(:build) { warn "Install the ronn gem to build the help pages" }
-  end
 end
 
 begin
@@ -376,6 +364,6 @@ end
 
 task :default => :spec
 
-Dir["task/*.{rb,rake}"].each(&method(:load))
+Dir["task/*.rake"].each(&method(:load))
 
 task :generate_files => Rake::Task.tasks.select {|t| t.name.start_with?("lib/bundler/generated") }
diff --git a/bin/rake b/bin/rake
index 47fcb49888..cf4131d5b0 100755
--- a/bin/rake
+++ b/bin/rake
@@ -5,17 +5,16 @@ load File.expand_path("../with_rubygems", __FILE__) if ENV["RGV"]
 
 require "rubygems"
 
-unless ARGV[0] == "spec:deps"
-  bundler_spec = Gem::Specification.load(File.expand_path("../../bundler.gemspec", __FILE__))
-  bundler_spec.dependencies.each do |dep|
-    begin
-      gem dep.name, dep.requirement
-    rescue Gem::LoadError => e
-      warn "#{e.message} (#{e.class})"
-    end
-  end
-
-  Gem.finish_resolve if Gem.respond_to?(:finish_resolve)
+bundler_spec = Gem::Specification.load(File.expand_path("../../bundler.gemspec", __FILE__))
+rake = bundler_spec.development_dependencies.find do |dep|
+  dep.name == "rake"
 end
 
-load Gem.bin_path("rake", "rake")
+rake_requirement = rake.requirement.to_s
+
+begin
+  gem "rake", rake_requirement
+  load Gem.bin_path("rake", "rake")
+rescue Gem::LoadError
+  warn "We couln't activate rake (#{rake_requirement}). Run `gem install rake:'#{rake_requirement}'`"
+end
diff --git a/bin/rspec b/bin/rspec
index 39109d66f2..a3980e6744 100755
--- a/bin/rspec
+++ b/bin/rspec
@@ -6,10 +6,15 @@ load File.expand_path("../with_rubygems", __FILE__) if ENV["RGV"]
 require "rubygems"
 
 bundler_spec = Gem::Specification.load(File.expand_path("../../bundler.gemspec", __FILE__))
-bundler_spec.dependencies.each do |dep|
-  gem dep.name, dep.requirement
+rspec = bundler_spec.development_dependencies.find do |dep|
+  dep.name == "rspec"
 end
 
-Gem.finish_resolve if Gem.respond_to?(:finish_resolve)
+rspec_requirement = rspec.requirement.to_s
 
-load Gem.bin_path("rspec-core", "rspec")
+begin
+  gem "rspec", rspec_requirement
+  load Gem.bin_path("rspec-core", "rspec")
+rescue Gem::LoadError
+  warn "We couln't activate rspec (#{rspec_requirement}). Try `gem install rspec:'#{rspec_requirement}'`"
+end
diff --git a/bin/rubocop b/bin/rubocop
index 8014aff639..ef86f084a9 100755
--- a/bin/rubocop
+++ b/bin/rubocop
@@ -6,12 +6,15 @@ load File.expand_path("../with_rubygems", __FILE__) if ENV["RGV"]
 require "rubygems"
 
 bundler_spec = Gem::Specification.load(File.expand_path("../../bundler.gemspec", __FILE__))
-bundler_spec.dependencies.each do |dep|
-  gem dep.name, dep.requirement
+rubocop = bundler_spec.dependencies.find do |dep|
+  dep.name == "rubocop"
 end
 
-gem "rubocop", "= 0.65.0"
+rubocop_requirement = rubocop.requirement.to_s
 
-Gem.finish_resolve if Gem.respond_to?(:finish_resolve)
-
-load Gem.bin_path("rubocop", "rubocop")
+begin
+  gem "rubocop", rubocop_requirement
+  load Gem.bin_path("rubocop", "rubocop")
+rescue Gem::LoadError
+  warn "We couln't activate rubocop (#{rubocop_requirement}). Try `gem install rubocop:'#{rubocop_requirement}'`"
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 92de792b1d..81c222ea6f 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -3,19 +3,6 @@
 $:.unshift File.expand_path("..", __FILE__)
 $:.unshift File.expand_path("../../lib", __FILE__)
 
-require "rubygems"
-
-begin
-  require File.expand_path("../support/path.rb", __FILE__)
-  spec = Gem::Specification.load(Spec::Path.gemspec.to_s)
-  rspec = spec.dependencies.find {|d| d.name == "rspec" }
-  gem "rspec", rspec.requirement.to_s
-  require "rspec"
-  require "diff/lcs"
-rescue LoadError
-  abort "Run rake spec:deps to install development dependencies"
-end
-
 require "bundler/psyched_yaml"
 require "bundler/vendored_fileutils"
 require "uri"