Auto merge of #3893 - bundler:reduce-ci-failures, r=segiddins

Try failing fast if prereq gems fail to install

One of the causes of builds that fail late is gems that didn't install but also didn't raise an error (until the tests that depend on them fail, much much later). This tries to fail those quickly, and maybe means that we can add some retry logic.

4 files changed, 72 insertions, 10 deletions

diff --git a/spec/realworld/edgecases_spec.rb b/spec/realworld/edgecases_spec.rb
index 028c30c293..008eab28d2 100644
--- a/spec/realworld/edgecases_spec.rb
+++ b/spec/realworld/edgecases_spec.rb
@@ -1,6 +1,6 @@
 require "spec_helper"
 
-describe "real world edgecases", :realworld => true do
+describe "real world edgecases", :realworld => true, :sometimes => true do
   # there is no rbx-relative-require gem that will install on 1.9
   it "ignores extra gems with bad platforms", :ruby => "~> 1.8.7" do
     install_gemfile <<-G
diff --git a/spec/realworld/parallel_spec.rb b/spec/realworld/parallel_spec.rb
index f168a0657e..7154e26dc9 100644
--- a/spec/realworld/parallel_spec.rb
+++ b/spec/realworld/parallel_spec.rb
@@ -1,6 +1,6 @@
 require "spec_helper"
 
-describe "parallel", :realworld => true do
+describe "parallel", :realworld => true, :sometimes => true do
   it "installs" do
     gemfile <<-G
       source "https://rubygems.org"
diff --git a/spec/support/rubygems_ext.rb b/spec/support/rubygems_ext.rb
index 5456ad2e90..a879955a05 100644
--- a/spec/support/rubygems_ext.rb
+++ b/spec/support/rubygems_ext.rb
@@ -13,15 +13,16 @@ module Spec
       unless File.exist?("#{Path.base_system_gems}")
         FileUtils.mkdir_p(Path.base_system_gems)
         puts "installing gems for the tests to use..."
-        `gem install fakeweb artifice --no-rdoc --no-ri`
-        `gem install sinatra --version 1.2.7 --no-rdoc --no-ri`
-        # Rake version has to be consistent for tests to pass
-        `gem install rake --version 10.0.2 --no-rdoc --no-ri`
-        # 3.0.0 breaks 1.9.2 specs
-        `gem install builder --version 2.1.2 --no-rdoc --no-ri`
-        `gem install rack --no-rdoc --no-ri`
+        %w[fakeweb artifice rack].each {|n| install_gem(n) }
+        {
+          "sinatra" => "1.2.7",
+          # Rake version has to be consistent for tests to pass
+          "rake" => "10.0.2",
+          # 3.0.0 breaks 1.9.2 specs
+          "builder" => "2.1.2"
+        }.each {|n, v| install_gem(n, v) }
         # ruby-graphviz is used by the viz tests
-        `gem install ruby-graphviz --no-rdoc --no-ri` if RUBY_VERSION >= "1.9.3"
+        install_gem("ruby-graphviz") if RUBY_VERSION >= "1.9.3"
       end
 
       ENV["HOME"] = Path.home.to_s
@@ -29,6 +30,12 @@ module Spec
       Gem::DefaultUserInteraction.ui = Gem::SilentUI.new
     end
 
+    def self.install_gem(name, version = nil)
+      cmd = "gem install #{name} --no-rdoc --no-ri"
+      cmd << " --version #{version}" if version
+      system(cmd) || raise("Installing gem #{name} for the tests to use failed!")
+    end
+
     def gem_command(command, args = "", options = {})
       if command == :exec && !options[:no_quote]
         args = args.gsub(/(?=")/, "\\")
@@ -37,5 +44,6 @@ module Spec
       lib  = File.join(File.dirname(__FILE__), "..", "..", "lib")
       %x{#{Gem.ruby} -I#{lib} -rubygems -S gem --backtrace #{command} #{args}}.strip
     end
+
   end
 end
diff --git a/spec/support/sometimes.rb b/spec/support/sometimes.rb
new file mode 100644
index 0000000000..99fc9e9444
--- /dev/null
+++ b/spec/support/sometimes.rb
@@ -0,0 +1,54 @@
+module Sometimes
+  def run_with_retries(example_to_run, retries)
+    example = RSpec.current_example
+    example.metadata[:retries] ||= retries
+
+    retries.times do |t|
+      example.metadata[:retried] = t + 1
+      example.instance_variable_set(:@exception, nil)
+      example_to_run.run
+      break unless example.exception
+    end
+
+    if e = example.exception
+      new_exception = e.exception(e.message + "[Retried #{retries} times]")
+      new_exception.set_backtrace e.backtrace
+      example.instance_variable_set(:@exception, new_exception)
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Sometimes
+  config.alias_example_to :sometimes, :sometimes => true
+  config.add_setting :sometimes_retry_count, :default => 5
+
+  config.around(:each, :sometimes => true) do |example|
+    retries = example.metadata[:retries] || RSpec.configuration.sometimes_retry_count
+    run_with_retries(example, retries)
+  end
+
+  config.after(:suite) do
+    message = Proc.new do |color, text|
+      colored = RSpec::Core::Formatters::ConsoleCodes.wrap(text, color)
+      notification = RSpec::Core::Notifications::MessageNotification.new(colored)
+      RSpec.configuration.formatters.first.message(notification)
+    end
+
+    retried_examples = RSpec.world.example_groups.map do |g|
+      g.descendants.map do |d|
+        d.filtered_examples.select do |e|
+          e.metadata[:sometimes] && e.metadata.fetch(:retried, 1) > 1
+        end
+      end
+    end.flatten
+
+    message.call(retried_examples.empty? ? :green : :yellow, "\n\nRetried examples: #{retried_examples.count}")
+
+    retried_examples.each do |e|
+      message.call(:cyan, "  #{e.full_description}")
+      path = RSpec::Core::Metadata.relative_path(e.location)
+      message.call(:cyan, "  [#{e.metadata[:retried]}/#{e.metadata[:retries]}] " + path)
+    end
+  end
+end