Merge pull request #4040 from bundler/original-source-header

Send X-Gemfile-Source header to source mirrors

10 files changed, 172 insertions, 4 deletions

diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml
index b674f399bb..7269f36893 100644
--- a/.rubocop_todo.yml
+++ b/.rubocop_todo.yml
@@ -256,6 +256,7 @@ Style/RescueModifier:
     - 'lib/bundler/fetcher.rb'
     - 'lib/bundler/resolver.rb'
     - 'spec/realworld/dependency_api_spec.rb'
+    - 'spec/realworld/gemfile_source_header_spec.rb'
 
 # Offense count: 1
 # Configuration parameters: Methods.
diff --git a/lib/bundler.rb b/lib/bundler.rb
index d222befc98..f4373662ba 100644
--- a/lib/bundler.rb
+++ b/lib/bundler.rb
@@ -3,6 +3,7 @@ require "pathname"
 require "rbconfig"
 require "thread"
 require "bundler/gem_path_manipulation"
+require "bundler/gem_remote_fetcher"
 require "bundler/rubygems_ext"
 require "bundler/rubygems_integration"
 require "bundler/version"
diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb
index c283dcef7b..868f415198 100644
--- a/lib/bundler/fetcher.rb
+++ b/lib/bundler/fetcher.rb
@@ -229,6 +229,7 @@ module Bundler
 
         con.read_timeout = Fetcher.api_timeout
         con.override_headers["User-Agent"] = user_agent
+        con.override_headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
         con
       end
     end
diff --git a/lib/bundler/gem_remote_fetcher.rb b/lib/bundler/gem_remote_fetcher.rb
new file mode 100644
index 0000000000..77cab0c4ad
--- /dev/null
+++ b/lib/bundler/gem_remote_fetcher.rb
@@ -0,0 +1,41 @@
+require "rubygems/remote_fetcher"
+
+module Bundler
+  # Adds support for setting custom HTTP headers when fetching gems from the
+  # server.
+  #
+  # TODO Get rid of this when and if gemstash only supports RubyGems versions
+  # that contain https://github.com/rubygems/rubygems/commit/3db265cc20b2f813.
+  class GemRemoteFetcher < Gem::RemoteFetcher
+    attr_accessor :headers
+
+    # Extracted from RubyGems 2.4.
+    def fetch_http(uri, last_modified = nil, head = false, depth = 0)
+      fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get
+      # beginning of change
+      response   = request uri, fetch_type, last_modified do |req|
+        headers.each {|k, v| req.add_field(k, v) } if headers
+      end
+      # end of change
+
+      case response
+      when Net::HTTPOK, Net::HTTPNotModified then
+        response.uri = uri if response.respond_to? :uri
+        head ? response : response.body
+      when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther,
+           Net::HTTPTemporaryRedirect then
+        raise FetchError.new("too many redirects", uri) if depth > 10
+
+        location = URI.parse response["Location"]
+
+        if https?(uri) && !https?(location)
+          raise FetchError.new("redirecting to non-https resource: #{location}", uri)
+        end
+
+        fetch_http(location, last_modified, head, depth + 1)
+      else
+        raise FetchError.new("bad response #{response.message} #{response.code}", uri)
+      end
+    end
+  end
+end
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index b5ae60d5ed..9750d98bb3 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -586,7 +586,8 @@ module Bundler
         uri = Bundler.settings.mirror_for(uri)
         proxy = configuration[:http_proxy]
         dns = Resolv::DNS.new
-        fetcher = Gem::RemoteFetcher.new(proxy, dns)
+        fetcher = Bundler::GemRemoteFetcher.new(proxy, dns)
+        fetcher.headers = { "X-Gemfile-Source" => spec.remote.original_uri.to_s } if spec.remote.original_uri
         fetcher.download(spec, uri, path)
       end
 
diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb
index 1e1a932d30..561bde194e 100644
--- a/lib/bundler/source/rubygems/remote.rb
+++ b/lib/bundler/source/rubygems/remote.rb
@@ -2,11 +2,12 @@ module Bundler
   class Source
     class Rubygems
       class Remote
-        attr_reader :uri,
-          :anonymized_uri
+        attr_reader :uri, :anonymized_uri, :original_uri
 
         def initialize(uri)
+          orig_uri = uri
           uri = Bundler.settings.mirror_for(uri)
+          @original_uri = orig_uri if orig_uri != uri
           fallback_auth = Bundler.settings.credentials_for(uri)
 
           @uri = apply_auth(uri, fallback_auth).freeze
diff --git a/spec/bundler/fetcher_spec.rb b/spec/bundler/fetcher_spec.rb
index 65bf1017e0..e21601555d 100644
--- a/spec/bundler/fetcher_spec.rb
+++ b/spec/bundler/fetcher_spec.rb
@@ -2,7 +2,10 @@ require "spec_helper"
 require "bundler/fetcher"
 
 describe Bundler::Fetcher do
-  subject(:fetcher) { Bundler::Fetcher.new(double("remote", :uri => URI("https://example.com"))) }
+  let(:uri) { URI("https://example.com") }
+  let(:remote) { double("remote", :uri => uri, :original_uri => nil) }
+
+  subject(:fetcher) { Bundler::Fetcher.new(remote) }
 
   before do
     allow(Bundler).to receive(:root) { Pathname.new("root") }
@@ -32,6 +35,33 @@ describe Bundler::Fetcher do
         end
       end
     end
+
+    context "when a rubygems source mirror is set" do
+      let(:orig_uri) { URI("http://zombo.com") }
+      let(:remote_with_mirror) do
+        double("remote", :uri => uri, :original_uri => orig_uri, :anonymized_uri => uri)
+      end
+
+      let(:fetcher) { Bundler::Fetcher.new(remote_with_mirror) }
+
+      it "sets the 'X-Gemfile-Source' header containing the original source" do
+        expect(
+          fetcher.send(:connection).override_headers["X-Gemfile-Source"]
+        ).to eq("http://zombo.com")
+      end
+    end
+
+    context "when there is no rubygems source mirror set" do
+      let(:remote_no_mirror) do
+        double("remote", :uri => uri, :original_uri => nil, :anonymized_uri => uri)
+      end
+
+      let(:fetcher) { Bundler::Fetcher.new(remote_no_mirror) }
+
+      it "does not set the 'X-Gemfile-Source' header" do
+        expect(fetcher.send(:connection).override_headers["X-Gemfile-Source"]).to be_nil
+      end
+    end
   end
 
   describe "#user_agent" do
diff --git a/spec/bundler/source/rubygems/remote_spec.rb b/spec/bundler/source/rubygems/remote_spec.rb
index cec243bfb1..2842077498 100644
--- a/spec/bundler/source/rubygems/remote_spec.rb
+++ b/spec/bundler/source/rubygems/remote_spec.rb
@@ -82,6 +82,10 @@ describe Bundler::Source::Rubygems::Remote do
     specify "#anonymized_uri returns the mirror URI without credentials" do
       expect(remote(uri).anonymized_uri).to eq(mirror_uri_no_auth)
     end
+
+    specify "#original_uri returns the original source" do
+      expect(remote(uri).original_uri).to eq(uri)
+    end
   end
 
   context "when a mirror with configured credentials is configured for the URI" do
@@ -101,5 +105,17 @@ describe Bundler::Source::Rubygems::Remote do
     specify "#anonymized_uri returns the mirror URI without credentials" do
       expect(remote(uri).anonymized_uri).to eq(mirror_uri_no_auth)
     end
+
+    specify "#original_uri returns the original source" do
+      expect(remote(uri).original_uri).to eq(uri)
+    end
+  end
+
+  context "when there is no mirror set" do
+    describe "#original_uri" do
+      it "is not set" do
+        expect(remote(uri_no_auth).original_uri).to be_nil
+      end
+    end
   end
 end
diff --git a/spec/realworld/gemfile_source_header_spec.rb b/spec/realworld/gemfile_source_header_spec.rb
new file mode 100644
index 0000000000..aff127db6c
--- /dev/null
+++ b/spec/realworld/gemfile_source_header_spec.rb
@@ -0,0 +1,63 @@
+require "spec_helper"
+require "thread"
+
+describe "fetching dependencies with a mirrored source", :rubygems => ">= 2.0" do
+  let(:mirror) { "https://server.example.org" }
+  let(:original) { "http://127.0.0.1:#{@port}" }
+
+  before do
+    setup_server
+    bundle "config --local mirror.#{mirror} #{original}"
+  end
+
+  after { @t.kill }
+
+  it "sets the 'X-Gemfile-Source' header and bundles successfully" do
+    gemfile <<-G
+      source "#{mirror}"
+      gem 'weakling'
+    G
+
+    bundle :install
+
+    expect(out).to include("Installing weakling")
+    expect(out).to include("Bundle complete")
+    should_be_installed "weakling 0.0.3"
+  end
+
+  private
+
+  def setup_server
+    # need to hack, so we can require rack
+    old_gem_home = ENV["GEM_HOME"]
+    ENV["GEM_HOME"] = Spec::Path.base_system_gems.to_s
+    require "rack"
+    ENV["GEM_HOME"] = old_gem_home
+
+    @port = 21_459
+    @port += 1 while TCPSocket.new("127.0.0.1", @port) rescue false
+    @server_uri = "http://127.0.0.1:#{@port}"
+
+    require File.expand_path("../../support/artifice/endpoint_mirror_source", __FILE__)
+
+    @t = Thread.new {
+      Rack::Server.start(:app       => EndpointMirrorSource,
+                         :Host      => "0.0.0.0",
+                         :Port      => @port,
+                         :server    => "webrick",
+                         :AccessLog => [])
+    }.run
+
+    wait_for_server(@port)
+  end
+
+  def wait_for_server(port, seconds = 15)
+    tries = 0
+    sleep 0.5
+    TCPSocket.new("127.0.0.1", port)
+  rescue => e
+    raise(e) if tries > (seconds * 2)
+    tries += 1
+    retry
+  end
+end
diff --git a/spec/support/artifice/endpoint_mirror_source.rb b/spec/support/artifice/endpoint_mirror_source.rb
new file mode 100644
index 0000000000..39cd3656fa
--- /dev/null
+++ b/spec/support/artifice/endpoint_mirror_source.rb
@@ -0,0 +1,13 @@
+require File.expand_path("../endpoint", __FILE__)
+
+class EndpointMirrorSource < Endpoint
+  get "/gems/:id" do
+    if request.env["HTTP_X_GEMFILE_SOURCE"] == "https://server.example.org/"
+      File.read("#{gem_repo1}/gems/#{params[:id]}")
+    else
+      halt 500
+    end
+  end
+end
+
+Artifice.activate_with(EndpointMirrorSource)