diff options
author | Douglas Barbosa Alexandre <dbalexandre@gmail.com> | 2019-05-02 15:28:54 +0000 |
---|---|---|
committer | Douglas Barbosa Alexandre <dbalexandre@gmail.com> | 2019-05-02 15:28:54 +0000 |
commit | 61c2f575922623ce0befe2d41c3127f227f2a5ab (patch) | |
tree | 04c51d947cc3e32a78e0f2bd6f5e80a72155f6e2 | |
parent | e51b3e4997a01c2e0e2a95820adb0e4b173cd981 (diff) | |
parent | 8195a94fb993ea28ec6d6150843dd720f7ac08ab (diff) | |
download | gitlab-ce-61c2f575922623ce0befe2d41c3127f227f2a5ab.tar.gz |
Merge branch 'dz-registry-proxy' into 'master'
Dependency proxy for containers
See merge request gitlab-org/gitlab-ee!9750
15 files changed, 625 insertions, 0 deletions
diff --git a/ee/app/controllers/groups/dependency_proxy_for_containers_controller.rb b/ee/app/controllers/groups/dependency_proxy_for_containers_controller.rb new file mode 100644 index 00000000000..e5fb6bc795d --- /dev/null +++ b/ee/app/controllers/groups/dependency_proxy_for_containers_controller.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +class Groups::DependencyProxyForContainersController < Groups::ApplicationController + include SendFileUpload + + before_action :ensure_feature_enabled! + before_action :ensure_token_granted! + + attr_reader :token + + def manifest + result = DependencyProxy::PullManifestService.new(image, tag, token).execute + + if result[:status] == :success + render json: result[:manifest] + else + render status: result[:http_status], json: result[:message] + end + end + + def blob + result = DependencyProxy::FindOrCreateBlobService + .new(group, image, token, params[:sha]).execute + + if result[:status] == :success + send_upload(result[:blob].file) + else + head result[:http_status] + end + end + + private + + def image + params[:image] + end + + def tag + params[:tag] + end + + def ensure_feature_enabled! + render_404 unless Gitlab.config.dependency_proxy.enabled && + group.feature_available?(:dependency_proxy) && + group.dependency_proxy_setting&.enabled + end + + def ensure_token_granted! + result = DependencyProxy::RequestTokenService.new(image).execute + + if result[:status] == :success + @token = result[:token] + else + render status: result[:http_status], json: result[:message] + end + end +end diff --git a/ee/app/services/dependency_proxy/base_service.rb b/ee/app/services/dependency_proxy/base_service.rb new file mode 100644 index 00000000000..1b2d4b14a27 --- /dev/null +++ b/ee/app/services/dependency_proxy/base_service.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module DependencyProxy + class BaseService < ::BaseService + private + + def registry + DependencyProxy::Registry + end + + def auth_headers + { + Authorization: "Bearer #{@token}" + } + end + end +end diff --git a/ee/app/services/dependency_proxy/download_blob_service.rb b/ee/app/services/dependency_proxy/download_blob_service.rb new file mode 100644 index 00000000000..3c690683bf6 --- /dev/null +++ b/ee/app/services/dependency_proxy/download_blob_service.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module DependencyProxy + class DownloadBlobService < DependencyProxy::BaseService + class DownloadError < StandardError + attr_reader :http_status + + def initialize(message, http_status) + @http_status = http_status + + super(message) + end + end + + def initialize(image, blob_sha, token) + @image = image + @blob_sha = blob_sha + @token = token + @temp_file = Tempfile.new + end + + def execute + File.open(@temp_file.path, "wb") do |file| + Gitlab::HTTP.get(blob_url, headers: auth_headers, stream_body: true) do |fragment| + if [301, 302, 307].include?(fragment.code) + # do nothing + elsif fragment.code == 200 + file.write(fragment) + else + raise DownloadError.new('Non-success response code on downloading blob fragment', fragment.code) + end + end + end + + success(file: @temp_file) + rescue DownloadError => exception + error(exception.message, exception.http_status) + rescue Timeout::Error => exception + error(exception.message, 599) + end + + private + + def blob_url + registry.blob_url(@image, @blob_sha) + end + end +end diff --git a/ee/app/services/dependency_proxy/find_or_create_blob_service.rb b/ee/app/services/dependency_proxy/find_or_create_blob_service.rb new file mode 100644 index 00000000000..bd06f9d7628 --- /dev/null +++ b/ee/app/services/dependency_proxy/find_or_create_blob_service.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module DependencyProxy + class FindOrCreateBlobService < DependencyProxy::BaseService + def initialize(group, image, token, blob_sha) + @group = group + @image = image + @token = token + @blob_sha = blob_sha + end + + def execute + file_name = @blob_sha.sub('sha256:', '') + '.gz' + blob = @group.dependency_proxy_blobs.find_or_build(file_name) + + unless blob.persisted? + result = DependencyProxy::DownloadBlobService + .new(@image, @blob_sha, @token).execute + + if result[:status] == :error + log_failure(result) + + return error('Failed to download the blob', result[:http_status]) + end + + blob.file = result[:file] + blob.size = result[:file].size + blob.save! + end + + success(blob: blob) + end + + private + + def log_failure(result) + log_error( + "Dependency proxy: Failed to download the blob." \ + "Blob sha: #{@blob_sha}." \ + "Error message: #{result[:message][0, 100]}" \ + "HTTP status: #{result[:http_status]}" + ) + end + end +end diff --git a/ee/app/services/dependency_proxy/pull_manifest_service.rb b/ee/app/services/dependency_proxy/pull_manifest_service.rb new file mode 100644 index 00000000000..fc54ef85c96 --- /dev/null +++ b/ee/app/services/dependency_proxy/pull_manifest_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module DependencyProxy + class PullManifestService < DependencyProxy::BaseService + def initialize(image, tag, token) + @image = image + @tag = tag + @token = token + end + + def execute + response = Gitlab::HTTP.get(manifest_url, headers: auth_headers) + + if response.success? + success(manifest: response.body) + else + error(response.body, response.code) + end + rescue Timeout::Error => exception + error(exception.message, 599) + end + + private + + def manifest_url + registry.manifest_url(@image, @tag) + end + end +end diff --git a/ee/app/services/dependency_proxy/request_token_service.rb b/ee/app/services/dependency_proxy/request_token_service.rb new file mode 100644 index 00000000000..021251c798b --- /dev/null +++ b/ee/app/services/dependency_proxy/request_token_service.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module DependencyProxy + class RequestTokenService < DependencyProxy::BaseService + def initialize(image) + @image = image + end + + def execute + response = Gitlab::HTTP.get(auth_url) + + if response.success? + success(token: JSON.parse(response.body)['token']) + else + error('Expected 200 response code for an access token', response.code) + end + rescue Timeout::Error => exception + error(exception.message, 599) + rescue JSON::ParserError + error('Failed to parse a response body for an access token', 500) + end + + private + + def auth_url + registry.auth_url(@image) + end + end +end diff --git a/ee/changelogs/unreleased/dz-registry-proxy.yml b/ee/changelogs/unreleased/dz-registry-proxy.yml new file mode 100644 index 00000000000..150d5670e2c --- /dev/null +++ b/ee/changelogs/unreleased/dz-registry-proxy.yml @@ -0,0 +1,5 @@ +--- +title: Add dependency proxy for containers +merge_request: 9750 +author: +type: added diff --git a/ee/config/routes/group.rb b/ee/config/routes/group.rb index eceeb1b214c..7964c0eb8e8 100644 --- a/ee/config/routes/group.rb +++ b/ee/config/routes/group.rb @@ -114,3 +114,11 @@ constraints(::Constraints::GroupUrlConstrainer.new) do :pipeline_quota, :hooks, :boards) end end + +# Dependency proxy for containers +# Because docker adds v2 prefix to URI this need to be outside of usual group routes +scope constraints: { format: nil } do + get 'v2', to: proc { [200, {}, ['']] } + get 'v2/*group_id/dependency_proxy/containers/:image/manifests/*tag' => 'groups/dependency_proxy_for_containers#manifest' + get 'v2/*group_id/dependency_proxy/containers/:image/blobs/:sha' => 'groups/dependency_proxy_for_containers#blob' +end diff --git a/ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb b/ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb new file mode 100644 index 00000000000..7711a89438c --- /dev/null +++ b/ee/spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Groups::DependencyProxyForContainersController do + let(:group) { create(:group) } + let(:token_response) { { status: :success, token: 'abcd1234' } } + + before do + allow(Gitlab.config.dependency_proxy) + .to receive(:enabled).and_return(true) + + allow_any_instance_of(DependencyProxy::RequestTokenService) + .to receive(:execute).and_return(token_response) + end + + describe 'GET #manifest' do + let(:manifest) { { foo: 'bar' }.to_json } + let(:pull_response) { { status: :success, manifest: manifest } } + + before do + allow_any_instance_of(DependencyProxy::PullManifestService) + .to receive(:execute).and_return(pull_response) + end + + context 'feature enabled' do + before do + enable_dependency_proxy + end + + context 'remote token request fails' do + let(:token_response) do + { + status: :error, + http_status: 503, + message: 'Service Unavailable' + } + end + + it 'proxies status from the remote token request' do + get_manifest + + expect(response).to have_gitlab_http_status(503) + expect(response.body).to eq('Service Unavailable') + end + end + + context 'remote manifest request fails' do + let(:pull_response) do + { + status: :error, + http_status: 400, + message: '' + } + end + + it 'proxies status from the remote manifest request' do + get_manifest + + expect(response).to have_gitlab_http_status(400) + expect(response.body).to be_empty + end + end + + it 'returns 200 with manifest file' do + get_manifest + + expect(response).to have_gitlab_http_status(200) + expect(response.body).to eq(manifest) + end + end + + it 'returns 404 when feature is disabled' do + get_manifest + + expect(response).to have_gitlab_http_status(404) + end + + def get_manifest + get :manifest, params: { group_id: group.to_param, image: 'alpine', tag: '3.9.2' } + end + end + + describe 'GET #blob' do + let(:blob) { create(:dependency_proxy_blob) } + let(:blob_sha) { blob.file_name.sub('.gz', '') } + let(:blob_response) { { status: :success, blob: blob } } + + before do + allow_any_instance_of(DependencyProxy::FindOrCreateBlobService) + .to receive(:execute).and_return(blob_response) + end + + context 'feature enabled' do + before do + enable_dependency_proxy + end + + context 'remote blob request fails' do + let(:blob_response) do + { + status: :error, + http_status: 400, + message: '' + } + end + + it 'proxies status from the remote blob request' do + get_blob + + expect(response).to have_gitlab_http_status(400) + expect(response.body).to be_empty + end + end + + it 'sends a file' do + expect(controller).to receive(:send_file).with(blob.file.path, {}) + + get_blob + end + + it 'returns Content-Disposition: attachment' do + get_blob + + expect(response).to have_gitlab_http_status(:ok) + expect(response.headers['Content-Disposition']).to match(/^attachment/) + end + end + + it 'returns 404 when feature is disabled' do + get_blob + + expect(response).to have_gitlab_http_status(404) + end + + def get_blob + get :blob, params: { group_id: group.to_param, image: 'alpine', sha: blob_sha } + end + end + + def enable_dependency_proxy + stub_licensed_features(dependency_proxy: true) + group.create_dependency_proxy_setting!(enabled: true) + end +end diff --git a/ee/spec/routing/group_routing_spec.rb b/ee/spec/routing/group_routing_spec.rb index 22779b8798e..8ad1e32f2c2 100644 --- a/ee/spec/routing/group_routing_spec.rb +++ b/ee/spec/routing/group_routing_spec.rb @@ -87,4 +87,20 @@ describe 'Group routing', "routing" do expect(get('/groups/gitlabhq/-/security/vulnerabilities/history')).to route_to('groups/security/vulnerabilities#history', group_id: 'gitlabhq') end end + + describe 'dependency proxy for containers' do + before do + allow(Group).to receive(:find_by_full_path).with('gitlabhq', any_args).and_return(true) + end + + it 'routes to #manifest' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/manifests/2.3.6')) + .to route_to('groups/dependency_proxy_for_containers#manifest', group_id: 'gitlabhq', image: 'ruby', tag: '2.3.6') + end + + it 'routes to #blob' do + expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/blobs/abc12345')) + .to route_to('groups/dependency_proxy_for_containers#blob', group_id: 'gitlabhq', image: 'ruby', sha: 'abc12345') + end + end end diff --git a/ee/spec/services/dependency_proxy/download_blob_service_spec.rb b/ee/spec/services/dependency_proxy/download_blob_service_spec.rb new file mode 100644 index 00000000000..97ed44fb90a --- /dev/null +++ b/ee/spec/services/dependency_proxy/download_blob_service_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DependencyProxy::DownloadBlobService do + include EE::DependencyProxyHelpers + + let(:image) { 'alpine' } + let(:token) { Digest::SHA256.hexdigest('123') } + let(:blob_sha) { Digest::SHA256.hexdigest('ruby:2.3.9') } + + subject { described_class.new(image, blob_sha, token).execute } + + context 'remote request is successful' do + before do + stub_blob_download(image, blob_sha) + end + + it { expect(subject[:status]).to eq(:success) } + it { expect(subject[:file]).to be_a(Tempfile) } + it { expect(subject[:file].size).to eq(6) } + end + + context 'remote request is not found' do + before do + stub_blob_download(image, blob_sha, 404) + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(404) } + it { expect(subject[:message]).to eq('Non-success response code on downloading blob fragment') } + end + + context 'net timeout exception' do + before do + blob_url = DependencyProxy::Registry.blob_url(image, blob_sha) + + stub_request(:get, blob_url).to_timeout + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(599) } + it { expect(subject[:message]).to eq('execution expired') } + end +end diff --git a/ee/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb b/ee/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb new file mode 100644 index 00000000000..b5a99d02173 --- /dev/null +++ b/ee/spec/services/dependency_proxy/find_or_create_blob_service_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DependencyProxy::FindOrCreateBlobService do + include EE::DependencyProxyHelpers + + let(:blob) { create(:dependency_proxy_blob) } + let(:group) { blob.group } + let(:image) { 'alpine' } + let(:tag) { '3.9' } + let(:token) { Digest::SHA256.hexdigest('123') } + let(:blob_sha) { '40bd001563085fc35165329ea1ff5c5ecbdbbeef' } + + subject { described_class.new(group, image, token, blob_sha).execute } + + before do + stub_registry_auth(image, token) + end + + context 'no cache' do + before do + stub_blob_download(image, blob_sha) + end + + it 'downloads blob from remote registry if there is no cached one' do + expect(subject[:status]).to eq(:success) + expect(subject[:blob]).to be_a(DependencyProxy::Blob) + expect(subject[:blob]).to be_persisted + end + end + + context 'cached blob' do + let(:blob_sha) { blob.file_name.sub('.gz', '') } + + it 'uses cached blob instead of downloading one' do + expect(subject[:status]).to eq(:success) + expect(subject[:blob]).to be_a(DependencyProxy::Blob) + expect(subject[:blob]).to eq(blob) + end + end + + context 'no such blob exists remotely' do + before do + stub_blob_download(image, blob_sha, 404) + end + + it 'returns error message and http status' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq('Failed to download the blob') + expect(subject[:http_status]).to eq(404) + end + end +end diff --git a/ee/spec/services/dependency_proxy/pull_manifest_service_spec.rb b/ee/spec/services/dependency_proxy/pull_manifest_service_spec.rb new file mode 100644 index 00000000000..808fdd35ccb --- /dev/null +++ b/ee/spec/services/dependency_proxy/pull_manifest_service_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DependencyProxy::PullManifestService do + include EE::DependencyProxyHelpers + + let(:image) { 'alpine' } + let(:tag) { '3.9' } + let(:token) { Digest::SHA256.hexdigest('123') } + let(:manifest) { { foo: 'bar' }.to_json } + + subject { described_class.new(image, tag, token).execute } + + context 'remote request is successful' do + before do + stub_manifest_download(image, tag) + end + + it { expect(subject[:status]).to eq(:success) } + it { expect(subject[:manifest]).to eq(manifest) } + end + + context 'remote request is not found' do + before do + stub_manifest_download(image, tag, 404, 'Not found') + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(404) } + it { expect(subject[:message]).to eq('Not found') } + end + + context 'net timeout exception' do + before do + manifest_link = DependencyProxy::Registry.manifest_url(image, tag) + + stub_request(:get, manifest_link).to_timeout + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(599) } + it { expect(subject[:message]).to eq('execution expired') } + end +end diff --git a/ee/spec/services/dependency_proxy/request_token_service_spec.rb b/ee/spec/services/dependency_proxy/request_token_service_spec.rb new file mode 100644 index 00000000000..247c238f667 --- /dev/null +++ b/ee/spec/services/dependency_proxy/request_token_service_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DependencyProxy::RequestTokenService do + include EE::DependencyProxyHelpers + + let(:image) { 'alpine:3.9' } + let(:token) { Digest::SHA256.hexdigest('123') } + + subject { described_class.new(image).execute } + + context 'remote request is successful' do + before do + stub_registry_auth(image, token) + end + + it { expect(subject[:status]).to eq(:success) } + it { expect(subject[:token]).to eq(token) } + end + + context 'remote request is not found' do + before do + stub_registry_auth(image, token, 404) + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(404) } + it { expect(subject[:message]).to eq('Expected 200 response code for an access token') } + end + + context 'failed to parse response body' do + before do + stub_registry_auth(image, token, 200, 'dasd1321: wow') + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(500) } + it { expect(subject[:message]).to eq('Failed to parse a response body for an access token') } + end + + context 'net timeout exception' do + before do + auth_link = DependencyProxy::Registry.auth_url(image) + + stub_request(:any, auth_link).to_timeout + end + + it { expect(subject[:status]).to eq(:error) } + it { expect(subject[:http_status]).to eq(599) } + it { expect(subject[:message]).to eq('execution expired') } + end +end diff --git a/ee/spec/support/helpers/ee/dependency_proxy_helpers.rb b/ee/spec/support/helpers/ee/dependency_proxy_helpers.rb new file mode 100644 index 00000000000..b27313b319d --- /dev/null +++ b/ee/spec/support/helpers/ee/dependency_proxy_helpers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module EE + module DependencyProxyHelpers + def stub_registry_auth(image, token, status = 200, body = nil) + auth_body = { 'token' => token }.to_json + auth_link = registry.auth_url(image) + + stub_request(:get, auth_link) + .to_return(status: status, body: body || auth_body) + end + + def stub_manifest_download(image, tag, status = 200, body = nil) + manifest_url = registry.manifest_url(image, tag) + + stub_request(:get, manifest_url) + .to_return(status: status, body: body || manifest) + end + + def stub_blob_download(image, blob_sha, status = 200, body = '123456') + download_link = registry.blob_url(image, blob_sha) + + stub_request(:get, download_link) + .to_return(status: status, body: body) + end + + private + + def registry + @registry ||= DependencyProxy::Registry + end + end +end |