diff options
27 files changed, 538 insertions, 4 deletions
diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb new file mode 100644 index 00000000000..c470789a5bb --- /dev/null +++ b/app/controllers/projects/container_registry_controller.rb @@ -0,0 +1,27 @@ +class Projects::ContainerRegistryController < Projects::ApplicationController + before_action :authorize_read_container_registry! + before_action :authorize_update_container_registry!, only: [:destroy] + layout 'project' + + def index + @tags = container_registry_repository.tags + end + + def destroy + if tag.delete + redirect_to namespace_project_container_registry_index_path(project.namespace, project) + else + redirect_to namespace_project_container_registry_index_path(project.namespace, project), alert: 'Failed to remove tag' + end + end + + private + + def container_registry_repository + @container_registry_repository ||= project.container_registry_repository + end + + def tag + @tag ||= container_registry_repository[params[:id]] + end +end diff --git a/app/helpers/gitlab_routing_helper.rb b/app/helpers/gitlab_routing_helper.rb index f07eff3fb57..2ce2d4e694f 100644 --- a/app/helpers/gitlab_routing_helper.rb +++ b/app/helpers/gitlab_routing_helper.rb @@ -33,6 +33,10 @@ module GitlabRoutingHelper namespace_project_builds_path(project.namespace, project, *args) end + def project_container_registry_path(project, *args) + namespace_project_container_registry_index_path(project.namespace, project, *args) + end + def activity_project_path(project, *args) activity_namespace_project_path(project.namespace, project, *args) end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index e1ab78df69e..7113e28924b 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -152,6 +152,10 @@ module ProjectsHelper nav_tabs << :builds end + if can?(current_user, :read_container_registry, project) + nav_tabs << :container_registry + end + if can?(current_user, :admin_project, project) nav_tabs << :settings end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9c942a8f4e3..da19462f265 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -110,6 +110,10 @@ class Namespace < ActiveRecord::Base # Ensure old directory exists before moving it gitlab_shell.add_namespace(path_was) + if any_project_has_container_registry_tags? + raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') + end + if gitlab_shell.mv_namespace(path_was, path) Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) @@ -131,6 +135,10 @@ class Namespace < ActiveRecord::Base end end + def any_project_has_container_registry_tags? + projects.any?(&:has_container_registry_tags?) + end + def send_update_instructions projects.each do |project| project.send_move_instructions("#{path_was}/#{project.path}") diff --git a/app/models/project.rb b/app/models/project.rb index 6e85841db44..ed7719ed31c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -328,9 +328,25 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry_url - if container_registry_enabled? && Gitlab.config.registry.enabled - "#{Gitlab.config.registry.host_with_port}/#{path_with_namespace}" + def container_registry_repository + @container_registry_repository ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(path_with_namespace) + url = Gitlab.config.registry.api_url + host = Gitlab.config.registry.host + registry = ContainerRegistry::Registry.new(url, token: token, path: host) + registry[path_with_namespace] + end + end + + def container_registry_repository_url + if Gitlab.config.registry.enabled + "#{Gitlab.config.registry.host}/#{path_with_namespace}" + end + end + + def has_container_registry_tags? + if Gitlab.config.registry.enabled + container_registry_repository.tags.any? end end @@ -749,6 +765,11 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) + if has_container_registry_tags? + # we currently doesn't support renaming repository if it contains tags in container registry + raise Exception.new('Project cannot be renamed, because tags are present in its container registry') + end + if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index bbbc84475c8..69ad634c368 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -14,6 +14,17 @@ module Auth { token: authorized_token(scope).encoded } end + def self.full_access_token(*names) + registry = Gitlab.config.registry + token = ::JWT::RSAToken.new(registry.key) + token.issuer = registry.issuer + token.audience = AUDIENCE + token[:access] = names.map do |name| + { type: 'repository', name: name, actions: %w(pull push) } + end + token.encoded + end + private def authorized_token(*accesses) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 48a6131b444..f09072975c3 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -26,6 +26,10 @@ module Projects Project.transaction do project.destroy! + unless remove_registry_tags + raise_error('Failed to remove project container registry. Please try again or contact administrator') + end + unless remove_repository(repo_path) raise_error('Failed to remove project repository. Please try again or contact administrator') end @@ -59,6 +63,12 @@ module Projects end end + def remove_registry_tags + return true unless Gitlab.config.registry.enabled + + project.container_registry_repository.delete_tags + end + def raise_error(message) raise DestroyError.new(message) end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 111b3ec05ea..03b57dea51e 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -34,6 +34,11 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end + if project.has_container_registry_tags? + # we currently doesn't support renaming repository if it contains tags in container registry + raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') + end + project.expire_caches_before_rename(old_path) # Apply new namespace id and visibility level diff --git a/app/views/layouts/nav/_project.html.haml b/app/views/layouts/nav/_project.html.haml index 479bde33719..d3d715aad3b 100644 --- a/app/views/layouts/nav/_project.html.haml +++ b/app/views/layouts/nav/_project.html.haml @@ -46,6 +46,13 @@ Builds %span.count.builds_counter= number_with_delimiter(@project.builds.running_or_pending.count(:all)) + - if project_nav_tab? :container_registry + = nav_link(controller: %w(container_registry)) do + = link_to project_container_registry_path(@project), title: 'Container Registry', class: 'shortcuts-container-registry' do + = icon('hdd-o fw') + %span + Container Registry + - if project_nav_tab? :graphs = nav_link(controller: %w(graphs)) do = link_to namespace_project_graph_path(@project.namespace, @project, current_ref), title: 'Graphs', class: 'shortcuts-graphs' do diff --git a/app/views/projects/container_registry/_header_title.html.haml b/app/views/projects/container_registry/_header_title.html.haml new file mode 100644 index 00000000000..f1863c52a3e --- /dev/null +++ b/app/views/projects/container_registry/_header_title.html.haml @@ -0,0 +1 @@ +- header_title project_title(@project, "Container Registry", project_container_registry_path(@project)) diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml new file mode 100644 index 00000000000..bf816d109b6 --- /dev/null +++ b/app/views/projects/container_registry/_tag.html.haml @@ -0,0 +1,21 @@ +%tr.tag + %td + = escape_once(tag.name) + = clipboard_button(clipboard_text: "docker pull #{tag.path}") + %td + - if layer = tag.layers.first + %span.has-tooltip(title="#{layer.revision}") + = layer.short_revision + - else + \- + %td + = number_to_human_size(tag.total_size) + · + = pluralize(tag.layers.size, "layer") + %td + = time_ago_in_words(tag.created_at) + - if can?(current_user, :update_container_registry, @project) + %td.content + .controls.hidden-xs.pull-right + = link_to namespace_project_container_registry_path(@project.namespace, @project, tag.name), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml new file mode 100644 index 00000000000..990253719bf --- /dev/null +++ b/app/views/projects/container_registry/index.html.haml @@ -0,0 +1,40 @@ +- page_title "Container Registry" += render "header_title" + +%hr + +%ul.content-list + - if @tags.blank? + %li + .nothing-here-block No images in Container Registry for this project. + + .light.prepend-top-default + %p + A 'container image' is a snapshot of a container. + You can host your container images with GitLab. + %br + To start using container images hosted on GitLab you first need to login: + %pre + %code + docker login #{Gitlab.config.registry.host} + %br + Then you are free to create and upload a container image with build and push commands: + %pre + docker build -t #{escape_once(@project.container_registry_repository_url)} . + %br + docker push #{escape_once(@project.container_registry_repository_url)} + + - else + .table-holder + %table.table.tags + %thead + %tr + %th Name + %th Image ID + %th Size + %th Created + - if can?(current_user, :update_container_registry, @project) + %th + + - @tags.each do |tag| + = render 'tag', tag: tag
\ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index e1b72556098..09264b5493e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -693,6 +693,8 @@ Rails.application.routes.draw do end end + resources :container_registry, only: [:index, :destroy], constraints: { id: Gitlab::Regex.container_registry_reference_regex } + resources :milestones, constraints: { id: /\d+/ } do member do put :sort_issues diff --git a/doc/permissions/permissions.md b/doc/permissions/permissions.md index 6be5ea0b486..30f6c75e1cf 100644 --- a/doc/permissions/permissions.md +++ b/doc/permissions/permissions.md @@ -39,6 +39,7 @@ documentation](../workflow/add-user/add-user.md). | Cancel and retry builds | | | ✓ | ✓ | ✓ | | Create or update commit status | | | ✓ | ✓ | ✓ | | Update a container registry | | | ✓ | ✓ | ✓ | +| Remove a container registry images | | | ✓ | ✓ | ✓ | | Create new milestones | | | | ✓ | ✓ | | Add new team members | | | | ✓ | ✓ | | Push to protected branches | | | | ✓ | ✓ | diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb new file mode 100644 index 00000000000..d59792a383e --- /dev/null +++ b/lib/container_registry/blob.rb @@ -0,0 +1,54 @@ +module ContainerRegistry + class Blob + attr_reader :repository, :config + + def initialize(repository, config) + @repository = repository + @config = config || {} + end + + def valid? + digest.present? + end + + def path + "#{repository.path}@#{digest}" + end + + def digest + config['digest'] + end + + def type + config['mediaType'] + end + + def size + config['size'] + end + + def revision + digest.split(':')[1] + end + + def short_revision + revision[0..8] + end + + def client + @client ||= repository.client + end + + def delete + client.delete_blob(repository.name, digest) + end + + def data + @data ||= client.blob(repository.name, digest, type) + end + + def mount_to(to_repository) + client.repository_mount_blob(to_repository.name, digest, repository.name) + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb new file mode 100644 index 00000000000..c250a4b6946 --- /dev/null +++ b/lib/container_registry/client.rb @@ -0,0 +1,69 @@ +require 'faraday' +require 'faraday_middleware' + +module ContainerRegistry + class Client + attr_accessor :uri + + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @faraday = Faraday.new(@base_uri) do |conn| + initialize_connection(conn, options) + end + end + + def repository_tags(name) + @faraday.get("/v2/#{name}/tags/list").body + end + + def repository_manifest(name, reference) + @faraday.get("/v2/#{name}/manifests/#{reference}").body + end + + def put_repository_manifest(name, reference, manifest) + @faraday.put("/v2/#{name}/manifests/#{reference}", manifest, { "Content-Type" => MANIFEST_VERSION }).success? + end + + def repository_mount_blob(name, digest, from) + @faraday.post("/v2/#{name}/blobs/uploads/?mount=#{digest}&from=#{from}").status == 201 + end + + def repository_tag_digest(name, reference) + response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response.headers['docker-content-digest'] if response.success? + end + + def delete_repository_tag(name, reference) + @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + end + + def blob(name, digest, type = nil) + headers = {} + headers['Accept'] = type if type + @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + end + + def delete_blob(name, digest) + @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + end + + private + + def initialize_connection(conn, options) + conn.request :json + conn.headers['Accept'] = MANIFEST_VERSION + + conn.response :json, content_type: /\bjson$/ + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + end +end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb new file mode 100644 index 00000000000..626b36cbaa9 --- /dev/null +++ b/lib/container_registry/config.rb @@ -0,0 +1,15 @@ +module ContainerRegistry + class Config + attr_reader :tag, :blob, :data + + def initialize(tag, blob) + @tag, @blob = tag, blob + @data = JSON.parse(blob.data) + end + + def [](key) + return unless data + data[key] + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb new file mode 100644 index 00000000000..a86ddb9326a --- /dev/null +++ b/lib/container_registry/registry.rb @@ -0,0 +1,15 @@ +module ContainerRegistry + class Registry + attr_reader :uri, :client, :path + + def initialize(uri, options = {}) + @path = uri || options[:path] + @uri = URI.parse(uri) + @client = ContainerRegistry::Client.new(uri, options) + end + + def [](name) + ContainerRegistry::Repository.new(self, name) + end + end +end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb new file mode 100644 index 00000000000..b30cb527b60 --- /dev/null +++ b/lib/container_registry/repository.rb @@ -0,0 +1,61 @@ +module ContainerRegistry + class Repository + attr_reader :registry, :name + + def initialize(registry, name) + @registry, @name = registry, name + end + + def client + @client ||= registry.client + end + + def path + [registry.path, name].compact.join('/') + end + + def [](tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + return @manifest if defined?(@manifest) + @manifest = client.repository_tags(name) + end + + def valid? + manifest.present? + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def delete_tags + return unless tags + + tags.each(:delete) + end + + def mount_blob(blob) + return unless blob + + client.repository_mount_blob(name, blob.digest, blob.repository.name) + end + + def mount_manifest(tag, manifest) + client.put_repository_manifest(name, tag, manifest) + end + + def copy_to(other_repository) + tags.all? do |tag| + tag.copy_to(other_repository) + end + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb new file mode 100644 index 00000000000..14cee8be889 --- /dev/null +++ b/lib/container_registry/tag.rb @@ -0,0 +1,79 @@ +module ContainerRegistry + class Tag + attr_reader :repository, :name + + def initialize(repository, name) + @repository, @name = repository, name + end + + def valid? + manifest.present? + end + + def manifest + return @manifest if defined?(@manifest) + @manifest = client.repository_manifest(repository.name, name) + end + + def path + "#{repository.path}:#{name}" + end + + def [](key) + return unless manifest + manifest[key] + end + + def digest + return @digest if defined?(@digest) + @digest = client.repository_tag_digest(repository.name, name) + end + + def config_blob + return @config_blob if defined?(@config_blob) + return unless manifest && manifest['config'] + @config_blob = ContainerRegistry::Blob.new(repository, manifest['config']) + end + + def config + return unless config_blob + @config ||= ContainerRegistry::Config.new(self, config_blob) + end + + def created_at + return unless config + @created_at ||= DateTime.rfc3339(config['created']) + end + + def layers + return @layers if defined?(@layers) + return unless manifest + @layers = manifest['layers'].map do |layer| + ContainerRegistry::Blob.new(repository, layer) + end + end + + def total_size + return unless layers + layers.map(&:size).sum + end + + def delete + return unless digest + client.delete_repository_tag(repository.name, digest) + end + + def copy_to(repository) + return unless manifest + layers.each do |blob| + repository.mount_blob(blob) + end + repository.mount_blob(config_blob) + repository.mount_manifest(name, manifest.to_json) + end + + def client + @client ||= repository.client + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ace906a6f59..1cbd6d945a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,5 +96,9 @@ module Gitlab (?<![\/.]) (?# rule #6-7) }x.freeze end + + def container_registry_reference_regex + git_reference_regex + end end end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb new file mode 100644 index 00000000000..be5910e4abb --- /dev/null +++ b/spec/features/container_registry_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +describe "Container Registry" do + let(:project) { create(:empty_project) } + let(:repository) { project.container_registry_repository } + let(:tag_name) { 'latest' } + let(:tags) { [tag_name] } + let(:registry_settings) do + { + enabled: true + } + end + + before do + login_as(:user) + project.team << [@user, :developer] + stub_container_registry(*tags) + allow(Gitlab.config.registry).to receive_messages(registry_settings) + allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') + end + + describe 'GET /:project/container_registry' do + before do + visit namespace_project_container_registry_index_path(project.namespace, project) + end + + context 'when no tags' do + let(:tags) { [] } + + it { expect(page).to have_content('No images in Container Registry for this project') } + end + + context 'when there are tags' do + it { expect(page).to have_content(tag_name)} + end + end + + describe 'DELETE /:project/container_registry/tag' do + before do + visit namespace_project_container_registry_index_path(project.namespace, project) + end + + it do + expect_any_instance_of(::ContainerRegistry::Tag).to receive(:delete).and_return(true) + + click_on 'Remove' + end + end +end diff --git a/spec/fixtures/container_registry/config_blob.json b/spec/fixtures/container_registry/config_blob.json new file mode 100644 index 00000000000..1028c994a24 --- /dev/null +++ b/spec/fixtures/container_registry/config_blob.json @@ -0,0 +1 @@ +{"architecture":"amd64","config":{"Hostname":"b14cd8298755","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"b14cd82987550b01af9a666a2f4c996280a6152e66873134fae5a0f223dc5976","container_config":{"Hostname":"b14cd8298755","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":["/bin/sh","-c","#(nop) ADD file:033ab063740d9ff4dcfb1c69eccf25f91d88729f57cd5a73050e014e3e094aa0 in /"],"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"created":"2016-04-01T20:53:00.160300546Z","docker_version":"1.9.1","history":[{"created":"2016-04-01T20:53:00.160300546Z","created_by":"/bin/sh -c #(nop) ADD file:033ab063740d9ff4dcfb1c69eccf25f91d88729f57cd5a73050e014e3e094aa0 in /"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:c56b7dabbc7aa730eeab07668bdcbd7e3d40855047ca9a0cc1bfed23a2486111"]}} diff --git a/spec/fixtures/container_registry/tag_manifest.json b/spec/fixtures/container_registry/tag_manifest.json new file mode 100644 index 00000000000..1b6008e2872 --- /dev/null +++ b/spec/fixtures/container_registry/tag_manifest.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/octet-stream","size":1145,"digest":"sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":2319870,"digest":"sha256:420890c9e918b6668faaedd9000e220190f2493b0693ee563ebd7b4cc754a57d"}]} diff --git a/spec/requests/ci/api/runners_spec.rb b/spec/requests/ci/api/runners_spec.rb index db8189ffb79..43f9fe89c8e 100644 --- a/spec/requests/ci/api/runners_spec.rb +++ b/spec/requests/ci/api/runners_spec.rb @@ -7,7 +7,6 @@ describe Ci::API::API do let(:registration_token) { 'abcdefg123456' } before do - stub_gitlab_calls stub_application_setting(runners_registration_token: registration_token) end diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index a2937368136..bae576f1670 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -147,7 +147,20 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'a forbidden' end end + end + + context 'for project without container registry' do + let(:project) { create(:empty_project, :public, container_registry_enabled: false) } + + before { project.update(container_registry_enabled: false) } + context 'disallow when pulling' do + let(:current_params) do + { scope: "repository:#{project.path_with_namespace}:pull" } + end + + it_behaves_like 'a forbidden' + end end end diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index b5ca34bc028..2c31cbe3faf 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -25,6 +25,18 @@ module StubGitlabCalls allow_any_instance_of(Project).to receive(:builds_enabled?).and_return(false) end + def stub_container_registry(*tags) + allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_tags).and_return( + { "tags" => tags } + ) + allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_manifest).and_return( + JSON.load(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json')) + ) + allow_any_instance_of(ContainerRegistry::Client).to receive(:blob).and_return( + File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json') + ) + end + private def gitlab_url |